From fec7b1b39da7c370b21b182facdfe98a6d966074 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 13:18:27 +0800 Subject: [PATCH 001/163] chore(nimble): requires "nim < 0.20" --- npython.nimble | 1 + 1 file changed, 1 insertion(+) diff --git a/npython.nimble b/npython.nimble index 64ad528..5f4ea8c 100644 --- a/npython.nimble +++ b/npython.nimble @@ -4,3 +4,4 @@ description = "(Subset of) Python programming language implemented in Nim" license = "CPython license" requires "cligen", "regex" +requires "nim < 0.20" \ No newline at end of file From a8d565952bb52e06b7cf27c0d3ed500848a57db5 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 10 Jul 2025 20:05:25 +0800 Subject: [PATCH 002/163] fix(nimc): nnkTupleConstr used to be nnkPar --- Python/asdl.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/asdl.nim b/Python/asdl.nim index c348e4b..f21810f 100644 --- a/Python/asdl.nim +++ b/Python/asdl.nim @@ -284,7 +284,7 @@ macro genAsdlTypes(inputTree: untyped): untyped = for child in inputTree: let parentName = "Asdl" & getDefName(child[0]) let right = child[1] - expectKind(right, nnkPar) + expectKind(right, {nnkPar, nnkTupleConstr}) # generate asdl tokens result.add(genAsdlToken(right, parentName)) @@ -300,7 +300,7 @@ macro genAsdlTypes(inputTree: untyped): untyped = for child in inputTree: let parentName = "Asdl" & getDefName(child[0]) let right = child[1] - expectKind(right, nnkPar) + expectKind(right, {nnkPar, nnkTupleConstr}) # generate ast types for subType in right: result.add( From c69dedd7a1c90d1632c2cf18e799f64f0080bec9 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 09:44:29 +0800 Subject: [PATCH 003/163] fix(nimc): get rid of `Error: illegal capture 'selfNoCast' ... ... because 'addPyIntObjectMagic' has the calling convention: ` See comment of `deepCopy` for details --- Objects/pyobject.nim | 10 ++++++---- Utils/macroutils.nim | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 Utils/macroutils.nim diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 967297a..6bd0b8c 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -9,7 +9,7 @@ import strutils import hashes import tables -import ../Utils/utils +import ../Utils/[utils, macroutils] import pyobjectBase export macros except name @@ -95,18 +95,20 @@ proc registerBltinMethod*(t: PyTypeObject, name: string, fun: BltinMethod) = # assert self type then cast macro castSelf*(ObjectType: untyped, code: untyped): untyped = + let selfNoCastId = code.params[1][0] + selfNoCastId.expectIdent "selfNoCast" code.body = newStmtList( nnkCommand.newTree( ident("assert"), nnkInfix.newTree( ident("of"), - ident("selfNoCast"), + selfNoCastId, ObjectType ) ), newLetStmt( ident("self"), - newCall(ObjectType, ident("selfNoCast")) + newCall(ObjectType, selfNoCastId) ), code.body ) @@ -314,7 +316,7 @@ proc implMethod*(prototype, ObjectType, pragmas, body: NimNode, kind: MethodKind ident("*"), # let other modules call without having to lookup in the type dict name, ), - params, + params.deepCopy, body, # the function body ) # add pragmas, the last to add is the first to execute diff --git a/Utils/macroutils.nim b/Utils/macroutils.nim new file mode 100644 index 0000000..423af11 --- /dev/null +++ b/Utils/macroutils.nim @@ -0,0 +1,24 @@ + +import std/macros +proc deepCopy*(s: seq[NimNode]): seq[NimNode] = + ##[To get rid of compile-error caused by Nim's backward non-compatibility change: + +Error: illegal capture 'selfNoCast' because 'addPyIntObjectMagic' has the calling convention: + +Bacause for all procs generated by `implMethod`, whose 1st param is `selfNoCast` ( +take it for example, so for other params), + +if not `deepCopy`, +their `selfNoCast` symbol was shared (there's only one instance), +so its loc would be updated each time `implMethod` is called. + +Finally `selfNoCast` would be considered to be placed at the place + where the last `implMethod` was called. +So for all `implMethod`-ed procs except the last, + their `selfNoCast` were mistakely considered as `captured` + +Old Nim compiler didn't behave so. + ]## + result = newSeq[NimNode](s.len) + for i, e in s: + result[i] = s[i].copyNimTree From 2b5efb6de930f59c756001d866bb562c387741b8 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 09:46:17 +0800 Subject: [PATCH 004/163] fix(nimc): type mismatch in Python/symtable Python/symtable.nim(149, 10) Error: type mismatch Expression: add(toVisit, (astRoot, SymTableEntry(nil))) [1] toVisit: seq[(AstNodeBase, SymTableEntry)] [2] (astRoot, SymTableEntry(nil)): (Asdlmodl, SymTableEntry) --- Python/symtable.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/symtable.nim b/Python/symtable.nim index d7c6898..3659000 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -146,7 +146,7 @@ proc freeVarsToSeq*(ste: SymTableEntry): seq[PyStrObject] = proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = var toVisit: seq[(AstNodeBase, SymTableEntry)] - toVisit.add((astRoot, nil)) + toVisit.add((AstNodeBase astRoot, SymTableEntry nil)) while toVisit.len != 0: let (astNode, parentSte) = toVisit.pop let ste = newSymTableEntry(parentSte) From 00e3909476f412d8a293a7c6eb7099e1dacfe6a8 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 10:04:34 +0800 Subject: [PATCH 005/163] fix(nimc): when gen repr for exceptions: `Error: got prototype: OpenSymChoice 29 "repr"` --- Objects/pyobject.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 6bd0b8c..c893a23 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -246,6 +246,9 @@ macro checkArgTypes*(nameAndArg, code: untyped): untyped = # works with thingks like `append(obj: PyObject)` # if no parenthesis, then return nil as argTypes, means do not check arg type proc getNameAndArgTypes*(prototype: NimNode): (NimNode, NimNode) = + var prototype = prototype + if prototype.kind == nnkOpenSymChoice: + prototype = prototype[0] # we only care its strVal, so pick from any if prototype.kind == nnkIdent or prototype.kind == nnkSym: return (prototype, nil) let argTypes = nnkPar.newTree() From dab5900b6cadab5247ed77dcfa620ff503c16ef7 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 10:07:49 +0800 Subject: [PATCH 006/163] fix(nimc): downcast in tuple now must be explicit? --- Objects/exceptionsImpl.nim | 2 +- Python/neval.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/exceptionsImpl.nim b/Objects/exceptionsImpl.nim index fdd93aa..976991b 100644 --- a/Objects/exceptionsImpl.nim +++ b/Objects/exceptionsImpl.nim @@ -72,5 +72,5 @@ proc isExceptionType*(obj: PyObject): bool = proc fromBltinSyntaxError*(e: SyntaxError, fileName: PyStrObject): PyExceptionObject = let excpObj = newSyntaxError(e.msg) # don't have code name - excpObj.traceBacks.add (fileName, nil, e.lineNo, e.colNo) + excpObj.traceBacks.add (PyObject fileName, PyObject nil, e.lineNo, e.colNo) excpObj diff --git a/Python/neval.nim b/Python/neval.nim index 70e0be4..2325e28 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -100,7 +100,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = proc setTraceBack(excp: PyExceptionObject) = let lineNo = f.code.lineNos[lastI] # for exceptions happened when evaluating the frame no colNo is set - excp.traceBacks.add (f.code.fileName, f.code.codeName, lineNo, -1) + excp.traceBacks.add (PyObject f.code.fileName, PyObject f.code.codeName, lineNo, -1) # in future, should get rid of the abstraction of seq and use a dynamically # created buffer directly. This can reduce time cost of the core neval function From 53253bc8a533b5c549e386e8d30813d47ee5326d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 11:17:43 +0800 Subject: [PATCH 007/163] fix(nimc): `Error: cannot bind another '=destroy' to: PyCodeObject:ObjectType`;impr... ...rm dup opCodes,opArgs data attr of PyCodeObject --- Objects/codeobject.nim | 28 ++-------------------------- Objects/pyobject.nim | 14 -------------- Python/neval.nim | 13 +------------ 3 files changed, 3 insertions(+), 52 deletions(-) diff --git a/Objects/codeobject.nim b/Objects/codeobject.nim index 70df3ca..7a0cb7b 100644 --- a/Objects/codeobject.nim +++ b/Objects/codeobject.nim @@ -11,8 +11,6 @@ type declarePyType Code(tpToken): # for convenient and not performance critical accessing code: seq[(OpCode, OpArg)] - opCodes: ptr OpCode # array of opcodes with `length`, same with `code` - opArgs: ptr OpArg # array of args with `length`, same with `code` lineNos: seq[int] constants: seq[PyObject] @@ -32,38 +30,16 @@ declarePyType Code(tpToken): # most attrs of code objects are set in compile.nim proc newPyCode*(codeName, fileName: PyStrObject, length: int): PyCodeObject = - when defined(js): - result = newPyCodeSimple() - else: - proc finalizer(obj: PyCodeObject) = - dealloc(obj.opCodes) - dealloc(obj.opArgs) - - newPyCodeFinalizer(result, finalizer) - result.opCodes = createU(OpCode, length) - result.opArgs = createU(OpArg, length) + result = newPyCodeSimple() + result.code = newSeqOfCap[(OpCode, OpArg)] length result.codeName = codeName result.fileName = fileName proc len*(code: PyCodeObject): int {. inline .} = code.code.len -template `[]`*(opCodes: ptr OpCode, idx: int): OpCode = - cast[ptr OpCode](cast[int](opCodes) + idx * sizeof(OpCode))[] - -template `[]`*(opArgs: ptr OpArg, idx: int): OpArg = - cast[ptr OpArg](cast[int](opArgs) + idx * sizeof(OpArg))[] - -template `[]=`(opCodes: ptr OpCode, idx: int, value: OpCode) = - cast[ptr OpCode](cast[int](opCodes) + idx * sizeof(OpCode))[] = value - -template `[]=`(opArgs: ptr OpArg, idx: int, value: OpArg) = - cast[ptr OpArg](cast[int](opArgs) + idx * sizeof(OpArg))[] = value - proc addOpCode*(code: PyCodeObject, instr: tuple[opCode: OpCode, opArg: OpArg, lineNo: int]) = - code.opCodes[code.len] = instr.opCode - code.opArgs[code.len] = instr.opArg code.code.add((instr.opCode, instr.opArg)) code.lineNos.add(instr.lineNo) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index c893a23..491138f 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -557,9 +557,6 @@ macro declarePyType*(prototype, fields: untyped): untyped = proc `ofPy name Object`*(obj: PyObject): bool {. cdecl, inline .} = obj.pyType.kind == PyTypeToken.`name` - # base constructor that should be used for any custom constructors except for - # code objects which require finalizers. - # make it public so that impl files can also use proc `newPy name Simple`*: `Py name Object` {. cdecl .}= # use `result` here seems to be buggy let obj = new `Py name Object` @@ -570,17 +567,6 @@ macro declarePyType*(prototype, fields: untyped): untyped = obj.dict = newPyDict() obj - # using function argument as a finalizer is buggy in nim 0.19.0, - # so use a template as workaround, gh-10376 - template `newPy name Finalizer`*( - obj: var `Py name Object`, - finalizer: proc (x: `Py name Object`) {. nimcall .}) = - new(obj, finalizer) - obj.pyType = `py name ObjectType` - when hasDict: - obj.dict = newPyDict() - - # default for __new__ hook, could be overrided at any time proc `newPy name Default`(args: seq[PyObject]): PyObject {. cdecl .} = `newPy name Simple`() diff --git a/Python/neval.nim b/Python/neval.nim index 2325e28..200005c 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -76,23 +76,12 @@ proc evalFrame*(f: PyFrameObject): PyObject = # instructions are fetched so frequently that we should build a local cache # instead of doing tons of dereference - when not defined(js): - let opCodes = f.code.opCodes - let opArgs = f.code.opArgs - var lastI = -1 # instruction helpers - var opCode: OpCode - var opArg: OpArg template fetchInstr: (OpCode, OpArg) = inc lastI - when defined(js): - f.code.code[lastI] - else: - opCode = opCodes[lastI] - opArg = opArgs[lastI] - (opCode, opArg) + f.code.code[lastI] template jumpTo(i: int) = lastI = i - 1 From b645cb63bb28a20887dc572329141896ed0093dc Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 12:09:34 +0800 Subject: [PATCH 008/163] fix(nimc): get rid of `Error: illegal capture 'args'... followup HEAD~5 Error: illegal capture 'args' because 'appendPyListObjectMethod' has the calling convention: --- Python/bltinmodule.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 6510448..f99c572 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -3,7 +3,7 @@ import strformat import neval import builtindict import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject] -import ../Utils/[utils, compat] +import ../Utils/[utils, macroutils, compat] proc registerBltinFunction(name: string, fun: BltinFunc) = @@ -27,7 +27,7 @@ macro implBltinFunc*(prototype, pyName, body: untyped): untyped = ident("*"), # let other modules call without having to lookup in the bltindict name, ), - bltinFuncParams, + bltinFuncParams.deepCopy, body, # the function body ) From 43fc39d7700689c58ad1c27622819a7903f08ccb Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 12:18:19 +0800 Subject: [PATCH 009/163] fix(rt): grammar:newGrammarNode: assignment to discriminant changes object branch... ...compile with -d:nimOldCaseObjects for a transition period [FieldDefect] --- Parser/grammar.nim | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Parser/grammar.nim b/Parser/grammar.nim index 9c97d7f..78cb6ac 100644 --- a/Parser/grammar.nim +++ b/Parser/grammar.nim @@ -90,20 +90,23 @@ proc matchToken*(node: GrammarNode, token: Token): bool let successGrammarNode* = newGrammarNode("s") # sentinel proc newGrammarNode(name: string, tokenString=""): GrammarNode = - new result - result.epsilonSet = initSet[GrammarNode]() case name[0] of 'A'..'H', '+', '?', '*': - result.kind = name[0] + result = GrammarNode(kind: name[0]) of 'a': - result.kind = 'a' - result.token = strTokenMap[tokenString] - result.nextSet = initSet[GrammarNode]() + result = GrammarNode( + kind: 'a', + token: strTokenMap[tokenString], + nextSet: initSet[GrammarNode](), + ) of 's': # finish sentinel - result.kind = 's' - result.nextSet = initSet[GrammarNode]() + result = GrammarNode( + kind: 's', + nextSet: initSet[GrammarNode](), + ) else: raise newException(ValueError, fmt"unknown name: {name}") + result.epsilonSet = initSet[GrammarNode]() # not to confuse with token terminator From e6bcaf3a083146c6822572b43284988ee72a1fd9 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 13:01:48 +0800 Subject: [PATCH 010/163] fix(rt): lexer:newTokenNode(followup HEAD^) --- Parser/lexer.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 150ad7e..9afae02 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -53,17 +53,17 @@ proc newTokenNode*(token: Token, new result if token == Token.Name and content in reserveNameSet: try: - result.token = strTokenMap[content] + result = TokenNode(token: strTokenMap[content]) except KeyError: unreachable else: - result.token = token case token of contentTokenSet: assert content != "" - result.content = content + result = TokenNode(token: token, content: content) else: assert content == "" + result = TokenNode(token: token) assert result.token != Token.NULLTOKEN if result.token.isTerminator: assert -1 < lineNo and -1 < colNo From 296461ae4c60f5eda7f7d363fc25b76eaac0da98 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 13:02:25 +0800 Subject: [PATCH 011/163] fix(rt): grammar:genExpsilonSet: the length of the HashSet changed while iterating over it --- Parser/grammar.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Parser/grammar.nim b/Parser/grammar.nim index 78cb6ac..763efe2 100644 --- a/Parser/grammar.nim +++ b/Parser/grammar.nim @@ -219,7 +219,8 @@ proc genEpsilonSet(root: GrammarNode) = allChildrenCollected = false break if allChildrenCollected: - for child in curNode.epsilonSet: + let epsilonSetView = curNode.epsilonSet + for child in epsilonSetView: curNode.epsilonSet.incl(child.epsilonSet) collected.incl(curNode) else: From a7e26a9279adc289976ef20011f35990f458993f Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 13:11:16 +0800 Subject: [PATCH 012/163] fix(rt): multimethods is off since 0.20 --- Python/compile.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Python/compile.nim b/Python/compile.nim index 92db730..b569842 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -382,15 +382,15 @@ macro compileMethod(astNodeName, funcDef: untyped): untyped = newEmptyNode(), nnkFormalParams.newTree( newEmptyNode(), + newIdentDefs( + ident("astNode"), + ident("Ast" & $astNodeName) + ), newIdentDefs( ident("c"), ident("Compiler") + ), ), - newIdentDefs( - ident("astNode"), - ident("Ast" & $astNodeName) - ) - ), newEmptyNode(), newEmptyNode(), funcdef, @@ -402,12 +402,15 @@ template compileSeq(c: Compiler, s: untyped) = c.compile(astNode) # todo: too many dispachers here! used astNode token to dispatch (if have spare time...) -method compile(c: Compiler, astNode: AstNodeBase) {.base.} = +method compile(astNode: AstNodeBase, c: Compiler) {.base.} = echo "!!!WARNING, ast node compile method not implemented" echo astNode echo "###WARNING, ast node compile method not implemented" # let it compile, the result shown is better for debugging +template compile*(c: Compiler, astNode: AstNodeBase) = + bind compile + compile(astNode, c) compileMethod Module: c.compileSeq(astNode.body) From 347937c29d6309335659e1766c44c9369534e7da Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 14:16:55 +0800 Subject: [PATCH 013/163] chore(nimble): requires "nim >= 1.6.14" --- npython.nimble | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/npython.nimble b/npython.nimble index 5f4ea8c..86c1338 100644 --- a/npython.nimble +++ b/npython.nimble @@ -2,6 +2,9 @@ version = "0.1.0" author = "Weitang Li" description = "(Subset of) Python programming language implemented in Nim" license = "CPython license" +srcDir = "Python" +bin = @["python"] +binDir = "bin" requires "cligen", "regex" -requires "nim < 0.20" \ No newline at end of file +requires "nim >= 1.6.14" # 2.* (at least till 2.3.1) is okey, too. From 472ac5f919282c4061ef90db8d61f0c53b205ffb Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 14:17:25 +0800 Subject: [PATCH 014/163] chore(gitignore): /bin/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3e9bbbf..bb14d23 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ trash/ ./Objects/test.nim __pycache__/ +/bin/ \ No newline at end of file From 5a90c4aa7220ba42e6101bfd47b6db3b7b45ca48 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 14:57:20 +0800 Subject: [PATCH 015/163] fix(warning): [Deprecated] in system/stdlib:... fileExists,initSet,ospaths,OutOfMemError,TaintedString --- Objects/listobject.nim | 2 +- Parser/grammar.nim | 16 ++++++++-------- Parser/lexer.nim | 4 ++-- Parser/parser.nim | 2 +- Parser/token.nim | 2 +- Python/cpython.nim | 2 +- Python/lifecycle.nim | 4 ++-- Python/neval.nim | 6 +++--- Python/symtable.nim | 4 ++-- Utils/compat.nim | 4 ++-- Utils/utils.nim | 10 +++++----- 11 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index b139357..123eadf 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -165,7 +165,7 @@ implListMethod remove(target: PyObject), [mutable: write]: return retObj assert retObj.ofPyIntObject let idx = PyIntObject(retObj).toInt - self.items.delete(idx, idx+1) + self.items.delete(idx .. idx+1) implListMagic init: diff --git a/Parser/grammar.nim b/Parser/grammar.nim index 763efe2..665d8d1 100644 --- a/Parser/grammar.nim +++ b/Parser/grammar.nim @@ -97,16 +97,16 @@ proc newGrammarNode(name: string, tokenString=""): GrammarNode = result = GrammarNode( kind: 'a', token: strTokenMap[tokenString], - nextSet: initSet[GrammarNode](), + nextSet: initHashSet[GrammarNode](), ) of 's': # finish sentinel result = GrammarNode( kind: 's', - nextSet: initSet[GrammarNode](), + nextSet: initHashSet[GrammarNode](), ) else: raise newException(ValueError, fmt"unknown name: {name}") - result.epsilonSet = initSet[GrammarNode]() + result.epsilonSet = initHashSet[GrammarNode]() # not to confuse with token terminator @@ -134,7 +134,7 @@ proc childTerminator(node: GrammarNode): GrammarNode = return childTerminator(node.children[0]) proc nextInTree(node: GrammarNode): HashSet[GrammarNode] = - result = initSet[GrammarNode]() + result = initHashSet[GrammarNode]() var curNode = node while true: let father = curNode.father @@ -177,7 +177,7 @@ proc assignId(node: GrammarNode) = proc genEpsilonSet(root: GrammarNode) = var toVisit = @[root] - var allNode = initSet[GrammarNode]() + var allNode = initHashSet[GrammarNode]() # collect direct epsilon set. Reachable by a single epsilon while 0 < toVisit.len: let curNode = toVisit.pop @@ -203,7 +203,7 @@ proc genEpsilonSet(root: GrammarNode) = toVisit.add(child) # collect epsilons of member of epsilon set recursively - var collected = initSet[GrammarNode]() + var collected = initHashSet[GrammarNode]() collected.incl(successGrammarNode) for curNode in allNode: if not collected.contains(curNode): @@ -232,7 +232,7 @@ proc genEpsilonSet(root: GrammarNode) = # exclude 'A' and 'F' in epsilon set if curNode.epsilonSet.len == 0: continue - var toExclude = initSet[GrammarNode]() + var toExclude = initHashSet[GrammarNode]() for child in curNode.epsilonSet: case child.kind of 'A', 'F': @@ -536,7 +536,7 @@ proc lexGrammar = colIdx = startColIdx inc(lineIdx) let grammar = newGrammar(name, grammarString) - grammarSet.add(strTokenMap[name], grammar) + grammarSet[strTokenMap[name]] = grammar proc genFirstSet(grammar: Grammar) = diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 9afae02..c625e2d 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -103,7 +103,7 @@ proc dedentAll*(lexer: Lexer) = # the function can probably be generated by a macro... proc getNextToken( lexer: Lexer, - line: TaintedString, + line: string, idx: var int): TokenNode {. raises: [SyntaxError, InternalError] .} = template raiseSyntaxError(msg: string) = @@ -268,7 +268,7 @@ proc getNextToken( assert result != nil -proc lexOneLine(lexer: Lexer, line: TaintedString) = +proc lexOneLine(lexer: Lexer, line: string) = # process one line at a time assert line.find("\n") == -1 diff --git a/Parser/parser.nim b/Parser/parser.nim index 1ac538a..c70cd1f 100644 --- a/Parser/parser.nim +++ b/Parser/parser.nim @@ -127,7 +127,7 @@ proc applyToken(node: ParseNode, token: TokenNode): ParseStatus = ParseStatus.Normal -proc parseWithState*(input: TaintedString, +proc parseWithState*(input: string, lexer: Lexer, mode=Mode.File, parseNodeArg: ParseNode = nil, diff --git a/Parser/token.nim b/Parser/token.nim index 9d518ff..3ca06bb 100644 --- a/Parser/token.nim +++ b/Parser/token.nim @@ -82,7 +82,7 @@ proc readGrammarToken: seq[string] {.compileTime.} = # everything inside pars proc readReserveName: HashSet[string] {.compileTime.} = let text = slurp(grammarFileName) - result = initSet[string]() + result = initHashSet[string]() var idx = 0 while idx < text.len: case text[idx] diff --git a/Python/cpython.nim b/Python/cpython.nim index 52d5e1a..48215d0 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -82,7 +82,7 @@ proc nPython(args: seq[string]) = if pyConfig.filepath == "": interactiveShell() - if not pyConfig.filepath.existsFile: + if not pyConfig.filepath.fileExists: echo fmt"File does not exist ({pyConfig.filepath})" quit() let input = readFile(pyConfig.filepath) diff --git a/Python/lifecycle.nim b/Python/lifecycle.nim index 13c90b4..2b50292 100644 --- a/Python/lifecycle.nim +++ b/Python/lifecycle.nim @@ -6,10 +6,10 @@ import ../Utils/utils when not defined(js): import os - import ospaths + proc outOfMemHandler = - let e = new OutOfMemError + let e = new OutOfMemDefect raise e system.outOfMemHook = outOfMemHandler diff --git a/Python/neval.nim b/Python/neval.nim index 200005c..cc17cd6 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -570,7 +570,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = else: let msg = fmt"!!! NOT IMPLEMENTED OPCODE {opCode} IN EVAL FRAME !!!" return newNotImplementedError(msg) # no need to handle - except OutOfMemError: + except OutOfMemDefect: excpObj = newMemoryError("Out of Memory") handleException(excpObj) except InterruptError: @@ -615,7 +615,7 @@ else: import os proc pyImport*(name: PyStrObject): PyObject = let filepath = pyConfig.path.joinPath(name.str).addFileExt("py") - if not filepath.existsFile: + if not filepath.fileExists: let msg = fmt"File {filepath} not found" return newImportError(msg) let input = readFile(filepath) @@ -698,7 +698,7 @@ proc runCode*(co: PyCodeObject): PyObject = f.evalFrame -proc runString*(input: TaintedString, fileName: string): PyObject = +proc runString*(input, fileName: string): PyObject = let compileRes = compile(input, fileName) if compileRes.isThrownException: return compileRes diff --git a/Python/symtable.nim b/Python/symtable.nim index 3659000..7091251 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -57,8 +57,8 @@ proc newSymTableEntry(parent: SymTableEntry): SymTableEntry = if not parent.isNil: # not root parent.children.add result result.argVars = initTable[PyStrObject, int]() - result.declaredVars = initSet[PyStrObject]() - result.usedVars = initSet[PyStrObject]() + result.declaredVars = initHashSet[PyStrObject]() + result.usedVars = initHashSet[PyStrObject]() result.scopes = initTable[PyStrObject, Scope]() result.names = initTable[PyStrObject, int]() result.localVars = initTable[PyStrObject, int]() diff --git a/Utils/compat.nim b/Utils/compat.nim index c16204a..ce86cc1 100644 --- a/Utils/compat.nim +++ b/Utils/compat.nim @@ -7,7 +7,7 @@ when defined(js): proc log*(prompt, info: cstring) {. importc .} # how to read from console? - template readLineCompat*(prompt): TaintedString = + template readLineCompat*(prompt): string = "" template echoCompat*(content: string) = @@ -27,7 +27,7 @@ else: import rdstdin import os - template readLineCompat*(prompt): TaintedString = + template readLineCompat*(prompt): string = readLineFromStdin(prompt) template echoCompat*(content) = diff --git a/Utils/utils.nim b/Utils/utils.nim index 0080c43..f118f0e 100644 --- a/Utils/utils.nim +++ b/Utils/utils.nim @@ -1,20 +1,20 @@ type # exceptions used internally - InternalError* = object of Exception + InternalError* = object of CatchableError - SyntaxError* = ref object of Exception + SyntaxError* = ref object of CatchableError fileName*: string lineNo*: int colNo*: int # internal error for wrong type of dict function (`hash` and `eq`) return value - DictError* = object of Exception + DictError* = object of CatchableError # internal error for not implemented bigint lib - IntError* = object of Exception + IntError* = object of CatchableError # internal error for keyboard interruption - InterruptError* = object of Exception + InterruptError* = object of OSError ## Python's is inherited from OSError proc newSyntaxError(msg, fileName: string, lineNo, colNo: int): SyntaxError = new result From 510ca4669706ed72636fde8cf7374b89a3d44cd6 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 16:36:48 +0800 Subject: [PATCH 016/163] fix(warning): [Deprecated] in pkg/regex: re,find shall be suffixed by '2' --- Parser/lexer.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Parser/lexer.nim b/Parser/lexer.nim index c625e2d..365f93b 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -42,8 +42,8 @@ proc getSource*(filePath: string, lineNo: int): string = proc `$`*(lexer: Lexer): string = $lexer.tokenNodes -var regexName = re(r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b") -var regexNumber = re(r"\b\d*\.?\d+([eE][-+]?\d+)?\b") +var regexName = re2(r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b") +var regexNumber = re2(r"\b\d*\.?\d+([eE][-+]?\d+)?\b") # used in parser.nim to construct non-terminators @@ -111,7 +111,7 @@ proc getNextToken( raiseSyntaxError(msg, "", lexer.lineNo, idx) template addRegexToken(tokenName:untyped, msg:string) = - var m: RegexMatch + var m: RegexMatch2 if not line.find(`regex tokenName`, m, start=idx): raiseSyntaxError(msg) let first = m.boundaries.a From 0d6c7482fbbd3c0e38e63f7cd28c114564fb309a Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 16:42:47 +0800 Subject: [PATCH 017/163] fix(warning): [UnusedImport] of stdlib; and a dup import --- Objects/bundle.nim | 4 ++-- Objects/frameobject.nim | 2 +- Objects/pyobject.nim | 2 +- Objects/typeobject.nim | 2 +- Parser/parser.nim | 2 +- Python/asdl.nim | 2 +- Python/ast.nim | 2 +- Python/compile.nim | 6 +++--- Python/coreconfig.nim | 4 ++-- Python/neval.nim | 2 +- Python/symtable.nim | 2 +- Utils/compat.nim | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Objects/bundle.nim b/Objects/bundle.nim index 5b99d83..63ba857 100644 --- a/Objects/bundle.nim +++ b/Objects/bundle.nim @@ -2,9 +2,9 @@ import baseBundle import codeobject, dictobject, frameobject, boolobjectImpl, listobject, moduleobject, methodobject, funcobject, pyobject, stringobjectImpl, rangeobject, exceptionsImpl, - sliceobject, tupleobject, cellobject, methodobject + sliceobject, tupleobject, cellobject export baseBundle export codeobject, dictobject, frameobject, boolobjectImpl, listobject, moduleobject, methodobject, funcobject, pyobject, stringobjectImpl, rangeobject, exceptionsImpl, - sliceobject, tupleobject, cellobject, methodobject + sliceobject, tupleobject, cellobject diff --git a/Objects/frameobject.nim b/Objects/frameobject.nim index 0517574..0e4958a 100644 --- a/Objects/frameobject.nim +++ b/Objects/frameobject.nim @@ -1,4 +1,4 @@ -import strutils + import pyobject import baseBundle diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 491138f..8cd7481 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -3,7 +3,7 @@ # definition import macros except name import sets -import sequtils + import strformat import strutils import hashes diff --git a/Objects/typeobject.nim b/Objects/typeobject.nim index e9953ae..12b0a78 100644 --- a/Objects/typeobject.nim +++ b/Objects/typeobject.nim @@ -1,6 +1,6 @@ import typetraits import strformat -import strutils + import tables import pyobject diff --git a/Parser/parser.nim b/Parser/parser.nim index c70cd1f..bf3ef72 100644 --- a/Parser/parser.nim +++ b/Parser/parser.nim @@ -1,5 +1,5 @@ import strformat -import deques + import strutils import sequtils import sets diff --git a/Python/asdl.nim b/Python/asdl.nim index f21810f..c9a1a27 100644 --- a/Python/asdl.nim +++ b/Python/asdl.nim @@ -2,7 +2,7 @@ import macros import hashes import sequtils import strutils -import strformat + import ../Objects/[pyobject, stringobject] diff --git a/Python/ast.nim b/Python/ast.nim index a0a1bc8..e82ee25 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -1,6 +1,6 @@ import macros import tables -import sequtils + import strutils import typetraits import strformat diff --git a/Python/compile.nim b/Python/compile.nim index b569842..f8049df 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -1,8 +1,8 @@ import algorithm -import sequtils -import strutils + + import macros -import strformat + import tables import ast diff --git a/Python/coreconfig.nim b/Python/coreconfig.nim index 9ddea00..73ec2a3 100644 --- a/Python/coreconfig.nim +++ b/Python/coreconfig.nim @@ -1,5 +1,5 @@ -import tables -import strutils + + type PyConfig = object diff --git a/Python/neval.nim b/Python/neval.nim index cc17cd6..d20d9cf 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -1,4 +1,4 @@ -import algorithm + import strformat import tables diff --git a/Python/symtable.nim b/Python/symtable.nim index 7091251..e137696 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -1,5 +1,5 @@ import tables -import sequtils + import sets import macros diff --git a/Utils/compat.nim b/Utils/compat.nim index ce86cc1..e1b3f6e 100644 --- a/Utils/compat.nim +++ b/Utils/compat.nim @@ -25,7 +25,7 @@ when defined(js): else: import rdstdin - import os + template readLineCompat*(prompt): string = readLineFromStdin(prompt) From 26b39acac0649ef69c3c0a62ec856311efe697b3 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 17:16:42 +0800 Subject: [PATCH 018/163] fix(warning): [UnusedImport] of inner libs --- Objects/descrobject.nim | 2 +- Objects/dictproxyobject.nim | 2 +- Objects/exceptions.nim | 2 +- Objects/frameobject.nim | 4 ++-- Objects/methodobject.nim | 4 ++-- Parser/grammar.nim | 2 +- Python/bltinmodule.nim | 2 +- Python/call.nim | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Objects/descrobject.nim b/Objects/descrobject.nim index 03e42ea..11bf9ef 100644 --- a/Objects/descrobject.nim +++ b/Objects/descrobject.nim @@ -5,7 +5,7 @@ import noneobject import exceptions import stringobject import methodobject -import funcobject + import ../Python/call import ../Utils/utils diff --git a/Objects/dictproxyobject.nim b/Objects/dictproxyobject.nim index 857fae4..c634055 100644 --- a/Objects/dictproxyobject.nim +++ b/Objects/dictproxyobject.nim @@ -1,6 +1,6 @@ import pyobject import baseBundle -import dictobject + # read only dict used for `__dict__` of types declarePyType DictProxy(): diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 2ed6ba1..95b96a6 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -9,7 +9,7 @@ import strformat import pyobject -import noneobject + type ExceptionToken* {. pure .} = enum Base, diff --git a/Objects/frameobject.nim b/Objects/frameobject.nim index 0e4958a..864b617 100644 --- a/Objects/frameobject.nim +++ b/Objects/frameobject.nim @@ -1,11 +1,11 @@ import pyobject -import baseBundle + import codeobject import dictobject import cellobject -import ../Python/opcode + declarePyType Frame(): # currently not used? diff --git a/Objects/methodobject.nim b/Objects/methodobject.nim index bec19ec..14fcd63 100644 --- a/Objects/methodobject.nim +++ b/Objects/methodobject.nim @@ -2,8 +2,8 @@ import strformat import pyobject import exceptions -import funcobject -import frameobject + + import stringobject type diff --git a/Parser/grammar.nim b/Parser/grammar.nim index 665d8d1..5148da3 100644 --- a/Parser/grammar.nim +++ b/Parser/grammar.nim @@ -9,7 +9,7 @@ import tables import deques import token -import ../Utils/[utils, compat] +import ../Utils/[compat] type diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index f99c572..8c337f1 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -1,5 +1,5 @@ import strformat - +{.used.} # this module contains toplevel code, so never `importButNotUsed` import neval import builtindict import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject] diff --git a/Python/call.nim b/Python/call.nim index 9280b94..5626180 100644 --- a/Python/call.nim +++ b/Python/call.nim @@ -1,4 +1,4 @@ -import neval + import ../Objects/[pyobject, baseBundle, methodobject, funcobjectImpl] From 7c382d70e2d6086574dc124ec28cf909d19e8df1 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 17:55:51 +0800 Subject: [PATCH 019/163] fix(nimc/js): no system.outOfMemHook on JS --- Python/lifecycle.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/lifecycle.nim b/Python/lifecycle.nim index 2b50292..cb7b24d 100644 --- a/Python/lifecycle.nim +++ b/Python/lifecycle.nim @@ -12,7 +12,9 @@ proc outOfMemHandler = let e = new OutOfMemDefect raise e -system.outOfMemHook = outOfMemHandler +when declared(system.outOfMemHook): + # if not JS + system.outOfMemHook = outOfMemHandler when not defined(js): proc controlCHandler {. noconv .} = From aa489e39f07d20fa16eabd1f9546bbca0c0a102d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 18:02:49 +0800 Subject: [PATCH 020/163] chore(nimble): task test: support passing arg for subTest --- npython.nimble | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/npython.nimble b/npython.nimble index 86c1338..06456bb 100644 --- a/npython.nimble +++ b/npython.nimble @@ -8,3 +8,40 @@ binDir = "bin" requires "cligen", "regex" requires "nim >= 1.6.14" # 2.* (at least till 2.3.1) is okey, too. + +# copied from nimpylib.nimble +# at 43378424222610f8ce4a10593bd719691fbb634b +func getArgs(taskName: string): seq[string] = + ## cmdargs: 1 2 3 4 5 -> 1 4 3 2 5 + var rargs: seq[string] + let argn = paramCount() + for i in countdown(argn, 0): + let arg = paramStr i + if arg == taskName: + break + rargs.add arg + if rargs.len > 1: + swap rargs[^1], rargs[0] # the file must be the last, others' order don't matter + return rargs + +template mytask(name: untyped, taskDesc: string, body){.dirty.} = + task name, taskDesc: + let taskName = astToStr(name) + body + +template taskWithArgs(name, taskDesc, body){.dirty.} = + mytask name, taskDesc: + var args = getArgs taskName + body + +let binPathWithoutExt = binDir & '/' & bin[0] +taskWithArgs test, "test all, assuming after build": + let subTest = + if args.len == 0: "asserts" + else: args[0] + let pyExe = binPathWithoutExt.toExe + if not fileExists pyExe: + raise newException(OSError, "please firstly run `nimble build`") + for i in listFiles "tests/" & subTest: + echo "testing " & i + exec pyExe & ' ' & i From d632b8f1cc34f7f8ef1cfe4926b12069b5d43878 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 18:05:35 +0800 Subject: [PATCH 021/163] bump-version: 0.1.1 --- npython.nimble | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/npython.nimble b/npython.nimble index 06456bb..6a9fa60 100644 --- a/npython.nimble +++ b/npython.nimble @@ -1,5 +1,5 @@ -version = "0.1.0" -author = "Weitang Li" +version = "0.1.1" +author = "Weitang Li, litlighilit" description = "(Subset of) Python programming language implemented in Nim" license = "CPython license" srcDir = "Python" From 4c7318ed65fc3f566e064ef53bb75acfa3b55372 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 18:37:25 +0800 Subject: [PATCH 022/163] fixup: repl now prints updated version; dep(!cligen) --- Python/cpython.nim | 48 ++++++++++++++++++++++++++++++++++++++---- Python/getversion.nim | 7 ++++++ Python/versionInfo.nim | 12 +++++++++++ npython.nimble | 8 +++++-- 4 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 Python/getversion.nim create mode 100644 Python/versionInfo.nim diff --git a/Python/cpython.nim b/Python/cpython.nim index 48215d0..0850bc7 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -2,28 +2,35 @@ when defined(js): {.error: "python.nim is for c target. Compile jspython.nim as js target" .} import strformat -import strutils + import os # file existence -import cligen # parse opt + import neval import compile import coreconfig import traceback import lifecycle +import ./getversion import ../Parser/[lexer, parser] import ../Objects/bundle import ../Utils/[utils, compat] +proc echoVersion(verbose=false) = + echoCompat "NPython " & ( + if verbose: Py_GetVersion() + else: Version + ) + proc interactiveShell = var finished = true # the root of the concrete syntax tree. Keep this when user input multiple lines var rootCst: ParseNode let lexer = newLexer("") var prevF: PyFrameObject - echoCompat "NPython 0.1.0" + echoVersion() while true: var input: string var prompt: string @@ -90,5 +97,38 @@ proc nPython(args: seq[string]) = if retObj.isThrownException: PyExceptionObject(retObj).printTb +proc echoUsage() = + echoCompat "usage: python [option] [file]" + +proc echoHelp() = + echoUsage() + echoCompat "Options:" + echoCompat "-V : print the Python version number and exit (also --version)" + echoCompat "Arguments:" + echoCompat "file : program read from script file" + + when isMainModule: - dispatch(nPython) + import std/parseopt + + var args: seq[string] + for kind, key, val in getopt( + shortNoVal={'h', 'V'}, + longNoVal = @["help", "version"], + ): + case kind + of cmdArgument: args.add key + of cmdLongOption, cmdShortOption: + case key: + of "help", "h": + echoHelp() + quit() + of "version", "V": + echoVersion() + quit() + else: + echoCompat "Unknown option: " & key + echoUsage() + quit 2 + of cmdEnd: assert(false) # cannot happen + nPython args diff --git a/Python/getversion.nim b/Python/getversion.nim new file mode 100644 index 0000000..141d670 --- /dev/null +++ b/Python/getversion.nim @@ -0,0 +1,7 @@ + +import ./versionInfo +export versionInfo + +proc Py_GetVersion*: string = + Version # TODO with buildinfo, compilerinfo in form of "%.80s (%.80s) %.80s" + diff --git a/Python/versionInfo.nim b/Python/versionInfo.nim new file mode 100644 index 0000000..06db3e6 --- /dev/null +++ b/Python/versionInfo.nim @@ -0,0 +1,12 @@ + +const + Major* = 0 + Minor* = 1 + Patch* = 1 + +const sep = '.' +template asVersion(major, minor, patch: int): string = + $major & sep & $minor & sep & $patch + +const + Version* = asVersion(Major, Minor, Patch) diff --git a/npython.nimble b/npython.nimble index 6a9fa60..caa541f 100644 --- a/npython.nimble +++ b/npython.nimble @@ -1,4 +1,5 @@ -version = "0.1.1" +import "Python/versionInfo" as libver +version = libver.Version author = "Weitang Li, litlighilit" description = "(Subset of) Python programming language implemented in Nim" license = "CPython license" @@ -6,7 +7,7 @@ srcDir = "Python" bin = @["python"] binDir = "bin" -requires "cligen", "regex" +requires "regex" requires "nim >= 1.6.14" # 2.* (at least till 2.3.1) is okey, too. # copied from nimpylib.nimble @@ -45,3 +46,6 @@ taskWithArgs test, "test all, assuming after build": for i in listFiles "tests/" & subTest: echo "testing " & i exec pyExe & ' ' & i + +task buildJs, "build JS": + selfExec "js -o:" & binPathWithoutExt & ".js " & srcDir & '/' & bin[0] From 344134f68096828c3158f192ebe02c5951f243c0 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 19:54:51 +0800 Subject: [PATCH 023/163] feat(cli): -V,--version --- Modules/getbuildinfo.nim | 52 ++++++++++++++++++++++++++++++++++++ Modules/os_findExe_patch.nim | 44 ++++++++++++++++++++++++++++++ Python/cpython.nim | 18 +++++++++---- Python/getversion.nim | 8 +++++- 4 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 Modules/getbuildinfo.nim create mode 100644 Modules/os_findExe_patch.nim diff --git a/Modules/getbuildinfo.nim b/Modules/getbuildinfo.nim new file mode 100644 index 0000000..41588a0 --- /dev/null +++ b/Modules/getbuildinfo.nim @@ -0,0 +1,52 @@ + + +when NimMajor > 1: + from std/paths import `/../`, Path, parentDir + template `/../`(a, b: string): untyped = string(Path(a) /../ Path b) + template parentDir(a: string): untyped = string(Path(a).parentDir) +else: + from std/os import `/../`, parentDir +import ./os_findExe_patch + +## see CPython/configure.ac + +const gitExe{.strdefine: "git".} = findExe("git") +const git = (exe: gitExe) +when git.exe == "": + template exec(git; sub: string): string = "" +else: + const srcdir_git = currentSourcePath().parentDir /../ ".git" + template exec(git: typeof(git); sub: string): string = + bind srcdir_git + let res = gorgeEx(git.exe & " --git-dir " & srcdir_git & " " & sub) + assert res.exitCode == 0, res.output + res.output + +const + version = git.exec"rev-parse --short HEAD" + tag = git.exec"describe --all --always --dirty" + branch = git.exec"name-rev --name-only HEAD" + +proc gitidentifier*: string = + result = tag + if result != "" and result != "undefined": + return + result = branch + + +proc getBuildInfo: string{.compileTime.} = + let revision = version + result = gitidentifier() + if revision != "": + result.add ':' + result.add revision + + result.add ", " + + #result.add &"{CompileDate:.20s}, {CompileTime:.9s}" + result.add CompileDate.substr(0, 19) + result.add ", " + result.add CompileTime.substr(0, 8) + +const buildinfo = getBuildInfo() +proc Py_GetBuildInfo*: string = buildinfo diff --git a/Modules/os_findExe_patch.nim b/Modules/os_findExe_patch.nim new file mode 100644 index 0000000..6e2c21d --- /dev/null +++ b/Modules/os_findExe_patch.nim @@ -0,0 +1,44 @@ + +import std/os +import std/strutils + +when true: + # copied from std/os, removed `readlink` part, see `XXX` below + proc findExe*(exe: string, followSymlinks: bool = true; + extensions: openArray[string]=ExeExts): string {. + tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} = + ## Searches for `exe` in the current working directory and then + ## in directories listed in the ``PATH`` environment variable. + ## + ## Returns `""` if the `exe` cannot be found. `exe` + ## is added the `ExeExts`_ file extensions if it has none. + ## + ## If the system supports symlinks it also resolves them until it + ## meets the actual file. This behavior can be disabled if desired + ## by setting `followSymlinks = false`. + + if exe.len == 0: return + template checkCurrentDir() = + for ext in extensions: + result = addFileExt(exe, ext) + if fileExists(result): return + when defined(posix): + if '/' in exe: checkCurrentDir() + else: + checkCurrentDir() + let path = getEnv("PATH") + for candidate in split(path, PathSep): + if candidate.len == 0: continue + when defined(windows): + var x = (if candidate[0] == '"' and candidate[^1] == '"': + substr(candidate, 1, candidate.len-2) else: candidate) / + exe + else: + var x = expandTilde(candidate) / exe + for ext in extensions: + var x = addFileExt(x, ext) + if fileExists(x): + # XXX: there was a branch of `when ...`, which doesn't work on nimvm + # due to `readlink`, so removed + return x + result = "" diff --git a/Python/cpython.nim b/Python/cpython.nim index 0850bc7..e49a232 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -104,6 +104,7 @@ proc echoHelp() = echoUsage() echoCompat "Options:" echoCompat "-V : print the Python version number and exit (also --version)" + echoCompat " when given twice, print more information about the build" echoCompat "Arguments:" echoCompat "file : program read from script file" @@ -111,7 +112,9 @@ proc echoHelp() = when isMainModule: import std/parseopt - var args: seq[string] + var + args: seq[string] + versionVerbosity = 0 for kind, key, val in getopt( shortNoVal={'h', 'V'}, longNoVal = @["help", "version"], @@ -124,11 +127,16 @@ when isMainModule: echoHelp() quit() of "version", "V": - echoVersion() - quit() + versionVerbosity.inc else: - echoCompat "Unknown option: " & key + var origKey = "-" + if kind == cmdLongOption: origKey.add '-' + origKey.add key + echoCompat "Unknown option: " & origKey echoUsage() quit 2 of cmdEnd: assert(false) # cannot happen - nPython args + case versionVerbosity + of 1: echoVersion() + of 2: echoVersion(verbose=true) + else: nPython args diff --git a/Python/getversion.nim b/Python/getversion.nim index 141d670..60f9b8a 100644 --- a/Python/getversion.nim +++ b/Python/getversion.nim @@ -1,7 +1,13 @@ +import std/strformat import ./versionInfo export versionInfo +import ../Modules/getbuildinfo + +proc Py_GetCompiler*: string = + "[Nim " & NimVersion & ']' + proc Py_GetVersion*: string = - Version # TODO with buildinfo, compilerinfo in form of "%.80s (%.80s) %.80s" + &"{Version:.80} ({Py_GetBuildInfo():.80}) {Py_GetCompiler():.80}" # TODO with buildinfo, compilerinfo in form of "%.80s (%.80s) %.80s" From d8c06d5fad6587662563a48241c5260dd157d1c4 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 20:04:03 +0800 Subject: [PATCH 024/163] chore(nimble): compile target now named npython over python --- npython.nimble | 8 +++++--- tests/run.sh | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/npython.nimble b/npython.nimble index caa541f..8bf74fa 100644 --- a/npython.nimble +++ b/npython.nimble @@ -4,9 +4,11 @@ author = "Weitang Li, litlighilit" description = "(Subset of) Python programming language implemented in Nim" license = "CPython license" srcDir = "Python" -bin = @["python"] binDir = "bin" +let srcName = "python" +namedBin[srcName] = "npython" + requires "regex" requires "nim >= 1.6.14" # 2.* (at least till 2.3.1) is okey, too. @@ -35,7 +37,7 @@ template taskWithArgs(name, taskDesc, body){.dirty.} = var args = getArgs taskName body -let binPathWithoutExt = binDir & '/' & bin[0] +let binPathWithoutExt = binDir & '/' & namedBin[srcName] taskWithArgs test, "test all, assuming after build": let subTest = if args.len == 0: "asserts" @@ -48,4 +50,4 @@ taskWithArgs test, "test all, assuming after build": exec pyExe & ' ' & i task buildJs, "build JS": - selfExec "js -o:" & binPathWithoutExt & ".js " & srcDir & '/' & bin[0] + selfExec "js -o:" & binPathWithoutExt & ".js " & srcDir & '/' & srcName diff --git a/tests/run.sh b/tests/run.sh index 6654283..cbb5c08 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,6 +1,6 @@ echo "tryexcept.py is expected to fail" echo "fib.py and import.py (imports fib.py) are expected to be slow" -PYTHON=../Python/python +PYTHON=../bin/npython for fname in ./asserts/*.py; do echo $fname $PYTHON $fname From 1984a25aa652614f2b027f1df9e6e89a060d43fc Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 20:08:41 +0800 Subject: [PATCH 025/163] fixup: HEAD~2: -VVV ('-V'>2) now means -VV than error --- Python/cpython.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/cpython.nim b/Python/cpython.nim index e49a232..8de9e31 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -137,6 +137,6 @@ when isMainModule: quit 2 of cmdEnd: assert(false) # cannot happen case versionVerbosity + of 0: nPython args of 1: echoVersion() - of 2: echoVersion(verbose=true) - else: nPython args + else: echoVersion(verbose=true) From 6f8e0566fe95949a9df06322e70f372e5bf7e412 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 21:25:52 +0800 Subject: [PATCH 026/163] feat(cli): -c --- Python/cpython.nim | 54 ++++++++++++++++++++++++++++++---------------- Python/neval.nim | 20 +++++++++++++++++ Utils/compat.nim | 5 +++++ 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/Python/cpython.nim b/Python/cpython.nim index 8de9e31..559a7d7 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -84,6 +84,8 @@ proc interactiveShell = else: prevF = f +template exit0or1(suc) = quit(if suc: 0 else: 1) + proc nPython(args: seq[string]) = pyInit(args) if pyConfig.filepath == "": @@ -93,18 +95,17 @@ proc nPython(args: seq[string]) = echo fmt"File does not exist ({pyConfig.filepath})" quit() let input = readFile(pyConfig.filepath) - let retObj = runString(input, pyConfig.filepath) - if retObj.isThrownException: - PyExceptionObject(retObj).printTb + runSimpleString(input, pyConfig.filepath).exit0or1 proc echoUsage() = - echoCompat "usage: python [option] [file]" + echoCompat "usage: python [option] [-c cmd | file]" proc echoHelp() = echoUsage() echoCompat "Options:" - echoCompat "-V : print the Python version number and exit (also --version)" - echoCompat " when given twice, print more information about the build" + echoCompat "-c cmd : program passed in as string (terminates option list)" + echoCompat "-V : print the Python version number and exit (also --version)" + echoCompat " when given twice, print more information about the build" echoCompat "Arguments:" echoCompat "file : program read from script file" @@ -112,30 +113,47 @@ proc echoHelp() = when isMainModule: import std/parseopt + proc unknownOption(p: OptParser){.noReturn.} = + var origKey = "-" + if p.kind == cmdLongOption: origKey.add '-' + origKey.add p.key + errEchoCompat "Unknown option: " & origKey + echoUsage() + quit 2 + template noLongOption(p: OptParser) = + if p.kind == cmdLongOption: + p.unknownOption() + var args: seq[string] versionVerbosity = 0 - for kind, key, val in getopt( + var p = initOptParser( shortNoVal={'h', 'V'}, + # Python can be considered not to allow: -c:CODE -c=code longNoVal = @["help", "version"], - ): - case kind - of cmdArgument: args.add key + ) + while true: + p.next() + case p.kind + of cmdArgument: + args.add p.key of cmdLongOption, cmdShortOption: - case key: + case p.key: of "help", "h": echoHelp() quit() of "version", "V": versionVerbosity.inc + of "c": + p.noLongOption() + #let argv = @["-c"] & p.remainingArgs() + let code = + if p.val != "": p.val + else: p.remainingArgs()[0] + runSimpleString(code, "").exit0or1 else: - var origKey = "-" - if kind == cmdLongOption: origKey.add '-' - origKey.add key - echoCompat "Unknown option: " & origKey - echoUsage() - quit 2 - of cmdEnd: assert(false) # cannot happen + p.unknownOption() + of cmdEnd: break case versionVerbosity of 0: nPython args of 1: echoVersion() diff --git a/Python/neval.nim b/Python/neval.nim index d20d9cf..382f1a7 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -7,6 +7,7 @@ import symtable import opcode import coreconfig import builtindict +import traceback import ../Objects/[pyobject, baseBundle, tupleobject, listobject, dictobject, sliceobject, codeobject, frameobject, funcobject, cellobject, exceptionsImpl, moduleobject, methodobject] @@ -703,3 +704,22 @@ proc runString*(input, fileName: string): PyObject = if compileRes.isThrownException: return compileRes runCode(PyCodeObject(compileRes)) + +template orPrintTb(retRes): bool{.dirty.} = + if retRes.isThrownException: + PyExceptionObject(retRes).printTb + false + else: + true + +proc runSimpleString*(input, fileName: string): bool = + ## returns if successful. + ## + ## a little like `_PyRun_SimpleStringFlagsWithName` + ## but as you may know, it only returns -1 for failure and + ## 0 for success, so returing a bool is better + let compileRes = compile(input, fileName) + result = compileRes.orPrintTb + if not result: return + let runRes = runCode(PyCodeObject(compileRes)) + result = runRes.orPrintTb diff --git a/Utils/compat.nim b/Utils/compat.nim index e1b3f6e..75c0a28 100644 --- a/Utils/compat.nim +++ b/Utils/compat.nim @@ -16,6 +16,8 @@ when defined(js): log(cstring" ", line) #stream.add((kstring"", kstring(content))) + template errEchoCompat*(content) = + echoCompat content # combining two seq directly leads to a bug in the compiler when compiled to JS # see gh-10651 @@ -33,6 +35,9 @@ else: template echoCompat*(content) = echo content + template errEchoCompat*(content) = + stderr.writeLine content + template addCompat*[T](a, b: seq[T]) = a.add b From 2093ab9b7dc57416852e2d43c54d8752930a75b1 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 22:55:39 +0800 Subject: [PATCH 027/163] feat(cli): -q; fix(py): repl now print platform --- Python/coreconfig.nim | 1 + Python/cpython.nim | 31 +++++++++++++++++++++---------- Utils/getplatform.nim | 7 +++++++ 3 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 Utils/getplatform.nim diff --git a/Python/coreconfig.nim b/Python/coreconfig.nim index 73ec2a3..aafee08 100644 --- a/Python/coreconfig.nim +++ b/Python/coreconfig.nim @@ -5,6 +5,7 @@ type PyConfig = object filepath*: string filename*: string + quiet*, verbose*: bool path*: string # sys.path, only one for now var pyConfig* = PyConfig() diff --git a/Python/cpython.nim b/Python/cpython.nim index 559a7d7..b9a8d73 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -15,14 +15,22 @@ import lifecycle import ./getversion import ../Parser/[lexer, parser] import ../Objects/bundle -import ../Utils/[utils, compat] - - -proc echoVersion(verbose=false) = - echoCompat "NPython " & ( - if verbose: Py_GetVersion() - else: Version - ) +import ../Utils/[utils, compat, getplatform] + +proc getVersionString(verbose=false): string = + result = "NPython " + if not verbose: + result.add Version + return + result.add Py_GetVersion() + result.add " on " + result.add PLATFORM +template echoVersion(verbose=false) = + echoCompat getVersionString(verbose) + +proc pymain_header = + if pyConfig.quiet: return + errEchoCompat getVersionString(verbose=true) proc interactiveShell = var finished = true @@ -30,7 +38,7 @@ proc interactiveShell = var rootCst: ParseNode let lexer = newLexer("") var prevF: PyFrameObject - echoVersion() + pymain_header() while true: var input: string var prompt: string @@ -104,6 +112,7 @@ proc echoHelp() = echoUsage() echoCompat "Options:" echoCompat "-c cmd : program passed in as string (terminates option list)" + echoCompat "-q : don't print version and copyright messages on interactive startup" echoCompat "-V : print the Python version number and exit (also --version)" echoCompat " when given twice, print more information about the build" echoCompat "Arguments:" @@ -128,7 +137,7 @@ when isMainModule: args: seq[string] versionVerbosity = 0 var p = initOptParser( - shortNoVal={'h', 'V'}, + shortNoVal={'h', 'V', 'q', 'v'}, # Python can be considered not to allow: -c:CODE -c=code longNoVal = @["help", "version"], ) @@ -144,6 +153,8 @@ when isMainModule: quit() of "version", "V": versionVerbosity.inc + of "q": pyConfig.quiet = true + of "v": pyConfig.verbose = true of "c": p.noLongOption() #let argv = @["-c"] & p.remainingArgs() diff --git a/Utils/getplatform.nim b/Utils/getplatform.nim new file mode 100644 index 0000000..3a3e3ce --- /dev/null +++ b/Utils/getplatform.nim @@ -0,0 +1,7 @@ + +const PLATFORM* = + when defined(js): "js" + elif defined(windows): "win32" + elif defined(macosx): "darwin" # hostOS is macosx + else: hostOS ## XXX: not right on all platform. PY-DIFF + ## see nimpylib/nimpylib src/pylib/Lib/sys_impl/genplatform.nim From ff7379029a46186e66cc5999b48015d602f99694 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 11 Jul 2025 22:58:14 +0800 Subject: [PATCH 028/163] doc(readme): update "How to use" & "Drawbacks" --- README.md | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b21015a..5298dae 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ Check out `./tests` to see more examples. ``` git clone https://github.com/liwt31/NPython.git cd NPython -nimble c ./Python/python -./Python/python +nimble build +bin/npython ``` ### Todo @@ -44,11 +44,25 @@ Currently, the performance bottlenecks are object allocation, seq accessing (com ### Drawbacks NPython aims for both C and JavaScript targets, so it's hard (if not impossible) to perform low-level address based optimization. -NPython currently relies on Nim GC. Frankly speaking it's not satisfactory. -* The GC uses thread-local heap, makes threading nearly impossible (for Python). -* The GC can hardly be shared between different dynamic libs, which means NPython can not import extensions written in Nim. -If memory is managed manually, hopefully these drawbacks can be overcomed. Of course that's a huge sacrifice. +#### Nim 0.x GC +NPython relies on Nim GC. Frankly speaking, in the past, it was not satisfactory. +* The GC uses thread-local heap, which once made threading once nearly impossible (for Python), though not so for Nimv1 and Nimv2. +* The GC could hardly be shared between different dynamic libs, which meant NPython can not import extensions written in Nim. + +If memory was managed manually, these drawbacks could be overcomed early. + +#### Nim v1 and v2 MM +However, in current years, Nim, specially v2, has improved a lot on GC, +which's now called MM(Memory Management). + +And Nimv2 uses ORC by default, which offers deterministic performance. + +Not only has threading programming been enhanced and become easy to write, +but also `setupForeignThreadGc()` and `tearDownForeignThreadGc()` come out here +for foreignal call to control Nim's MM. + +In short those difficulties that once held us back have disappeared. ### License From 28059a916ae16c9da15489986f43f3a47f2acbf3 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sat, 12 Jul 2025 01:06:14 +0800 Subject: [PATCH 029/163] fixup --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5298dae..1998b02 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ NPython aims for both C and JavaScript targets, so it's hard (if not impossible) #### Nim 0.x GC NPython relies on Nim GC. Frankly speaking, in the past, it was not satisfactory. -* The GC uses thread-local heap, which once made threading once nearly impossible (for Python), though not so for Nimv1 and Nimv2. +* The GC used thread-local heap, which once made threading once nearly impossible (for Python), though not so for Nimv1 and Nimv2. * The GC could hardly be shared between different dynamic libs, which meant NPython can not import extensions written in Nim. If memory was managed manually, these drawbacks could be overcomed early. @@ -56,7 +56,7 @@ If memory was managed manually, these drawbacks could be overcomed early. However, in current years, Nim, specially v2, has improved a lot on GC, which's now called MM(Memory Management). -And Nimv2 uses ORC by default, which offers deterministic performance. +And Nimv2 uses ORC by default, which offers deterministic performance and uses a shared heap. Not only has threading programming been enhanced and become easy to write, but also `setupForeignThreadGc()` and `tearDownForeignThreadGc()` come out here From 54cd9c7c1da6a137b78585efe87364b301aa1025 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sat, 12 Jul 2025 20:14:53 +0800 Subject: [PATCH 030/163] fix(nimc): CC:clang not compile Objects/exceptions.nim:`new*Error` sth like `&&blitTmp_4->Sup->Sup;` c code was generented --- Objects/exceptions.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 95b96a6..5d17415 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -9,6 +9,7 @@ import strformat import pyobject +import stringobject type ExceptionToken* {. pure .} = enum @@ -95,14 +96,14 @@ declareErrors template newProcTmpl(excpName) = # use template for lazy evaluation to use PyString # theses two templates are used internally to generate errors (default thrown) - template `new excpName Error`*: PyBaseErrorObject = + proc `new excpName Error`*: PyBaseErrorObject{.inline.} = let excp = `newPy excpName ErrorSimple`() excp.tk = ExceptionToken.`excpName` excp.thrown = true excp - template `new excpName Error`*(msgStr:string): PyBaseErrorObject = + proc `new excpName Error`*(msgStr:string): PyBaseErrorObject{.inline.} = let excp = `newPy excpName ErrorSimple`() excp.tk = ExceptionToken.`excpName` excp.thrown = true From ba0244afd6076ef2caa2ad803cb98ba1dd36735b Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 03:26:23 +0800 Subject: [PATCH 031/163] fixup: 6f8e0566fe95949: -c not init py --- Python/cpython.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/cpython.nim b/Python/cpython.nim index b9a8d73..69fe238 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -161,6 +161,7 @@ when isMainModule: let code = if p.val != "": p.val else: p.remainingArgs()[0] + pyInit(@[]) runSimpleString(code, "").exit0or1 else: p.unknownOption() From e3bce62d4e75726597e4976a9f1733488970f836 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sun, 13 Jul 2025 20:15:27 +0800 Subject: [PATCH 032/163] refine(dep): use pkg/regex iff -d:npythonUseRegex;std/nre iff -d:npythonUseNre... impr(re): ct compile re if possible --- Parser/lexer.nim | 11 +++++----- Parser/re_utils.nim | 53 +++++++++++++++++++++++++++++++++++++++++++++ npython.nimble | 4 +++- 3 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 Parser/re_utils.nim diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 365f93b..cac5593 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -1,4 +1,4 @@ -import regex +import ./re_utils import deques import sets import strformat @@ -42,8 +42,8 @@ proc getSource*(filePath: string, lineNo: int): string = proc `$`*(lexer: Lexer): string = $lexer.tokenNodes -var regexName = re2(r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b") -var regexNumber = re2(r"\b\d*\.?\d+([eE][-+]?\d+)?\b") +compileLiteralRe regexName, r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b" +compileLiteralRe regexNumber, r"\b\d*\.?\d+([eE][-+]?\d+)?\b" # used in parser.nim to construct non-terminators @@ -111,13 +111,12 @@ proc getNextToken( raiseSyntaxError(msg, "", lexer.lineNo, idx) template addRegexToken(tokenName:untyped, msg:string) = - var m: RegexMatch2 + var m: RegexMatch if not line.find(`regex tokenName`, m, start=idx): raiseSyntaxError(msg) - let first = m.boundaries.a let last = m.boundaries.b idx = last + 1 - result = newTokenNode(Token.tokenName, lexer.lineNo, first, line[first..last]) + result = newTokenNode(Token.tokenName, lexer.lineNo, m.boundaries.a, m.capturedStr line) template addSingleCharToken(tokenName) = result = newTokenNode(Token.tokenName, lexer.lineNo, idx) diff --git a/Parser/re_utils.nim b/Parser/re_utils.nim new file mode 100644 index 0000000..198f7f7 --- /dev/null +++ b/Parser/re_utils.nim @@ -0,0 +1,53 @@ + + +const canCTCompileAndRe2 = defined(npythonUseRegex) + +const npythonUseRe{.booldefine.} = true + +template declCapturedStr(ret){.dirty.} = + template capturedStr*(r: RegexMatch, line: string): string = ret + +when canCTCompileAndRe2: + import regex + type RegexMatch* = regex.RegexMatch2 + export regex.find + declCapturedStr line[r.boundaries] + +elif npythonUseRe: + import std/re + + type RegexMatch* = Slice[int] + template boundaries*(r: RegexMatch): untyped = r + declCapturedStr line[r] + proc find*(buf: string, pattern: Regex, match: var RegexMatch, start=0): bool{.inline.} = + (match.a, match.b) = findBounds(buf, pattern, start)# != (-1,0) + match.a != -1 + +elif defined(npythonUseNre): + import std/nre + import ../Utils/utils + export RegexMatch + template boundaries*(r: RegexMatch): untyped = r.captureBounds[-1] + declCapturedStr r.captures[-1] + proc find*(buf: string, pattern: Regex, match: var RegexMatch, start=0): bool{.inline.} = + let opt = try: + buf.find(pattern, start) + except ValueError: unreachable() + # the `ValueError` is from: find <- matchImpl <- captureCount <- getinfo <- strutils.`%` + # but as the format string is static and you can tell it never ValueError + except InvalidUnicodeError, RegexInternalError: + # on returning false, the callee will raiseSyntaxError + raise newException(InternalError, "std/nre.find: InvalidUnicode or RegexInternal") + result = opt.isSome() + if result: + match = opt.unsafeGet() + +when canCTCompileAndRe2: + template compileLiteralRe*(name; str) = + bind re2 + const name = re2(str) +else: + template compileLiteralRe*(name; str) = + # std/re, std/nre can't compile at compile-time + bind re + let name = re(str) diff --git a/npython.nimble b/npython.nimble index 8bf74fa..094b994 100644 --- a/npython.nimble +++ b/npython.nimble @@ -9,7 +9,9 @@ binDir = "bin" let srcName = "python" namedBin[srcName] = "npython" -requires "regex" +when defined(npythonUseRegex): + requires "regex" + # otherwise uses std/re requires "nim >= 1.6.14" # 2.* (at least till 2.3.1) is okey, too. # copied from nimpylib.nimble From ce97b39ad06b1d84ec77730fa4c6cf7a8b78670f Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sun, 13 Jul 2025 23:56:09 +0800 Subject: [PATCH 033/163] impr(lexer): purge usage of re --- Parser/lexer.nim | 22 +++++++----------- Parser/lexer_utils.nim | 38 ++++++++++++++++++++++++++++++ Parser/re_utils.nim | 53 ------------------------------------------ npython.nimble | 3 --- 4 files changed, 47 insertions(+), 69 deletions(-) create mode 100644 Parser/lexer_utils.nim delete mode 100644 Parser/re_utils.nim diff --git a/Parser/lexer.nim b/Parser/lexer.nim index cac5593..e7173e3 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -1,4 +1,4 @@ -import ./re_utils +import ./lexer_utils import deques import sets import strformat @@ -42,9 +42,6 @@ proc getSource*(filePath: string, lineNo: int): string = proc `$`*(lexer: Lexer): string = $lexer.tokenNodes -compileLiteralRe regexName, r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b" -compileLiteralRe regexNumber, r"\b\d*\.?\d+([eE][-+]?\d+)?\b" - # used in parser.nim to construct non-terminators proc newTokenNode*(token: Token, @@ -99,7 +96,6 @@ proc dedentAll*(lexer: Lexer) = lexer.add(Token.Dedent, lexer.indentLevel * 4) dec lexer.indentLevel - # the function can probably be generated by a macro... proc getNextToken( lexer: Lexer, @@ -110,13 +106,13 @@ proc getNextToken( # fileName set elsewhere raiseSyntaxError(msg, "", lexer.lineNo, idx) - template addRegexToken(tokenName:untyped, msg:string) = - var m: RegexMatch - if not line.find(`regex tokenName`, m, start=idx): + template addToken(tokenName:untyped, msg:string) = + var content: string + let first = idx + if not `parse tokenName`(line, content, start=idx): raiseSyntaxError(msg) - let last = m.boundaries.b - idx = last + 1 - result = newTokenNode(Token.tokenName, lexer.lineNo, m.boundaries.a, m.capturedStr line) + idx.inc content.len + result = newTokenNode(Token.tokenName, lexer.lineNo, first, content) template addSingleCharToken(tokenName) = result = newTokenNode(Token.tokenName, lexer.lineNo, idx) @@ -137,9 +133,9 @@ proc getNextToken( case line[idx] of 'a'..'z', 'A'..'Z', '_': - addRegexToken(Name, "Invalid identifier") + addToken(Name, "Invalid identifier") of '0'..'9': - addRegexToken(Number, "Invalid number") + addToken(Number, "Invalid number") of '"', '\'': let pairingChar = line[idx] diff --git a/Parser/lexer_utils.nim b/Parser/lexer_utils.nim new file mode 100644 index 0000000..d687e99 --- /dev/null +++ b/Parser/lexer_utils.nim @@ -0,0 +1,38 @@ + +from std/strutils import Digits, IdentChars +import std/parseutils + +template parseName*(s: string, res: var string, start=0): bool = + bind parseIdent + parseIdent(s, res, start) != 0 + +proc parseNumber*(s: string, res: var string, start=0): bool = + ## r"\b\d*\.?\d+([eE][-+]?\d+)?\b" + {.push boundChecks: off.} + let hi = s.high + template strp: untyped = s.toOpenArray(idx, hi) + template ret = + idx.dec + res = s[start..idx] + return + template cur: untyped = + if idx <= hi: + s[idx] + else: + ret + result = true + var idx = start + template eatDigits = + idx.inc strp.skipWhile Digits + eatDigits + if cur != '.': ret + idx.inc + eatDigits + if cur not_in {'e', 'E'}: ret + idx.inc + if cur in {'+', '-'}: idx.inc + eatDigits + if idx < hi and s[idx+1] in IdentChars: + result = false + ret + {.pop.} \ No newline at end of file diff --git a/Parser/re_utils.nim b/Parser/re_utils.nim deleted file mode 100644 index 198f7f7..0000000 --- a/Parser/re_utils.nim +++ /dev/null @@ -1,53 +0,0 @@ - - -const canCTCompileAndRe2 = defined(npythonUseRegex) - -const npythonUseRe{.booldefine.} = true - -template declCapturedStr(ret){.dirty.} = - template capturedStr*(r: RegexMatch, line: string): string = ret - -when canCTCompileAndRe2: - import regex - type RegexMatch* = regex.RegexMatch2 - export regex.find - declCapturedStr line[r.boundaries] - -elif npythonUseRe: - import std/re - - type RegexMatch* = Slice[int] - template boundaries*(r: RegexMatch): untyped = r - declCapturedStr line[r] - proc find*(buf: string, pattern: Regex, match: var RegexMatch, start=0): bool{.inline.} = - (match.a, match.b) = findBounds(buf, pattern, start)# != (-1,0) - match.a != -1 - -elif defined(npythonUseNre): - import std/nre - import ../Utils/utils - export RegexMatch - template boundaries*(r: RegexMatch): untyped = r.captureBounds[-1] - declCapturedStr r.captures[-1] - proc find*(buf: string, pattern: Regex, match: var RegexMatch, start=0): bool{.inline.} = - let opt = try: - buf.find(pattern, start) - except ValueError: unreachable() - # the `ValueError` is from: find <- matchImpl <- captureCount <- getinfo <- strutils.`%` - # but as the format string is static and you can tell it never ValueError - except InvalidUnicodeError, RegexInternalError: - # on returning false, the callee will raiseSyntaxError - raise newException(InternalError, "std/nre.find: InvalidUnicode or RegexInternal") - result = opt.isSome() - if result: - match = opt.unsafeGet() - -when canCTCompileAndRe2: - template compileLiteralRe*(name; str) = - bind re2 - const name = re2(str) -else: - template compileLiteralRe*(name; str) = - # std/re, std/nre can't compile at compile-time - bind re - let name = re(str) diff --git a/npython.nimble b/npython.nimble index 094b994..ea36125 100644 --- a/npython.nimble +++ b/npython.nimble @@ -9,9 +9,6 @@ binDir = "bin" let srcName = "python" namedBin[srcName] = "npython" -when defined(npythonUseRegex): - requires "regex" - # otherwise uses std/re requires "nim >= 1.6.14" # 2.* (at least till 2.3.1) is okey, too. # copied from nimpylib.nimble From 660a10dbd3c0caf53846f61d28fd681f9c1457b9 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 00:25:51 +0800 Subject: [PATCH 034/163] fix(list.remove): `LockError: Read failed because object is been written.`; seq.delete usage wrong --- Objects/listobject.nim | 10 ++++++---- Objects/pyobject.nim | 4 ++++ tests/asserts/list.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 123eadf..0f46588 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -1,4 +1,4 @@ -import sequtils + import strformat import strutils @@ -160,13 +160,15 @@ implListMethod pop(), [mutable: write]: self.items.pop implListMethod remove(target: PyObject), [mutable: write]: - let retObj = indexPyListObjectMethod(selfNoCast, @[target]) + var retObj: PyObject + allowSelfReadWhenBeforeRealWrite: + retObj = indexPyListObjectMethod(selfNoCast, @[target]) if retObj.isThrownException: return retObj assert retObj.ofPyIntObject let idx = PyIntObject(retObj).toInt - self.items.delete(idx .. idx+1) - + self.items.delete(idx) + pyNone implListMagic init: if 1 < args.len: diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 8cd7481..7a1fcb7 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -400,6 +400,10 @@ macro reprLock*(code: untyped): untyped = ) code +template allowSelfReadWhenBeforeRealWrite*(body) = + self.writeLock = false + body + self.writeLock = true macro mutable*(kind, code: untyped): untyped = if kind.strVal != "read" and kind.strVal != "write": diff --git a/tests/asserts/list.py b/tests/asserts/list.py index 4c63516..9bac6f7 100644 --- a/tests/asserts/list.py +++ b/tests/asserts/list.py @@ -32,5 +32,8 @@ assert l.pop() == 100 +ls2 = [1, 2, 2] +ls2.remove(2) +assert ls2[-1] == 2 print("ok") From 4694f65e5f33d455def86c0ca272c5a2e89d9f3d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 01:18:51 +0800 Subject: [PATCH 035/163] feat(tuple): `__contains__`,index,count;feat(list): `__eq__` --- Objects/listobject.nim | 90 ++------------------- Objects/tupleobject.nim | 172 +++++++++++++++++++++++++--------------- 2 files changed, 112 insertions(+), 150 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 0f46588..1463943 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -6,6 +6,7 @@ import pyobject import baseBundle import sliceobject import iterobject +import ./tupleobject import ../Utils/[utils, compat] declarePyType List(reprLock, mutable, tpToken): @@ -19,50 +20,11 @@ proc newPyList*(items: seq[PyObject]): PyListObject = result = newPyList() result.items = items - -implListMagic contains, [mutable: read]: - for idx, item in self.items: - let retObj = item.callMagic(eq, other) - if retObj.isThrownException: - return retObj - if retObj == pyTrueObj: - return pyTrueObj - return pyFalseObj - - -implListMagic iter, [mutable: read]: - newPySeqIter(self.items) - - -implListMagic repr, [mutable: read, reprLock]: - var ss: seq[string] - for item in self.items: - var itemRepr: PyStrObject - let retObj = item.callMagic(repr) - errorIfNotString(retObj, "__repr__") - itemRepr = PyStrObject(retObj) - ss.add(itemRepr.str) - return newPyString("[" & ss.join(", ") & "]") - - -implListMagic len, [mutable: read]: - newPyInt(self.items.len) - -implListMagic getitem, [mutable: read]: - if other.ofPyIntObject: - let idx = getIndex(PyIntObject(other), self.items.len) - return self.items[idx] - if other.ofPySliceObject: - let slice = PySliceObject(other) - let newList = newPyList() - let retObj = slice.getSliceItems(self.items.addr, newList.items.addr) - if retObj.isThrownException: - return retObj - else: - return newList - - return newIndexTypeError("list", other) - +genSequenceMagics "list", + implListMagic, implListMethod, + ofPyListObject, PyListObject, + newPyListSimple, [mutable: read], [reprLock, mutable: read], + '[', ']' implListMagic setitem, [mutable: write]: if arg1.ofPyIntObject: @@ -90,16 +52,6 @@ implListMethod copy(), [mutable: read]: newL -implListMethod count(target: PyObject), [mutable: read]: - var count: int - for item in self.items: - let retObj = item.callMagic(eq, target) - if retObj.isThrownException: - return retObj - if retObj == pyTrueObj: - inc count - newPyInt(count) - # some test methods just for debugging when not defined(release): # for lock testing @@ -129,18 +81,6 @@ when not defined(release): # todo # -implListMethod index(target: PyObject), [mutable: read]: - for idx, item in self.items: - let retObj = item.callMagic(eq, target) - if retObj.isThrownException: - return retObj - if retObj == pyTrueObj: - return newPyInt(idx) - let msg = fmt"{target} is not in list" - newValueError(msg) - - - implListMethod insert(idx: PyIntObject, item: PyObject), [mutable: write]: var intIdx: int if idx.negative: @@ -170,21 +110,3 @@ implListMethod remove(target: PyObject), [mutable: write]: self.items.delete(idx) pyNone -implListMagic init: - if 1 < args.len: - let msg = fmt"list expected at most 1 args, got {args.len}" - return newTypeError(msg) - if self.items.len != 0: - self.items.setLen(0) - if args.len == 1: - let (iterable, nextMethod) = getIterableWithCheck(args[0]) - if iterable.isThrownException: - return iterable - while true: - let nextObj = nextMethod(iterable) - if nextObj.isStopIter: - break - if nextObj.isThrownException: - return nextObj - self.items.add nextObj - pyNone diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 10798cf..e4bca92 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -17,41 +17,117 @@ proc newPyTuple*(items: seq[PyObject]): PyTupleObject = result.items = items -implTupleMagic eq: - if not other.ofPyTupleObject: - return pyFalseObj - let tOther = PyTupleObject(other) - if self.items.len != tOther.items.len: - return pyFalseObj - for i in 0.. Date: Mon, 14 Jul 2025 01:41:06 +0800 Subject: [PATCH 036/163] feat(tuple,list): __add__; list.extend --- Objects/listobject.nim | 20 +++++++++++++++++--- Objects/tupleobject.nim | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 1463943..cf2ae64 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -77,9 +77,23 @@ when not defined(release): implListMethod hello(), [hello]: pyNone -# implListMethod extend: -# todo -# + +implListMethod extend(other: PyObject), [mutable: write]: + if other.ofPyListObject: + self.items &= PyListObject(other).items + else: + let (iterable, nextMethod) = getIterableWithCheck(other) + if iterable.isThrownException: + return iterable + while true: + let nextObj = nextMethod(iterable) + if nextObj.isStopIter: + break + if nextObj.isThrownException: + return nextObj + self.items.add nextObj + pyNone + implListMethod insert(idx: PyIntObject, item: PyObject), [mutable: write]: var intIdx: int diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index e4bca92..c745389 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -23,6 +23,24 @@ template genSequenceMagics*(nameStr, newPyNameSimple; mutRead, mutReadRepr; reprs_left, reprs_right: char): untyped{.dirty.} = + implNameMagic add, mutRead: + var res = newPyNameSimple() + if other.ofPyNameObject: + res.items = self.items & PyNameObject(other).items + return res + else: + res.items = self.items + let (iterable, nextMethod) = getIterableWithCheck(other) + if iterable.isThrownException: + return iterable + while true: + let nextObj = nextMethod(iterable) + if nextObj.isStopIter: + break + if nextObj.isThrownException: + return nextObj + res.items.add nextObj + return res implNameMagic eq, mutRead: if not other.ofPyNameObject: From 19c58ddeea8b38739c9c46853bdeb05a76a738a8 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 02:02:22 +0800 Subject: [PATCH 037/163] feat(str): fallback to call `__repr__` if no `__str__` --- Objects/stringobjectImpl.nim | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index fda33e2..b9063be 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -1,5 +1,5 @@ import hashes - +import std/strformat import pyobject import baseBundle import stringobject @@ -39,6 +39,14 @@ implStrMagic repr: implStrMagic hash: newPyInt(self.hash) - +# TODO: encoding, errors params implStrMagic New(tp: PyObject, obj: PyObject): - obj.callMagic(str) + # ref: unicode_new -> unicode_new_impl -> PyObject_Str + let fun = obj.getMagic(str) + if fun.isNil: + return obj.callMagic(repr) + result = fun(obj) + if not result.ofPyStrObject: + return newTypeError( + &"__str__ returned non-string (type {result.pyType.name:.200s})") + From d377f1b5c9a63a1dc0fab1096ac107d119580699 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 03:23:37 +0800 Subject: [PATCH 038/163] fix(ast): (1,) was regarded as 1 over tuple --- Python/ast.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index e82ee25..8978e1a 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -949,8 +949,11 @@ ast atom, [AsdlExpr]: raiseSyntaxError("Yield expression not implemented", child) of Token.testlist_comp: let testListComp = astTestlistComp(child) - # no tuple, just things like (1 + 2) * 3 - if testListComp.len == 1: + # 1-element tuple or things like (1 + 2) * 3 + if testListComp.len == 1 and not ( + child.children.len == 2 and # 1-element tuple. e.g. (1,) + child.children[1].tokenNode.token == Token.Comma + ): if testListComp[0].kind == AsdlExprTk.ListComp: raiseSyntaxError("generator expression not implemented", child) result = testListComp[0] From 9125f01e3203324e56cb10df0a4279fd6d59a576 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 03:36:45 +0800 Subject: [PATCH 039/163] fix(tuple): repr((1,)) was "(1)" --- Objects/listobject.nim | 4 +++- Objects/tupleobject.nim | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index cf2ae64..a846688 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -20,11 +20,13 @@ proc newPyList*(items: seq[PyObject]): PyListObject = result = newPyList() result.items = items +template lsSeqToStr(ss): string = '[' & ss.join", " & ']' + genSequenceMagics "list", implListMagic, implListMethod, ofPyListObject, PyListObject, newPyListSimple, [mutable: read], [reprLock, mutable: read], - '[', ']' + lsSeqToStr implListMagic setitem, [mutable: write]: if arg1.ofPyIntObject: diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index c745389..171a2cc 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -21,7 +21,7 @@ template genSequenceMagics*(nameStr, implNameMagic, implNameMethod; ofPyNameObject, PyNameObject, newPyNameSimple; mutRead, mutReadRepr; - reprs_left, reprs_right: char): untyped{.dirty.} = + seqToStr): untyped{.dirty.} = implNameMagic add, mutRead: var res = newPyNameSimple() @@ -81,7 +81,7 @@ template genSequenceMagics*(nameStr, errorIfNotString(retObj, "__repr__") itemRepr = PyStrObject(retObj) ss.add(itemRepr.str) - return newPyString(reprs_left & ss.join(", ") & reprs_right) + return newPyString(seqToStr(ss)) implNameMagic len, mutRead: newPyInt(self.items.len) @@ -140,11 +140,23 @@ template genSequenceMagics*(nameStr, inc count newPyInt(count) +proc tupleSeqToString(ss: openArray[string]): string = + ## one-element tuple must be out as "(1,)" + result = "(" + case ss.len + of 0: discard + of 1: + result.add ss[0] + result.add ',' + else: + result.add ss.join", " + result.add ')' + genSequenceMagics "tuple", implTupleMagic, implTupleMethod, ofPyTupleObject, PyTupleObject, newPyTupleSimple, [], [reprLock], - '(', ')' + tupleSeqToString implTupleMagic hash: From 6a470f65d2d0892e4393dbb1df2346fc0cfb6df9 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 04:10:12 +0800 Subject: [PATCH 040/163] fixup! feat(tuple,list): __add__; list.extend --- Objects/listobject.nim | 2 +- Objects/tupleobject.nim | 77 +++++++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index a846688..62161b3 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -26,7 +26,7 @@ genSequenceMagics "list", implListMagic, implListMethod, ofPyListObject, PyListObject, newPyListSimple, [mutable: read], [reprLock, mutable: read], - lsSeqToStr + lsSeqToStr implListMagic setitem, [mutable: write]: if arg1.ofPyIntObject: diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 171a2cc..746224d 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -17,11 +17,10 @@ proc newPyTuple*(items: seq[PyObject]): PyTupleObject = result.items = items -template genSequenceMagics*(nameStr, - implNameMagic, implNameMethod; - ofPyNameObject, PyNameObject, - newPyNameSimple; mutRead, mutReadRepr; - seqToStr): untyped{.dirty.} = +template genCollectMagics*(items, `&`, `&=`, + implNameMagic, newPyNameSimple, + ofPyNameObject, PyNameObject, + mutRead, mutReadRepr, seqToStr){.dirty.} = implNameMagic add, mutRead: var res = newPyNameSimple() @@ -39,28 +38,12 @@ template genSequenceMagics*(nameStr, break if nextObj.isThrownException: return nextObj - res.items.add nextObj + `&=` res.items, nextObj return res - implNameMagic eq, mutRead: - if not other.ofPyNameObject: - return pyFalseObj - let tOther = PyNameObject(other) - if self.items.len != tOther.items.len: - return pyFalseObj - for i in 0.. Date: Mon, 14 Jul 2025 05:49:19 +0800 Subject: [PATCH 041/163] feat(set): new (some methods not impl yet); ... builtins.hash;type.__hash__,__eq__ --- Objects/bundle.nim | 4 +- Objects/dictobject.nim | 29 +-------- Objects/hash.nim | 36 +++++++++++ Objects/pyobject.nim | 6 ++ Objects/pyobjectBase.nim | 2 + Objects/setobject.nim | 125 +++++++++++++++++++++++++++++++++++++++ Objects/tupleobject.nim | 41 +++++++------ Objects/typeobject.nim | 12 +++- Python/ast.nim | 50 +++++++++++----- Python/bltinmodule.nim | 4 ++ Python/compile.nim | 11 ++++ Python/neval.nim | 11 +++- 12 files changed, 264 insertions(+), 67 deletions(-) create mode 100644 Objects/hash.nim create mode 100644 Objects/setobject.nim diff --git a/Objects/bundle.nim b/Objects/bundle.nim index 63ba857..275cda7 100644 --- a/Objects/bundle.nim +++ b/Objects/bundle.nim @@ -2,9 +2,9 @@ import baseBundle import codeobject, dictobject, frameobject, boolobjectImpl, listobject, moduleobject, methodobject, funcobject, pyobject, stringobjectImpl, rangeobject, exceptionsImpl, - sliceobject, tupleobject, cellobject + sliceobject, tupleobject, cellobject, setobject export baseBundle export codeobject, dictobject, frameobject, boolobjectImpl, listobject, moduleobject, methodobject, funcobject, pyobject, stringobjectImpl, rangeobject, exceptionsImpl, - sliceobject, tupleobject, cellobject + sliceobject, tupleobject, cellobject, setobject diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index 9dce188..b611469 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -1,5 +1,5 @@ import strformat -import hashes + import strutils import tables import macros @@ -9,31 +9,8 @@ import listobject import baseBundle import ../Utils/utils - -# hash functions for py objects -# raises an exception to indicate type error. Should fix this -# when implementing custom dict -proc hash*(obj: PyObject): Hash = - let fun = obj.pyType.magicMethods.hash - if fun.isNil: - return hash(addr(obj[])) - else: - let retObj = fun(obj) - if not retObj.ofPyIntObject: - raise newException(DictError, retObj.pyType.name) - return hash(PyIntObject(retObj)) - - -proc `==`*(obj1, obj2: PyObject): bool {. inline, cdecl .} = - let fun = obj1.pyType.magicMethods.eq - if fun.isNil: - return obj1.id == obj2.id - else: - let retObj = fun(obj1, obj2) - if not retObj.ofPyBoolObject: - raise newException(DictError, retObj.pyType.name) - return PyBoolObject(retObj).b - +import ./hash +export hash # currently not ordered # nim ordered table has O(n) delete time diff --git a/Objects/hash.nim b/Objects/hash.nim new file mode 100644 index 0000000..1243261 --- /dev/null +++ b/Objects/hash.nim @@ -0,0 +1,36 @@ +import std/hashes +import ./pyobject +import ./baseBundle +import ../Utils/utils + +proc rawHash*(obj: PyObject): Hash = + ## for type.__hash__ + hash(obj.id) + +# hash functions for py objects +# raises an exception to indicate type error. Should fix this +# when implementing custom dict +proc hash*(obj: PyObject): Hash = + ## for builtins.hash + let fun = obj.pyType.magicMethods.hash + if fun.isNil: + return rawHash(obj) + else: + let retObj = fun(obj) + if not retObj.ofPyIntObject: + raise newException(DictError, retObj.pyType.name) + return hash(PyIntObject(retObj)) + +proc rawEq*(obj1, obj2: PyObject): bool = + ## for type.__eq__ + obj1.id == obj2.id + +proc `==`*(obj1, obj2: PyObject): bool {. inline, cdecl .} = + let fun = obj1.pyType.magicMethods.eq + if fun.isNil: + return rawEq(obj1, obj2) + else: + let retObj = fun(obj1, obj2) + if not retObj.ofPyBoolObject: + raise newException(DictError, retObj.pyType.name) + return PyBoolObject(retObj).b diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 7a1fcb7..b3e8f7d 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -280,6 +280,12 @@ proc implMethod*(prototype, ObjectType, pragmas, body: NimNode, kind: MethodKind # pragmas: custom pragmas # body: function body var (methodName, argTypes) = getNameAndArgTypes(prototype) + if methodName.kind == nnkAccQuoted: # for reversed keyword + var ls = methodName + var name = "" + for i in ls: + name.add i.strVal + methodName = ident name methodName.expectKind({nnkIdent, nnkSym}) ObjectType.expectKind(nnkIdent) body.expectKind(nnkStmtList) diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index 0071a21..f75edc8 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -24,6 +24,8 @@ type BoundMethod, Slice, Cell, + Set, + FrozenSet, when defined(js): var objectId = 0 diff --git a/Objects/setobject.nim b/Objects/setobject.nim new file mode 100644 index 0000000..0c7153f --- /dev/null +++ b/Objects/setobject.nim @@ -0,0 +1,125 @@ + + +import strformat +import strutils +import std/sets +import pyobject +import baseBundle + + +import ./tupleobject +import ../Utils/[utils] +import ./hash + +declarePyType Set(reprLock, mutable, tpToken): + items: HashSet[PyObject] + +declarePyType FrozenSet(reprLock, tpToken): + items: HashSet[PyObject] + +template setSeqToStr(ss): string = '{' & ss.join", " & '}' + +template getItems(s: PyObject): HashSet = + if s.ofPySetObject: PySetObject(s).items + elif s.ofPyFrozenSetObject: PyFrozenSetObject(s).items + else: return newNotImplementedError"" # TODO + +template genOp(S, mutRead, pyOp, nop){.dirty.} = + `impl S Magic` pyOp, mutRead: + let res = `newPy S Simple`() + res.items = nop(self.items, other.getItems) + return res +template genMe(S, mutRead, pyMethod, nop){.dirty.} = + `impl S Method` pyMethod(other: PyObject), mutRead: + let res = `newPy S Simple`() + res.items = nop(self.items, other.getItems) + return res +template genBOp(S, mutRead, pyOp, nop){.dirty.} = + `impl S Magic` pyOp, mutRead: + if nop(self.items, other.getItems): pyTrueObj + else: pyFalseObj +template genBMe(S, mutRead, pyMethod, nop){.dirty.} = + `impl S Method` pyMethod(other: PyObject), mutRead: + if nop(self.items, other.getItems): pyTrueObj + else: pyFalseObj + +template genSet(S, mutRead, mutReadRepr){.dirty.} = + proc `newPy S`*: `Py S Object` = + `newPy S Simple`() + + proc `newPy S`*(items: HashSet[PyObject]): `Py S Object` = + result = `newPy S`() + result.items = items + + `impl S Method` copy(), mutRead: + let newL = `newPy S`() + newL.items = self.items # shallow copy + newL + + genCollectMagics items, + `impl S Magic`, `newPy S Simple`, + `ofPy S Object`, `Py S Object`, + mutRead, mutReadRepr, + setSeqToStr + + + `impl S Magic` hash: + hashImpl items + + `impl S Magic` init: + if 1 < args.len: + let msg = $S & fmt" expected at most 1 args, got {args.len}" + return newTypeError(msg) + if self.items.len != 0: + self.items.clear() + if args.len == 1: + let (iterable, nextMethod) = getIterableWithCheck(args[0]) + if iterable.isThrownException: + return iterable + while true: + let nextObj = nextMethod(iterable) + if nextObj.isStopIter: + break + if nextObj.isThrownException: + return nextObj + self.items.incl nextObj + pyNone + + + genOp S, mutRead, Or, `+` + genMe S, mutRead, union, `+` + genOp S, mutRead, And, `*` + genMe S, mutRead, interaction, `*` + genOp S, mutRead, Xor, `-+-` + genMe S, mutRead, symmetric_difference, `-+-` + genOp S, mutRead, sub, `-` + genMe S, mutRead, difference, `-` + genBOp S, mutRead, le, `<` + genBOp S, mutRead, eq, `==` + genBMe S, mutRead, isdisjoint, disjoint + genBMe S, mutRead, issubset, `<=` + genBMe S, mutRead, issuperset, `>=` + +genSet Set, [mutable: read], [mutable: read, reprLock] +genSet FrozenSet, [], [reprLock] + + +implSetMethod clear(), [mutable: write]: + self.items.clear() + pyNone + +implSetMethod add(item: PyObject), [mutable: write]: + self.items.incl(item) + pyNone + +implSetMethod `discard`(item: PyObject), [mutable: write]: + self.items.excl(item) + pyNone + +implSetMethod pop(), [mutable: write]: + if self.items.len == 0: + let msg = "pop from empty set" + return newKeyError(msg) + self.items.pop + +# TODO: more ... diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 746224d..24dee4c 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -17,30 +17,11 @@ proc newPyTuple*(items: seq[PyObject]): PyTupleObject = result.items = items -template genCollectMagics*(items, `&`, `&=`, +template genCollectMagics*(items, implNameMagic, newPyNameSimple, ofPyNameObject, PyNameObject, mutRead, mutReadRepr, seqToStr){.dirty.} = - implNameMagic add, mutRead: - var res = newPyNameSimple() - if other.ofPyNameObject: - res.items = self.items & PyNameObject(other).items - return res - else: - res.items = self.items - let (iterable, nextMethod) = getIterableWithCheck(other) - if iterable.isThrownException: - return iterable - while true: - let nextObj = nextMethod(iterable) - if nextObj.isStopIter: - break - if nextObj.isThrownException: - return nextObj - `&=` res.items, nextObj - return res - implNameMagic contains, mutRead: for item in self.items: @@ -74,11 +55,29 @@ template genSequenceMagics*(nameStr, seqToStr): untyped{.dirty.} = bind genCollectMagics - genCollectMagics items, `&`, `&=`, + genCollectMagics items, implNameMagic, newPyNameSimple, ofPyNameObject, PyNameObject, mutRead, mutReadRepr, seqToStr + implNameMagic add, mutRead: + var res = newPyNameSimple() + if other.ofPyNameObject: + res.items = self.items & PyNameObject(other).items + return res + else: + res.items = self.items + let (iterable, nextMethod) = getIterableWithCheck(other) + if iterable.isThrownException: + return iterable + while true: + let nextObj = nextMethod(iterable) + if nextObj.isStopIter: + break + if nextObj.isThrownException: + return nextObj + `&=` res.items, nextObj + return res implNameMagic eq, mutRead: if not other.ofPyNameObject: return pyFalseObj diff --git a/Objects/typeobject.nim b/Objects/typeobject.nim index 12b0a78..a826d71 100644 --- a/Objects/typeobject.nim +++ b/Objects/typeobject.nim @@ -9,7 +9,7 @@ import methodobject import funcobjectImpl import descrobject import dictproxyobject - +import ./hash import ../Utils/utils import ../Python/call @@ -59,6 +59,14 @@ proc defaultGe(o1, o2: PyObject): PyObject {. cdecl .} = let eq = o1.callMagic(eq, o2) gt.callMagic(Or, eq) +proc hashDefault(self: PyObject): PyObject {. cdecl .} = + let res = cast[BiggestInt](rawHash(self)) # CPython does so + newPyInt(res) + +proc defaultEq(o1, o2: PyObject): PyObject {. cdecl .} = + if rawEq(o1, o2): pyTrueObj + else: pyFalseObj + proc reprDefault(self: PyObject): PyObject {. cdecl .} = newPyString(fmt"<{self.pyType.name} at {self.idStr}>") @@ -128,9 +136,11 @@ proc addGeneric(t: PyTypeObject) = trySetSlot(ne, defaultNe) if (not nilMagic(ge)) and (not nilMagic(eq)): trySetSlot(ge, defaultGe) + trySetSlot(eq, defaultEq) trySetSlot(getattr, getAttr) trySetSlot(setattr, setAttr) trySetSlot(repr, reprDefault) + trySetSlot(hash, hashDefault) trySetSlot(str, t.magicMethods.repr) diff --git a/Python/ast.nim b/Python/ast.nim index 8978e1a..40f25ce 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -1156,23 +1156,41 @@ ast testlist, [AsdlExpr]: # (comp_for | (',' (test | star_expr))* [','])) ) ast dictorsetmaker, [AsdlExpr]: let children = parseNode.children - let d = newAstDict() + let le = children.len + if le == 0: # {} -> dict() + return newAstDict() + elif le == 1: + let s = newAstSet() + s.elts.add astTest children[0] + return s + # Then `children[1]` won't go out of bound + let leFix = le + 1 # XXX: + 1 to add tailing comma. So for list,etc. FIXME: allow trailing comma + let isDict = children[1].tokenNode.token == Token.Colon # no need to care about setting lineNo and colOffset, because `atom` does so - for idx in 0..<((children.len+1) div 4): - let i = idx * 4 - if children.len < i + 3: - raiseSyntaxError("dict definition too complex (no set, no comprehension)") - let c1 = children[i] - if not (c1.tokenNode.token == Token.test): - raiseSyntaxError("dict definition too complex (no set, no comprehension)", c1) - d.keys.add(astTest(c1)) - if not (children[i+1].tokenNode.token == Token.Colon): - raiseSyntaxError("dict definition too complex (no set, no comprehension)") - let c3 = children[i+2] - if not (c3.tokenNode.token == Token.test): - raiseSyntaxError("dict definition too complex (no set, no comprehension)", c3) - d.values.add(astTest(c3)) - result = d + if isDict: + let d = newAstDict() + for idx in 0..<(leFix div 4): + let i = idx * 4 + if children.len < i + 3: + raiseSyntaxError("dict definition too complex (no comprehension)") + let c1 = children[i] + if not (c1.tokenNode.token == Token.test): + raiseSyntaxError("dict definition too complex (no comprehension)", c1) + d.keys.add(astTest(c1)) + if not (children[i+1].tokenNode.token == Token.Colon): + raiseSyntaxError("dict definition too complex (no comprehension)") + let c3 = children[i+2] + if not (c3.tokenNode.token == Token.test): + raiseSyntaxError("dict definition too complex (no comprehension)", c3) + d.values.add(astTest(c3)) + result = d + else: + let s = newAstSet() + for i in 0..<(leFix div 2): + let c = children[i * 2] + s.elts.add(astTest(c)) + result = s + # classdef: 'class' NAME ['(' [arglist] ')'] ':' suite ast classdef, [AstClassDef]: diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 8c337f1..2fcad88 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -89,6 +89,8 @@ implBltinFunc len(obj: PyObject): obj.callMagic(len) +implBltinFunc hash(obj: PyObject): obj.callMagic(hash) + implBltinFunc iter(obj: PyObject): obj.callMagic(iter) implBltinFunc repr(obj: PyObject): obj.callMagic(repr) @@ -110,6 +112,8 @@ registerBltinObject("range", pyRangeObjectType) registerBltinObject("list", pyListObjectType) registerBltinObject("tuple", pyTupleObjectType) registerBltinObject("dict", pyDictObjectType) +registerBltinObject("set", pySetObjectType) +registerBltinObject("frozenset", pyFrozenSetObjectType) registerBltinObject("int", pyIntObjectType) registerBltinObject("str", pyStrObjectType) registerBltinObject("property", pyPropertyObjectType) diff --git a/Python/compile.nim b/Python/compile.nim index f8049df..29308b5 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -668,6 +668,17 @@ compileMethod UnaryOp: let opCode = astNode.op.toOpCode c.addOp(newInstr(opCode, astNode.lineNo.value)) +compileMethod Set: + let n = astNode.elts.len + for i in 0.. Date: Mon, 14 Jul 2025 07:29:28 +0800 Subject: [PATCH 042/163] feat(inner): dedup iterable loop as pyForIn --- Objects/iterobject.nim | 13 +++++++++++++ Objects/listobject.nim | 10 +--------- Objects/tupleobject.nim | 24 ++++-------------------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/Objects/iterobject.nim b/Objects/iterobject.nim index 5775e1a..c846857 100644 --- a/Objects/iterobject.nim +++ b/Objects/iterobject.nim @@ -18,3 +18,16 @@ implSeqIterMagic iternext: proc newPySeqIter*(items: seq[PyObject]): PySeqIterObject = result = newPySeqIterSimple() result.items = items + +template pyForIn*(it; iterableToLoop: PyObject; doWithIt) = + ## pesudo code: `for it in iterableToLoop: doWithIt` + let (iterable, nextMethod) = getIterableWithCheck(iterableToLoop) + if iterable.isThrownException: + return iterable + while true: + let it = nextMethod(iterable) + if it.isStopIter: + break + if it.isThrownException: + return it + doWithIt diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 62161b3..a2cfab1 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -84,15 +84,7 @@ implListMethod extend(other: PyObject), [mutable: write]: if other.ofPyListObject: self.items &= PyListObject(other).items else: - let (iterable, nextMethod) = getIterableWithCheck(other) - if iterable.isThrownException: - return iterable - while true: - let nextObj = nextMethod(iterable) - if nextObj.isStopIter: - break - if nextObj.isThrownException: - return nextObj + pyForIn nextObj, other: self.items.add nextObj pyNone diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 24dee4c..07a5b47 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -67,16 +67,8 @@ template genSequenceMagics*(nameStr, return res else: res.items = self.items - let (iterable, nextMethod) = getIterableWithCheck(other) - if iterable.isThrownException: - return iterable - while true: - let nextObj = nextMethod(iterable) - if nextObj.isStopIter: - break - if nextObj.isThrownException: - return nextObj - `&=` res.items, nextObj + pyForIn i, other: + `&=` res.items, i return res implNameMagic eq, mutRead: if not other.ofPyNameObject: @@ -103,16 +95,8 @@ template genSequenceMagics*(nameStr, if self.items.len != 0: self.items.setLen(0) if args.len == 1: - let (iterable, nextMethod) = getIterableWithCheck(args[0]) - if iterable.isThrownException: - return iterable - while true: - let nextObj = nextMethod(iterable) - if nextObj.isStopIter: - break - if nextObj.isThrownException: - return nextObj - self.items.add nextObj + pyForIn i, args[0]: + self.items.add i pyNone From 1e18879cddc483677b821001e3f1ee5c099ec107 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 07:37:25 +0800 Subject: [PATCH 043/163] feat(set): all method (not test) --- Objects/setobject.nim | 68 +++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/Objects/setobject.nim b/Objects/setobject.nim index 0c7153f..2c58691 100644 --- a/Objects/setobject.nim +++ b/Objects/setobject.nim @@ -5,7 +5,7 @@ import strutils import std/sets import pyobject import baseBundle - +import ./iterobject import ./tupleobject import ../Utils/[utils] @@ -19,10 +19,20 @@ declarePyType FrozenSet(reprLock, tpToken): template setSeqToStr(ss): string = '{' & ss.join", " & '}' -template getItems(s: PyObject): HashSet = +template getItems(s: PyObject, elseDo): HashSet = if s.ofPySetObject: PySetObject(s).items elif s.ofPyFrozenSetObject: PyFrozenSetObject(s).items - else: return newNotImplementedError"" # TODO + else: elseDo + +template getItems(s: PyObject): HashSet = + getItems s: return newNotImplementedError"" # TODO + + +template getItemsMayIter(s: PyObject): HashSet = + getItems s: + PyFrozenSetObject( + pyFrozenSetObjectType.pyType.magicMethods.init(s, @[]) + ).items template genOp(S, mutRead, pyOp, nop){.dirty.} = `impl S Magic` pyOp, mutRead: @@ -73,16 +83,8 @@ template genSet(S, mutRead, mutReadRepr){.dirty.} = if self.items.len != 0: self.items.clear() if args.len == 1: - let (iterable, nextMethod) = getIterableWithCheck(args[0]) - if iterable.isThrownException: - return iterable - while true: - let nextObj = nextMethod(iterable) - if nextObj.isStopIter: - break - if nextObj.isThrownException: - return nextObj - self.items.incl nextObj + pyForIn i, args[0]: + self.items.incl i pyNone @@ -104,6 +106,25 @@ genSet Set, [mutable: read], [mutable: read, reprLock] genSet FrozenSet, [], [reprLock] +implSetMethod update(args), [mutable: write]: + for other in args: + self.items.incl(other.getItemsMayIter) + pyNone + +implSetMethod intersection_update(args), [mutable: write]: + for other in args: + self.items = self.items * (other.getItemsMayIter) + pyNone + +implSetMethod difference_update(args), [mutable: write]: + for other in args: + self.items = self.items - (other.getItemsMayIter) + pyNone + +implSetMethod symmetric_difference_update(other: PyObject), [mutable: write]: + self.items = self.items -+- (other.getItemsMayIter) + + implSetMethod clear(), [mutable: write]: self.items.clear() pyNone @@ -113,13 +134,30 @@ implSetMethod add(item: PyObject), [mutable: write]: pyNone implSetMethod `discard`(item: PyObject), [mutable: write]: - self.items.excl(item) + if item.ofPyFrozenSetObject or item.ofPySetObject: + self.items.excl item.getItems + else: + self.items.excl(item) pyNone +proc removeImpl(self: PySetObject, item: PyObject): PyObject = + if self.items.missingOrExcl(item): + newKeyError(PyStrObject(item.callMagic(repr)).str) + else: + pyNone + +implSetMethod remove(item: PyObject), [mutable: write]: + if item.ofPyFrozenSetObject or item.ofPySetObject: + return self.removeImpl(item) + else: + pyForIn i, args[0]: + result = self.removeImpl(i) + if result.isThrownException: + return + implSetMethod pop(), [mutable: write]: if self.items.len == 0: let msg = "pop from empty set" return newKeyError(msg) self.items.pop -# TODO: more ... From 2208c7f9d9a0da5757ce380425aa34d27d1195c7 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 16:26:22 +0800 Subject: [PATCH 044/163] feat(syntax/indent): allow any number of spaces (tab as 8 spaces) --- Parser/lexer.nim | 67 +++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/Parser/lexer.nim b/Parser/lexer.nim index e7173e3..2ced3fe 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -21,11 +21,13 @@ type Eval Lexer* = ref object - indentLevel: int + indentStack: seq[int] # Stack to track indentation levels lineNo: int tokenNodes*: seq[TokenNode] # might be consumed by parser fileName*: string +template indentLevel(lexer: Lexer): int = lexer.indentStack[^1] + var sourceFiles = initTable[string, Source]() proc addSource*(filePath, content: string) = @@ -73,6 +75,7 @@ proc newTokenNode*(token: Token, proc newLexer*(fileName: string): Lexer = new result result.fileName = fileName + result.indentStack = @[0] # Start with a single zero on the stack # when we need a fresh start in interactive mode proc clearTokens*(lexer: Lexer) = @@ -90,11 +93,15 @@ proc add(lexer: Lexer, token: Token, colNo:int) = assert token.isTerminator lexer.add(newTokenNode(token, lexer.lineNo, colNo)) +template delLast(s: seq) = + discard s.pop() +# At the end of the file, generate DEDENT tokens for remaining stack levels proc dedentAll*(lexer: Lexer) = - while lexer.indentLevel != 0: - lexer.add(Token.Dedent, lexer.indentLevel * 4) - dec lexer.indentLevel + while lexer.indentStack.len > 1: + lexer.indentStack.delLast() + lexer.add(Token.Dedent, lexer.indentStack[^1]) + #dec lexer.indentLevel # the function can probably be generated by a macro... proc getNextToken( @@ -263,40 +270,54 @@ proc getNextToken( assert result != nil -proc lexOneLine(lexer: Lexer, line: string) = - # process one line at a time +proc lexOneLine(lexer: Lexer, line: string, mode: Mode) {.inline.} = + # Process one line at a time assert line.find("\n") == -1 var idx = 0 + var indentLevel = 0 + + # Calculate the indentation level based on spaces and tabs + while idx < line.len: + case line[idx] + of ' ': + indentLevel += 1 + inc(idx) + of '\t': + indentLevel += 8 # XXX: Assume a tab equals 8 spaces + inc(idx) + else: + break - while idx < line.len and line[idx] == ' ': - inc(idx) - if idx == line.len or line[idx] == '#': # full of spaces or comment line + if idx == line.len or line[idx] == '#': # Full of spaces or comment line return - if idx mod 4 != 0: - raiseSyntaxError("Indentation must be 4 spaces.", "", lexer.lineNo, 0) - let indentLevel = idx div 4 - let diff = indentLevel - lexer.indentLevel - if diff < 0: - for i in diff..<0: - lexer.add(Token.Dedent, (lexer.indentLevel+i)*4) - else: - for i in 0.. currentIndent: + lexer.indentStack.add(indentLevel) + lexer.add(Token.Indent, idx) + elif indentLevel < currentIndent: + while lexer.indentLevel > indentLevel: + lexer.indentStack.delLast() + lexer.add(Token.Dedent, idx) + if lexer.indentLevel != indentLevel: + raiseSyntaxError "Indentation error", lexer.fileName, lexer.lineNo + + # Update the lexer's current indentation level lexer.indentLevel = indentLevel + # Process the rest of the line for tokens while idx < line.len: case line[idx] - of ' ': + of Whitespace - Newlines: inc idx - of '#': + of '#': # Comment line break else: lexer.add(getNextToken(lexer, line, idx)) lexer.add(Token.NEWLINE, idx) - proc lexString*(lexer: Lexer, input: string, mode=Mode.File) = assert mode != Mode.Eval # eval not tested @@ -312,7 +333,7 @@ proc lexString*(lexer: Lexer, input: string, mode=Mode.File) = # lineNo starts from 1 inc lexer.lineNo addSource(lexer.fileName, input) - lexer.lexOneLine(line) + lexer.lexOneLine(line, mode) when defined(debug): echo lexer.tokenNodes From 3a5b72a48e4b86a9d1d4337438eec814927758d0 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 18:23:09 +0800 Subject: [PATCH 045/163] fix(int): -1+10 = -9 --- Objects/numobjects.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index 418be1c..bd559de 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -250,7 +250,7 @@ proc `+`*(a, b: PyIntObject): PyIntObject = of Zero: return a of Positive: - return doSub(a, b) + return doSub(b, a) of Zero: return b of Positive: From 0c52a12dfc2bee96f9590b0b5ed4bfe0db57d91b Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 14 Jul 2025 18:23:58 +0800 Subject: [PATCH 046/163] fix(int): 1-10 = 8589934583 --- Objects/numobjects.nim | 51 +++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index bd559de..b49eb80 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -131,6 +131,7 @@ proc inplaceAdd(a: PyIntObject, b: Digit) = a.digits.add truncate(carry) +# assuming all positive, return a + b proc doAdd(a, b: PyIntObject): PyIntObject = if a.digits.len < b.digits.len: return doAdd(b, a) @@ -148,32 +149,40 @@ proc doAdd(a, b: PyIntObject): PyIntObject = # assuming all positive, return a - b proc doSub(a, b: PyIntObject): PyIntObject = - if a.digits.len < b.digits.len: - let c = doSub(b, a) - c.sign = Negative - return c - var carry = Digit(0) result = newPyIntSimple() - for i in 0.. Date: Tue, 15 Jul 2025 03:52:53 +0800 Subject: [PATCH 047/163] impr(int) `$` faster --- Objects/numobjects.nim | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index b49eb80..26d2c95 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -472,21 +472,20 @@ proc fromStr(s: string): PyIntObject = else: result.sign = Positive -method `$`*(i: PyIntObject): string = - var strSeq: seq[string] +method `$`*(i: PyIntObject): string = if i.zero: return "0" var ii = i.copy() var r: PyIntObject while true: (ii, r) = ii.doDiv pyIntTen - strSeq.add($int(r.digits[0])) + result.add(char(r.digits[0] + Digit('0'))) if ii.digits.len == 0: break #strSeq.add($i.digits) if i.negative: - strSeq.add("-") - strSeq.reversed.join() + result.add '-' + result.reverse proc hash*(self: PyIntObject): Hash {. inline, cdecl .} = result = hash(self.sign) From 77872057ef2a1bca44c5984e98ecc8a62fec8da0 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 06:28:15 +0800 Subject: [PATCH 048/163] feat(int,float): support: a//b for int where b is big; `%` for int,float --- Objects/numobjects.nim | 517 ++++++++++++++++++++++++++++++---------- Objects/rangeobject.nim | 4 +- 2 files changed, 394 insertions(+), 127 deletions(-) diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index 26d2c95..de6281b 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -8,14 +8,24 @@ import macros import strformat import strutils import math +import std/bitops +const BitPerByte = 8 +proc bit_length*(self: SomeInteger): int = + when defined(noUndefinedBitOpts): + sizeof(x) * BitPerByte bitops.countLeadingZeroBits x + else: + 1 + fastLog2( + when self is SomeSignedInt: abs(self) + else: self + ) + import pyobject import exceptions import boolobject import stringobject import ../Utils/utils - -# this is a **very slow** bigint lib, div support is not complete. +# this is a **very slow** bigint lib. # Why reinvent such a bad wheel? # because we seriously need low level control on our modules # todo: make it a decent bigint module @@ -25,12 +35,10 @@ when defined(js): type Digit = uint16 TwoDigits = uint32 + SDigit = int32 const digitBits = 16 - proc `$`(i: uint16|uint32): string = - $int(i) - template truncate(x: TwoDigits): Digit = const mask = 0x0000FFFF Digit(x and mask) @@ -39,16 +47,18 @@ else: type Digit = uint32 TwoDigits = uint64 + SDigit = int64 const digitBits = 32 template truncate(x: TwoDigits): Digit = Digit(x) -const maxValue = TwoDigits(high(Digit)) + 1 +type STwoDigit = SDigit -template promote(x: Digit): TwoDigits = - TwoDigits(x) shl digitBits + +const maxValue = TwoDigits(high(Digit)) + 1 +const sMaxValue = SDigit(high(Digit)) + 1 template demote(x: TwoDigits): Digit = Digit(x shr digitBits) @@ -65,15 +75,50 @@ declarePyType Int(tpToken): sign: IntSign digits: seq[Digit] +#proc compatSign(op: PyIntObject): SDigit{.inline.} = cast[SDigit](op.sign) +# NOTE: CPython uses 0,1,2 for IntSign, so its `_PyLong_CompactSign` is `1 - sign` + #proc newPyInt(i: Digit): PyIntObject -proc newPyInt*(i: int): PyIntObject +proc newPyInt*(o: PyIntObject): PyIntObject = + ## deep copy, returning a new object + result = newPyIntSimple() + result.sign = o.sign + result.digits = o.digits -# currently avoid using setLen because of gh-10651 -proc setXLen(intObj: PyIntObject, l: int) = - if intObj.digits.len == 0: - intObj.digits = newSeq[Digit](l) +proc newPyInt*(i: Digit): PyIntObject = + result = newPyIntSimple() + if i != 0: + result.digits.add i + result.sign = Positive + # can't be negative else: - intObj.digits.setLen(l) + result.sign = Zero + +proc newPyInt*[I: SomeSignedInt](i: I): PyIntObject = + result = newPyIntSimple() + var ui = abs(i) + while ui != 0: + result.digits.add Digit( + when sizeof(I) <= sizeof(SDigit): ui + else: ui mod I(sMaxValue) + ) + ui = ui shr digitBits + + if i < 0: + result.sign = Negative + elif i == 0: + result.sign = Zero + else: + result.sign = Positive + +proc newPyIntOfLen(l: int): PyIntObject = + ## `long_alloc` + ## + ## result sign is `Positive` if l != 0; `Zero` otherwise + result = newPyIntSimple() + result.digits.setLen(l) + if l != 0: + result.sign = Positive let pyIntZero* = newPyInt(0) let pyIntOne* = newPyInt(1) @@ -90,6 +135,7 @@ proc positive*(intObj: PyIntObject): bool {. inline .} = intObj.sign == Positive proc copy(intObj: PyIntObject): PyIntObject = + ## XXX: copy only digits (sign uninit!) let newInt = newPyIntSimple() newInt.digits = intObj.digits newInt @@ -308,6 +354,9 @@ proc `-`*(a, b: PyIntObject): PyIntObject = of Positive: return doSub(a, b) +proc negate*(self: PyIntObject){.inline.} = + self.sign = Negative + proc `-`*(a: PyIntObject): PyIntObject = result = a.copy() result.sign = IntSign(-int(a.sign)) @@ -343,98 +392,335 @@ proc `*`*(a, b: PyIntObject): PyIntObject = return c -proc doDiv(n, d: PyIntObject): (PyIntObject, PyIntObject) = - var - nn = n.digits.len - dn = d.digits.len - assert nn != 0 - - if nn < dn: - return (pyIntZero, n) - elif dn == 1: - let dd = d.digits[0] - let q = newPyIntSimple() - var rr: Digit - q.setXLen(n.digits.len) - - for i in countdown(n.digits.high, 0): - let tmp = TwoDigits(n.digits[i]) + rr.promote - q.digits[i] = truncate(tmp div TwoDigits(dd)) - rr = truncate(tmp mod Twodigits(dd)) - - q.normalize() - let r = newPyIntSimple() - r.digits.add rr - return (q, r) +template fastExtract(a, b){.dirty.} = + let + left = SDigit a.digits[0] + right = SDigit b.digits[0] + when check: + assert a.digits.len == 1 + assert b.digits.len == 1 + +proc fast_floor_div(a, b: PyIntObject; check: static[bool] = true): PyIntObject = + fastExtract a, b + newPyInt floorDiv(left, right) + +proc fast_mod(a, b: PyIntObject; check: static[bool] = true): PyIntObject = + fastExtract a, b + #let sign = b.compatSign + newPyInt floorMod(left, right) + +proc inplaceDivRem1(pout: var openArray[Digit], pin: openArray[Digit], size: int, n: Digit): Digit = + ## Perform in-place division with remainder for a single digit + var remainder: TwoDigits = 0 + assert n > 0 and n < maxValue + + for i in countdown(size - 1, 0): + let dividend = (remainder shl digitBits) or TwoDigits(pin[i]) + let quotient = truncate(dividend div n) + remainder = dividend mod n + pout[i] = quotient + + return Digit(remainder) + +proc divRem1(a: PyIntObject, n: Digit, remainder: var Digit): PyIntObject = + ## Divide an integer by a single digit, returning both quotient and remainder + ## The sign of a is ignored; n should not be zero. + ## + ## the result's sign is always positive + assert n > 0 and n < maxValue + let size = a.digits.len + let quotient = newPyIntOfLen(size) + remainder = inplaceDivRem1(quotient.digits, a.digits, size, n) + quotient.normalize() + return quotient + +proc inplaceRem1(pin: var seq[Digit], size: int, n: Digit): Digit = + ## Compute the remainder of a multi-digit integer divided by a single digit. + ## `pin` points to the least significant digit (LSD). + var rem: TwoDigits = 0 + assert n > 0 and n <= maxValue - 1 + + for i in countdown(size - 1, 0): + rem = ((rem shl digitBits) or TwoDigits(pin[i])) mod TwoDigits(n) + + return Digit(rem) + +proc rem1(a: PyIntObject, n: Digit): PyIntObject = + ## Get the remainder of an integer divided by a single digit. + ## The sign of `a` is ignored; `n` should not be zero. + assert n > 0 and n <= maxValue - 1 + let size = a.digits.len + let remainder = inplaceRem1(a.digits, size, n) + return newPyInt(remainder) + +proc tryRem(a, b: PyIntObject, prem: var PyIntObject): bool +proc lMod(v, w: PyIntObject, modRes: var PyIntObject): bool = + ## Compute modulus: *modRes = v % w + ## returns w != 0 + #assert modRes != nil + if v.digits.len == 1 and w.digits.len == 1: + modRes = fast_mod(v, w) + return not modRes.isNil + + if not tryRem(v, w, modRes): + return + + # Adjust signs if necessary + if (modRes.sign == Negative and w.sign == Positive) or + (modRes.sign == Positive and w.sign == Negative): + modRes = modRes + w + +template retZeroDiv = + return newZeroDivisionError"division by zero" + +proc `%`*(a, b: PyIntObject): PyObject = + var res: PyIntObject + if lMod(a, b, res): + retZeroDiv + result = res + +proc lDivmod(v, w: PyIntObject, divRes, modRes: var PyIntObject): bool + +template fastDivIf1len(a, b: PyIntObject) = + if a.digits.len == 1 and b.digits.len == 1: + return fast_floor_div(a, b) + +template lDiv(a, b: PyIntObject; result): bool = + var unused: PyIntObject + lDivmod(a, b, result, unused) + +proc floorDivNonZero*(a, b: PyIntObject): PyIntObject = + ## `long_div` + ## Integer division + ## + ## assuming b is non-zero + fastDivIf1len a, b + assert lDiv(a, b, result) + +proc `//`*(a, b: PyIntObject): PyObject = + ## `long_div` + ## Integer division + ## + ## .. note:: this may returns ZeroDivisionError + fastDivIf1len a, b + var res: PyIntObject + if lDiv(a, b, res): + return res + retZeroDiv + +proc divmodNonZero*(a, b: PyIntObject): tuple[d, m: PyIntObject] = + assert lDivmod(a, b, result[0], result[1]) + +proc divmod*(a, b: PyIntObject): (PyIntObject, PyIntObject) = + ## + var d, m: PyIntObject + if not lDivmod(a, b, d, m): + raise newException(ValueError, "division by zero") + (d, m) + + +proc vLShift(z, a: var seq[Digit], m: int, d: int): Digit = + ## Shift digit vector `a[0:m]` left by `d` bits, with 0 <= d < digitBits. + ## Put the result in `z[0:m]`, and return the `d` bits shifted out of the top. + assert d >= 0 and d < digitBits + var carry: Digit = 0 + for i in 0..= 0 and d < digitBits + var carry: Digit = 0 + let mask = (Digit(1) shl d) - 1 + + for i in countdown(m - 1, 0): + let acc = (TwoDigits(carry) shl digitBits) or TwoDigits(a[i]) + carry = Digit(acc and mask) + z[i] = Digit(acc shr d) + + return carry +proc xDivRem(v1, w1: PyIntObject, prem: var PyIntObject): PyIntObject = + ## `x_divrem` + ## Perform unsigned integer division with remainder + var v, w, a: PyIntObject + var sizeV = v1.digits.len + var sizeW = w1.digits.len + assert sizeV >= sizeW and sizeW >= 2 + + # Allocate space for v and w + v = newPyIntSimple() + v.digits.setLen(sizeV + 1) + w = newPyIntSimple() + w.digits.setLen(sizeW) + + # Normalize: shift w1 left so its top digit is >= maxValue / 2 + let d = digitBits - bitLength(w1.digits[^1]) + let carryW = vLShift(w.digits, w1.digits, sizeW, d) + assert carryW == 0 + let carryV = vLShift(v.digits, v1.digits, sizeV, d) + if carryV != 0 or v.digits[^1] >= w.digits[^1]: + v.digits.add carryV + inc sizeV + + # Quotient has at most `k = sizeV - sizeW` digits + let k = sizeV - sizeW + assert k >= 0 + a = newPyIntSimple() + a.digits.setLen(k) + + var v0 = v.digits + let w0 = w.digits + let wm1 = w0[^1] + let wm2 = w0[^2] + + for vk in countdown(k - 1, 0): + # Estimate quotient digit `q` + let vtop = v0[vk + sizeW] + assert vtop <= wm1 + let vv = (TwoDigits(vtop) shl digitBits) or TwoDigits(v0[vk + sizeW - 1]) + var q = Digit(vv div wm1) + var r = Digit(vv mod wm1) + + while TwoDigits(wm2) * TwoDigits(q) > ((TwoDigits(r) shl digitBits) or TwoDigits(v0[vk + sizeW - 2])): + dec q + r += wm1 + if r >= maxValue: + break + assert q <= maxValue + + # Subtract `q * w0[0:sizeW]` from `v0[vk:vk+sizeW+1]` + var zhi: SDigit = 0 + for i in 0..= dn and dn >= 2 - raise newException(IntError, "") - - -proc `div`*(a, b: PyIntObject): PyIntObject = - case a.sign - of Negative: - case b.sign - of Negative: - let (q, r) = doDiv(a, b) - if q.digits.len == 0: - q.sign = Zero - else: - q.sign = Positive - return q - of Zero: - assert false - of Positive: - let (q, r) = doDiv(a, b) - if q.digits.len == 0: - q.sign = Zero - else: - q.sign = Negative - return q - of Zero: - return pyIntZero - of Positive: - case b.sign - of Negative: - let (q, r) = doDiv(a, b) - if q.digits.len == 0: - q.sign = Zero - else: - q.sign = Negative - return q - of Zero: - assert false - of Positive: - let (q, r) = doDiv(a, b) - if q.digits.len == 0: - q.sign = Zero - else: - q.sign = Positive - return q + discard xDivRem(a, b, prem) + + #[ Set the sign.]# + if (a.sign == Negative) and not prem.zero(): + prem.negate() + +proc tryDivrem(a, b: PyIntObject, pdiv, prem: var PyIntObject): bool = + ## `long_divrem` + ## Integer division with remainder + ## + ## returns if `b` is non-zero (only false when b is zero) + let sizeA = a.digits.len + let sizeB = b.digits.len + + if sizeB == 0: + #raise newException(ZeroDivisionError, "division by zero") + return + + result = true + if sizeA < sizeB or ( + sizeA == sizeB and a.digits[^1] < b.digits[^1]): + # |a| < |b| + prem = newPyInt(a) + pdiv = pyIntZero + return + + if sizeB == 1: + var remainder: Digit + pdiv = divRem1(a, b.digits[0], remainder) + prem = newPyInt(remainder) + else: + pdiv = xDivRem(a, b, prem) + + #[ Set the signs. + The quotient pdiv has the sign of a*b; + the remainder prem has the sign of a, + so a = b*z + r.]# + if (a.sign == Negative) != (b.sign == Negative): + pdiv.negate() + if (a.sign == Negative) and not prem.zero(): + prem.negate() + + +proc lDivmod(v, w: PyIntObject, divRes, modRes: var PyIntObject): bool = + ## Python's returns -1 on failure, which is only to be Memory Alloc failure + ## where nim will just `SIGSEGV` + ## + ## returns w != 0 + result = true + + # Fast path for single-digit longs + if v.digits.len == 1 and w.digits.len == 1: + divRes = fast_floor_div(v, w, off) + modRes = fast_mod(v, w, off) + return + + # Perform long division and remainder + if not tryDivrem(v, w, divRes, modRes): return false + + # Adjust signs if necessary + if (modRes.sign == Negative and w.sign == Positive) or + (modRes.sign == Positive and w.sign == Negative): + modRes = modRes + w + divRes = divRes - pyIntOne # a**b proc pow(a, b: PyIntObject): PyIntObject = assert(not b.negative) if b.zero: return pyIntOne - let new_b = b div pyIntTwo + # we have checked b is not zero + let new_b = b.floorDivNonZero pyIntTwo let half_c = pow(a, new_b) if b.digits[0] mod 2 == 1: return half_c * half_c * a else: return half_c * half_c -#[ -proc newPyInt(i: Digit): PyIntObject = - result = newPyIntSimple() - if i != 0: - result.digits.add i - # can't be negative - if i == 0: - result.sign = Zero - else: - result.sign = Positive +#[ proc newPyInt(i: int): PyIntObject = var ii: int if i < 0: @@ -453,18 +739,15 @@ proc newPyInt(i: int): PyIntObject = ]# proc fromStr(s: string): PyIntObject = result = newPyIntSimple() - var start = 0 - var sign: IntSign + var sign = s[0] == '-' # assume s not empty - if s[0] == '-': - start = 1 result.digits.add 0 - for i in start.. Date: Tue, 15 Jul 2025 06:47:22 +0800 Subject: [PATCH 049/163] feat(float): divmod (inner) --- Objects/numobjects.nim | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index de6281b..52c23da 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -509,15 +509,12 @@ proc `//`*(a, b: PyIntObject): PyObject = retZeroDiv proc divmodNonZero*(a, b: PyIntObject): tuple[d, m: PyIntObject] = - assert lDivmod(a, b, result[0], result[1]) + ## export for builtins.divmod + assert lDivmod(a, b, result.d, result.m) -proc divmod*(a, b: PyIntObject): (PyIntObject, PyIntObject) = - ## - var d, m: PyIntObject - if not lDivmod(a, b, d, m): +proc divmod*(a, b: PyIntObject): tuple[d, m: PyIntObject] = + if not lDivmod(a, b, result.d, result.m): raise newException(ValueError, "division by zero") - (d, m) - proc vLShift(z, a: var seq[Digit], m: int, d: int): Digit = ## Shift digit vector `a[0:m]` left by `d` bits, with 0 <= d < digitBits. @@ -966,12 +963,34 @@ implFloatMagic mul, [castOther]: implFloatMagic trueDiv, [castOther]: newPyFloat(self.v / casted.v) +proc floorDivNonZero(a, b: PyFloatObject): PyFloatObject = + newPyFloat(floor(a.v / b.v)) + +proc floorModNonZero(a, b: PyFloatObject): PyFloatObject = + newPyFloat(floorMod(a.v, b.v)) + +template genDivOrMod(dm, mag){.dirty.} = + proc `floor dm`(a, b: PyFloatObject): PyObject = + if b.v == 0: + retZeroDiv + `floor dm NonZero` a, b -implFloatMagic floorDiv, [castOther]: - newPyFloat(floor(self.v / casted.v)) + implFloatMagic mag, [castOther]: + `floor dm` self, casted + +genDivOrMod Div, floorDiv +genDivOrMod Mod, Mod + +proc divmodNonZero*(a, b: PyFloatObject): tuple[d, m: PyFloatObject] = + ## export for builtins.divmod + result.d = a.floorDivNonZero b + result.m = a.floorModNonZero b + +proc divmod*(a, b: PyFloatObject): tuple[d, m: PyFloatObject] = + if b.v == 0.0: + raise newException(ValueError, "division by zero") + divmodNonZero(a, b) -implFloatMagic Mod, [castOther]: - newPyFloat((self.v.floorMod casted.v)) implFloatMagic pow, [castOther]: newPyFloat(self.v.pow(casted.v)) From 14dc7a3ce31cd99b8a3495c9e1e6087a4e1ba91e Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 07:11:03 +0800 Subject: [PATCH 050/163] fix(py): repr for self-containing dict,list was "..." --- Objects/dictobject.nim | 2 +- Objects/listobject.nim | 2 +- Objects/pyobject.nim | 10 ++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index b611469..952ace3 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -54,7 +54,7 @@ implDictMagic contains, [mutable: read]: else: return pyTrueObj -implDictMagic repr, [mutable: read, reprLock]: +implDictMagic repr, [mutable: read, reprLockWithMsg"{...}"]: var ss: seq[string] for k, v in self.table.pairs: let kRepr = k.callMagic(repr) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index a2cfab1..9f48687 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -25,7 +25,7 @@ template lsSeqToStr(ss): string = '[' & ss.join", " & ']' genSequenceMagics "list", implListMagic, implListMethod, ofPyListObject, PyListObject, - newPyListSimple, [mutable: read], [reprLock, mutable: read], + newPyListSimple, [mutable: read], [reprLockWithMsg"[...]", mutable: read], lsSeqToStr implListMagic setitem, [mutable: write]: diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index b3e8f7d..e377bdf 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -384,10 +384,10 @@ proc implMethod*(prototype, ObjectType, pragmas, body: NimNode, kind: MethodKind discard -macro reprLock*(code: untyped): untyped = +proc reprLockImpl(s, code: NimNode): NimNode = let reprEnter = quote do: if self.reprLock: - return newPyString("...") + return newPyString(`s`) self.reprLock = true let reprLeave = quote do: @@ -406,6 +406,12 @@ macro reprLock*(code: untyped): untyped = ) code +macro reprLockWithMsg*(s: string, code: untyped): untyped = + reprLockImpl(s, code) + +macro reprLock*(code: untyped): untyped = + reprLockImpl(newLit"...", code) + template allowSelfReadWhenBeforeRealWrite*(body) = self.writeLock = false body From 1be7e5235afe238eab8706cf086d21e00677a209 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 07:27:16 +0800 Subject: [PATCH 051/163] fix(py): type name was captial e.g. Set --- Objects/pyobject.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index e377bdf..23b3eae 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -589,7 +589,7 @@ macro declarePyType*(prototype, fields: untyped): untyped = `py name ObjectType`.magicMethods.New = `newPy name Default` result.add(getAst(initTypeTmpl(nameIdent, - nameIdent.strVal, + nameIdent.strVal.toLowerAscii, newLit(tpToken), newLit(dict) ))) From 2698008997bdd2ef4bb02b51a54d199da12206e6 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 07:27:40 +0800 Subject: [PATCH 052/163] fix(py): repr(set()) was "{}" --- Objects/setobject.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.nim b/Objects/setobject.nim index 2c58691..0bd5792 100644 --- a/Objects/setobject.nim +++ b/Objects/setobject.nim @@ -17,7 +17,11 @@ declarePyType Set(reprLock, mutable, tpToken): declarePyType FrozenSet(reprLock, tpToken): items: HashSet[PyObject] -template setSeqToStr(ss): string = '{' & ss.join", " & '}' +template setSeqToStr(ss): string = + if ss.len == 0: + self.pyType.name & "()" + else: + '{' & ss.join", " & '}' template getItems(s: PyObject, elseDo): HashSet = if s.ofPySetObject: PySetObject(s).items From 1d5a0c8d006624284840afc892acc860e0ec7ea0 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 10:31:12 +0800 Subject: [PATCH 053/163] fix(js): AstNodeBase.hash crash bc cast to int from ref --- Python/asdl.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Python/asdl.nim b/Python/asdl.nim index c9a1a27..48567d1 100644 --- a/Python/asdl.nim +++ b/Python/asdl.nim @@ -20,7 +20,12 @@ type method hash*(node: AstNodeBase): Hash {. base .} = - hash(cast[int](node)) + when defined(js): + hashes.hash(cast[pointer](node)) + # NIM-BUG: if cast[int]: + # at rt: SyntaxError: Cannot convert [object Object] to a BigInt + else: + hash(cast[int](node)) proc genMember(member: NimNode): NimNode = From e95c3fb4a8da1f6391cb74bb67f3d0b4537fad3d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 10:45:25 +0800 Subject: [PATCH 054/163] fix(js): `log` func may be of karax's --- Utils/compat.nim | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Utils/compat.nim b/Utils/compat.nim index 75c0a28..c5eb846 100644 --- a/Utils/compat.nim +++ b/Utils/compat.nim @@ -1,10 +1,11 @@ when defined(js): import strutils + import std/jsconsole #[ include karax/prelude var stream*: seq[(kstring, kstring)] - ]# proc log*(prompt, info: cstring) {. importc .} + ]# # how to read from console? template readLineCompat*(prompt): string = @@ -12,18 +13,16 @@ when defined(js): template echoCompat*(content: string) = echo content - for line in content.split("\n"): - log(cstring" ", line) #stream.add((kstring"", kstring(content))) template errEchoCompat*(content) = - echoCompat content + console.error content - # combining two seq directly leads to a bug in the compiler when compiled to JS - # see gh-10651 + # Years ago... + # combining two seq directly leaded to a bug in the compiler when compiled to JS + # see gh-10651 (have been closed) template addCompat*[T](a, b: seq[T]) = - for item in b: - a.add item + a.add b else: import rdstdin From 73b3c0e2ac57f3576054a2b4a145ab0e1275b9dc Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 10:49:14 +0800 Subject: [PATCH 055/163] fixup(js): findExe not work CT when JS --- Modules/os_findExe_patch.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/os_findExe_patch.nim b/Modules/os_findExe_patch.nim index 6e2c21d..78b5be2 100644 --- a/Modules/os_findExe_patch.nim +++ b/Modules/os_findExe_patch.nim @@ -2,6 +2,10 @@ import std/os import std/strutils +when defined(js) and not compiles(static(fileExists".")): + proc fileExists(fp: string): bool = true # XXX: TODO: I've tried many ways + # to support compile-time check, but Nim just rejects when JS + when true: # copied from std/os, removed `readlink` part, see `XXX` below proc findExe*(exe: string, followSymlinks: bool = true; From 98a85c870efb334266eba12b796117c6d0d8dfd7 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 11:13:04 +0800 Subject: [PATCH 056/163] fix(js): make lefecycle.pyInit init pyConfig, too (like in C) --- Python/lifecycle.nim | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Python/lifecycle.nim b/Python/lifecycle.nim index cb7b24d..8f33c97 100644 --- a/Python/lifecycle.nim +++ b/Python/lifecycle.nim @@ -4,8 +4,7 @@ import bltinmodule import ../Objects/[pyobject, typeobject] import ../Utils/utils -when not defined(js): - import os +import std/os proc outOfMemHandler = @@ -26,16 +25,13 @@ proc pyInit*(args: seq[string]) = for t in bltinTypes: t.typeReady - when defined(js): - discard + if args.len == 0: + pyConfig.path = os.getCurrentDir() else: - if args.len == 0: - pyConfig.path = os.getCurrentDir() - else: - pyConfig.filepath = joinPath(os.getCurrentDir(), args[0]) - pyConfig.filename = pyConfig.filepath.extractFilename() - pyConfig.path = pyConfig.filepath.parentDir() - when defined(debug): - echo "Python path: " & pyConfig.path + pyConfig.filepath = joinPath(os.getCurrentDir(), args[0]) + pyConfig.filename = pyConfig.filepath.extractFilename() + pyConfig.path = pyConfig.filepath.parentDir() + when defined(debug): + echo "Python path: " & pyConfig.path From 3da36ad89d4cbb11a5dd746591867988a1124fe4 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 20:21:27 +0800 Subject: [PATCH 057/163] feat(js): nodejs run .py file --- Python/cpython.nim | 96 ++++++++++++++++++++++++++------------------- Python/jspython.nim | 83 +++++++++++++++++---------------------- 2 files changed, 91 insertions(+), 88 deletions(-) diff --git a/Python/cpython.nim b/Python/cpython.nim index 69fe238..90fbb12 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -1,12 +1,7 @@ -when defined(js): - {.error: "python.nim is for c target. Compile jspython.nim as js target" .} +import std/parseopt import strformat -import os # file existence - - - import neval import compile import coreconfig @@ -32,48 +27,31 @@ proc pymain_header = if pyConfig.quiet: return errEchoCompat getVersionString(verbose=true) -proc interactiveShell = - var finished = true - # the root of the concrete syntax tree. Keep this when user input multiple lines - var rootCst: ParseNode - let lexer = newLexer("") - var prevF: PyFrameObject - pymain_header() - while true: - var input: string - var prompt: string - if finished: - prompt = ">>> " - rootCst = nil - lexer.clearIndent - else: - prompt = "... " - assert (not rootCst.isNil) - - try: - input = readLineCompat(prompt) - except EOFError, IOError: - quit(0) +const Fstdin = "" +proc parseCompileEval*(input: string, lexer: Lexer, + rootCst: var ParseNode, prevF: var PyFrameObject, finished: var bool + ): bool{.discardable.} = + ## stuff to change, just a compatitable layer for ./jspython try: rootCst = parseWithState(input, lexer, Mode.Single, rootCst) except SyntaxError: let e = SyntaxError(getCurrentException()) - let excpObj = fromBltinSyntaxError(e, newPyStr("")) + let excpObj = fromBltinSyntaxError(e, newPyStr(Fstdin)) excpObj.printTb finished = true - continue + return true if rootCst.isNil: - continue + return true finished = rootCst.finished if not finished: - continue + return false - let compileRes = compile(rootCst, "") + let compileRes = compile(rootCst, Fstdin) if compileRes.isThrownException: PyExceptionObject(compileRes).printTb - continue + return true let co = PyCodeObject(compileRes) when defined(debug): @@ -92,19 +70,49 @@ proc interactiveShell = else: prevF = f +proc interactiveShell = + var finished = true + # the root of the concrete syntax tree. Keep this when user input multiple lines + var rootCst: ParseNode + let lexer = newLexer(Fstdin) + var prevF: PyFrameObject + pymain_header() + while true: + var input: string + var prompt: string + if finished: + prompt = ">>> " + rootCst = nil + lexer.clearIndent + else: + prompt = "... " + assert (not rootCst.isNil) + + try: + input = readLineCompat(prompt) + except EOFError, IOError: + quit(0) + + parseCompileEval(input, lexer, rootCst, prevF, finished) + + template exit0or1(suc) = quit(if suc: 0 else: 1) -proc nPython(args: seq[string]) = +proc nPython*(args: seq[string], + fileExists: proc(fp: string): bool, + readFile: proc(fp: string): string, + ) = pyInit(args) if pyConfig.filepath == "": interactiveShell() - if not pyConfig.filepath.fileExists: + if not fileExists(pyConfig.filepath): echo fmt"File does not exist ({pyConfig.filepath})" quit() let input = readFile(pyConfig.filepath) runSimpleString(input, pyConfig.filepath).exit0or1 + proc echoUsage() = echoCompat "usage: python [option] [-c cmd | file]" @@ -119,9 +127,8 @@ proc echoHelp() = echoCompat "file : program read from script file" -when isMainModule: - import std/parseopt - +proc main*(cmdline: string|seq[string] = "", + nPython: proc (args: seq[string])) = proc unknownOption(p: OptParser){.noReturn.} = var origKey = "-" if p.kind == cmdLongOption: origKey.add '-' @@ -136,7 +143,7 @@ when isMainModule: var args: seq[string] versionVerbosity = 0 - var p = initOptParser( + var p = initOptParser(cmdline, shortNoVal={'h', 'V', 'q', 'v'}, # Python can be considered not to allow: -c:CODE -c=code longNoVal = @["help", "version"], @@ -170,3 +177,12 @@ when isMainModule: of 0: nPython args of 1: echoVersion() else: echoVersion(verbose=true) + +when isMainModule: + import std/os # file existence + proc wrap_nPython(args: seq[string]) = + nPython(args, os.fileExists, readFile) + when defined(js): + {.error: "python.nim is for c target. Compile jspython.nim as js target" .} + + main nPython=wrap_nPython diff --git a/Python/jspython.nim b/Python/jspython.nim index 3c78c24..217992c 100644 --- a/Python/jspython.nim +++ b/Python/jspython.nim @@ -1,11 +1,8 @@ -import neval -import compile -import coreconfig -import traceback -import lifecycle + +import ./neval import ../Parser/[lexer, parser] -import ../Objects/bundle -import ../Utils/[utils, compat] +import ./cpython +import ../Objects/frameobject var finished = true var rootCst: ParseNode @@ -14,50 +11,42 @@ var prevF: PyFrameObject proc interactivePython(input: cstring): bool {. exportc .} = echo input - if finished: - rootCst = nil - lexerInst.clearIndent - else: - assert (not rootCst.isNil) - - try: - rootCst = parseWithState($input, lexerInst, Mode.Single, rootCst) - except SyntaxError: - let e = SyntaxError(getCurrentException()) - let excpObj = fromBltinSyntaxError(e, newPyStr("")) - excpObj.printTb - finished = true - return true + return parseCompileEval($input, lexerInst, rootCst, prevF, finished) - if rootCst.isNil: - return true - finished = rootCst.finished - if not finished: - return false +import std/jsffi - let compileRes = compile(rootCst, "") - if compileRes.isThrownException: - PyExceptionObject(compileRes).printTb - return true - let co = PyCodeObject(compileRes) +when isMainModule and defined(nodejs): + let fs = require("fs"); + proc fileExists(fp: string): bool = + fs.existsSync(fp.cstring).to bool + proc readFile(fp: string): string = + let buf = fs.readFileSync(fp.cstring) + let n = buf.length.to int + # without {'encoding': ...} option, Buffer returned + when declared(newStringUninit): + result = newStringUninit(n) + for i in 0.. Date: Tue, 15 Jul 2025 22:39:46 +0800 Subject: [PATCH 058/163] feat(js): nodejs repl --- Python/cpython.nim | 30 ++++---- Python/jspython.nim | 106 ++++++++++++++-------------- Utils/compat.nim | 166 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 221 insertions(+), 81 deletions(-) diff --git a/Python/cpython.nim b/Python/cpython.nim index 90fbb12..c3edbd1 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -70,7 +70,7 @@ proc parseCompileEval*(input: string, lexer: Lexer, else: prevF = f -proc interactiveShell = +proc interactiveShell{.mayAsync.} = var finished = true # the root of the concrete syntax tree. Keep this when user input multiple lines var rootCst: ParseNode @@ -89,7 +89,7 @@ proc interactiveShell = assert (not rootCst.isNil) try: - input = readLineCompat(prompt) + input = mayAwait readLineCompat(prompt) except EOFError, IOError: quit(0) @@ -101,16 +101,16 @@ template exit0or1(suc) = quit(if suc: 0 else: 1) proc nPython*(args: seq[string], fileExists: proc(fp: string): bool, readFile: proc(fp: string): string, - ) = + ){.mayAsync.} = pyInit(args) if pyConfig.filepath == "": - interactiveShell() - - if not fileExists(pyConfig.filepath): - echo fmt"File does not exist ({pyConfig.filepath})" - quit() - let input = readFile(pyConfig.filepath) - runSimpleString(input, pyConfig.filepath).exit0or1 + mayAwait interactiveShell() + else: + if not fileExists(pyConfig.filepath): + echo fmt"File does not exist ({pyConfig.filepath})" + quit() + let input = readFile(pyConfig.filepath) + runSimpleString(input, pyConfig.filepath).exit0or1 proc echoUsage() = @@ -128,7 +128,7 @@ proc echoHelp() = proc main*(cmdline: string|seq[string] = "", - nPython: proc (args: seq[string])) = + nPython: proc (args: seq[string]){.mayAsync.}){.mayAsync.} = proc unknownOption(p: OptParser){.noReturn.} = var origKey = "-" if p.kind == cmdLongOption: origKey.add '-' @@ -174,15 +174,15 @@ proc main*(cmdline: string|seq[string] = "", p.unknownOption() of cmdEnd: break case versionVerbosity - of 0: nPython args + of 0: mayAwait nPython args of 1: echoVersion() else: echoVersion(verbose=true) when isMainModule: import std/os # file existence - proc wrap_nPython(args: seq[string]) = - nPython(args, os.fileExists, readFile) + proc wrap_nPython(args: seq[string]){.mayAsync.} = + mayAwait nPython(args, os.fileExists, readFile) when defined(js): {.error: "python.nim is for c target. Compile jspython.nim as js target" .} - main nPython=wrap_nPython + mayWaitFor main(nPython=wrap_nPython) diff --git a/Python/jspython.nim b/Python/jspython.nim index 217992c..70494b4 100644 --- a/Python/jspython.nim +++ b/Python/jspython.nim @@ -3,24 +3,23 @@ import ./neval import ../Parser/[lexer, parser] import ./cpython import ../Objects/frameobject +import ../Utils/compat -var finished = true -var rootCst: ParseNode -let lexerInst = newLexer("") -var prevF: PyFrameObject +import std/jsffi except require -proc interactivePython(input: cstring): bool {. exportc .} = - echo input - return parseCompileEval($input, lexerInst, rootCst, prevF, finished) - -import std/jsffi - -when isMainModule and defined(nodejs): - let fs = require("fs"); +template isMainDefined(m): bool = isMainModule and defined(m) +when isMainDefined(nodejs): + {.emit: """/*INCLUDESECTION*/ + import {existsSync, readFileSync} from 'node:fs'; + import {argv} from 'node:process'; + """.} + let argv{.importc, nodecl.}: JsObject + proc existsSync(fp: cstring): bool {.importc.} + proc readFileSync(fp: cstring): JsObject {.importc.} proc fileExists(fp: string): bool = - fs.existsSync(fp.cstring).to bool + existsSync(fp.cstring) proc readFile(fp: string): string = - let buf = fs.readFileSync(fp.cstring) + let buf = readFileSync(fp.cstring) let n = buf.length.to int # without {'encoding': ...} option, Buffer returned when declared(newStringUninit): @@ -31,54 +30,57 @@ when isMainModule and defined(nodejs): for i in 0..") + var prevF: PyFrameObject -# karax not working. gh-86 -#[ -include karax/prelude -import karax/kdom + proc interactivePython(input: cstring): bool {. exportc, discardable .} = + echo input + return parseCompileEval($input, lexerInst, rootCst, prevF, finished) -proc createDom(): VNode = - result = buildHtml(tdiv): - tdiv(class="stream"): - echo stream.len - for line in stream: - let (prompt, content) = line - tdiv(class="line"): - p(class="prompt"): - if prompt.len == 0: - text kstring" " - else: - text prompt - p: - text content - tdiv(class="line editline"): - p(class="prompt"): - text prompt - p(class="edit", contenteditable="true"): - proc onKeydown(ev: Event, n: VNode) = - if KeyboardEvent(ev).keyCode == 13: - let input = n.dom.innerHTML - echo input - interactivePython($input) - n.dom.innerHTML = kstring"" - ev.preventDefault + include karax/prelude + import karax/kdom -setRenderer createDom + proc createDom(): VNode = + result = buildHtml(tdiv): + tdiv(class="stream"): + echo stream.len + for line in stream: + let (prompt, content) = line + tdiv(class="line"): + p(class="prompt"): + if prompt.len == 0: + text kstring" " + else: + text prompt + p: + text content + tdiv(class="line editline"): + p(class="prompt"): + text prompt + p(class="edit", contenteditable="true"): + proc onKeydown(ev: Event, n: VNode) = + if KeyboardEvent(ev).keyCode == 13: + let input = n.dom.innerHTML + echo input + interactivePython($input) + n.dom.innerHTML = kstring"" + ev.preventDefault -]# + setRenderer createDom diff --git a/Utils/compat.nim b/Utils/compat.nim index c5eb846..0aee04f 100644 --- a/Utils/compat.nim +++ b/Utils/compat.nim @@ -1,22 +1,150 @@ +# from ./utils import InterruptError +# currently uses EOFError when defined(js): - import strutils import std/jsconsole - #[ - include karax/prelude - var stream*: seq[(kstring, kstring)] - proc log*(prompt, info: cstring) {. importc .} - ]# + template errEchoCompat*(content: string) = + console.error cstring content + const karax* = defined(karax) + when karax: + include karax/prelude + var stream*: seq[(kstring, kstring)] + proc log*(prompt, info: cstring) {. importc .} + + import std/jsffi - # how to read from console? - template readLineCompat*(prompt): string = - "" + when defined(nodejs): - template echoCompat*(content: string) = - echo content - #stream.add((kstring"", kstring(content))) + type + InterfaceConstructor = JsObject + InterfaceConstructorWrapper = object + obj: InterfaceConstructor + Promise[T] = JsObject + template wrapPromise[T](x: T): Promise[T] = cast[Promise[T]](x) ## \ + ## async's result will be wrapped by JS as Promise' + ## this is just to bypass Nim's type system + import std/macros + macro async(def): untyped = + var origType = def.params[0] + let none = origType.kind == nnkEmpty + if none: origType = bindSym"void" - template errEchoCompat*(content) = - console.error content + def.params[0] = nnkBracketExpr.newTree(bindSym"Promise", origType) + if def.kind in RoutineNodes: + def.addPragma nnkExprColonExpr.newTree( + ident"codegenDecl", + newLit"async function $2($3)" + ) + if not none: + def.body = newCall(bindSym"wrapPromise", def.body) + def + + #template await*[T](exp: Promise[T]): T = {.emit: ["await ", exp].} + + ## XXX: top level await, cannot be in functions + template waitFor[T](exp: Promise[T]): T = + let e = exp + var t: T + {.emit: [t, " = await ", e].} + # " <- for code hint + t + template waitFor(exp: Promise[void]) = + let e = exp + {.emit: ["await ", e].} + + template await[T](exp: Promise[T]): T = + waitFor exp + + {.emit: """/*INCLUDESECTION*/ + import {createInterface} from 'node:readline/promises'; + import { stdin as input, stdout as output } from 'node:process'; + """.} # """ <- for code hint + proc initReadLine: InterfaceConstructorWrapper = + {.emit: """ + // top level await must be on ES module + //const { createInterface } = require('node:readline'); + //const { stdin: input, stdout: output } = require('node:process'); + + const rl = createInterface({ input, output }); + rl.on("SIGINT", ()=>{}); + // XXX: TODO: correctly handle ctrl-c (SIGINT) + """.} + # Python does not exit on ctrl-c + # but re-asking a new input + # I'd tried to implement that but failed, + # current impl of handler is just doing nothing (an empty function) + {.emit: [result.obj, "= rl;"].} + when defined(nimPreviewNonVarDestructor): + proc `=destroy`(o: InterfaceConstructorWrapper) = o.obj.close() + else: + proc `=destroy`(o: var InterfaceConstructorWrapper) = o.obj.close() + + proc cursorToNewLine{.noconv.} = + console.log(cstring"") + let rl = initReadLine() + proc question(rl: InterfaceConstructor, ps: cstring): Promise[cstring]{.importcpp.} + proc questionHandledEof(rl: InterfaceConstructor, ps: cstring + ): Promise[cstring] = + ## rl.question(ps) but catch EOF and raise as EOFError + {.emit: [ + result, " = ", + rl, ".question(", ps, """).catch(e=>{ + if (typeof(e) === "object" && e.code === "ABORT_ERR") {""", + r"return '\0';", + """ + } + });""" + # """ <- for code hint + ].} + + + proc readLineCompat*(ps: cstring): cstring{.async.} = + let res = await rl.obj.questionHandledEof ps + if res == cstring("\0"): + cursorToNewLine() + raise new EOFError + res + proc readLineCompat*(prompt: string): string{.async.} = + $(await prompt.cstring.readLineCompat) + + template mayAsync*(def): untyped = + bind async + async(def) + template mayAwait*(x): untyped = + bind await + await x + template mayWaitFor*(x): untyped = + ## top level await + bind waitFor + waitFor x + + else: + import std/jsffi + proc readLineCompat*(prompt: cstring): JsObject#[cstring or null]#{.importc: "prompt".} + + proc readLineCompat*(prompt: string): string = + let res = prompt.cstring.readLineCompat + if res.isNull: + raise new EOFError + $(res.to(cstring)) + when defined(deno): + proc cwd(): cstring{.importc: "Deno.cwd".} + proc getCurrentDir*(): string = $cwd() + proc quitCompat*(e=0){.importc: "Deno.exit".} + proc quitCompat*(e: string) = + bind errEchoCompat, quitCompat + errEchoCompat(e) + quitCompat QuitFailure + when karax: + template echoCompat*(content: string) = + echo content + stream.add((kstring"", kstring(content))) + elif defined(jsAlert): + proc alert(s: cstring){.importc.} + template echoCompat*(content: string) = + alert cstring content + else: + template echoCompat*(content: string) = + echo content # Years ago... # combining two seq directly leaded to a bug in the compiler when compiled to JS @@ -40,4 +168,14 @@ else: template addCompat*[T](a, b: seq[T]) = a.add b +when not declared(async): + template mayAsync*(def): untyped = def + template mayAwait*(x): untyped = x + template mayWaitFor*(x): untyped = x + +when not declared(getCurrentDir): + import std/os + export getCurrentDir +when not declared(quitCompat): + template quitCompat*(e: untyped = 0) = quit(e) From b7b9886e967afba18876d7ae9a30995ef66c45c3 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 15 Jul 2025 23:09:23 +0800 Subject: [PATCH 059/163] chore(nimble): allow pass cmd args for buildJs task --- npython.nimble | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/npython.nimble b/npython.nimble index ea36125..1581356 100644 --- a/npython.nimble +++ b/npython.nimble @@ -48,5 +48,7 @@ taskWithArgs test, "test all, assuming after build": echo "testing " & i exec pyExe & ' ' & i -task buildJs, "build JS": - selfExec "js -o:" & binPathWithoutExt & ".js " & srcDir & '/' & srcName +import std/os +taskWithArgs buildJs, "build JS": + selfExec "js -o:" & binPathWithoutExt & ".js " & + args.quoteShellCommand & ' '& srcDir & '/' & srcName From 682eb2f97995fa8b09d7fe86f1c830d5a2395821 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 16 Jul 2025 15:41:39 +0800 Subject: [PATCH 060/163] feat(js): sup deno; sup -d:jsAlert to gen a prompt&alert repl; sup -d:karax (but bad html style) --- Python/cpython.nim | 21 ++++--- Python/jspython.nim | 142 ++++++++++++++++++++++++++----------------- Python/lifecycle.nim | 16 +++-- Utils/compat.nim | 13 ++-- 4 files changed, 116 insertions(+), 76 deletions(-) diff --git a/Python/cpython.nim b/Python/cpython.nim index c3edbd1..c05c43f 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -70,7 +70,7 @@ proc parseCompileEval*(input: string, lexer: Lexer, else: prevF = f -proc interactiveShell{.mayAsync.} = +proc interactiveShell*{.mayAsync.} = var finished = true # the root of the concrete syntax tree. Keep this when user input multiple lines var rootCst: ParseNode @@ -91,12 +91,12 @@ proc interactiveShell{.mayAsync.} = try: input = mayAwait readLineCompat(prompt) except EOFError, IOError: - quit(0) + quitCompat(0) parseCompileEval(input, lexer, rootCst, prevF, finished) -template exit0or1(suc) = quit(if suc: 0 else: 1) +template exit0or1(suc) = quitCompat(if suc: 0 else: 1) proc nPython*(args: seq[string], fileExists: proc(fp: string): bool, @@ -108,7 +108,7 @@ proc nPython*(args: seq[string], else: if not fileExists(pyConfig.filepath): echo fmt"File does not exist ({pyConfig.filepath})" - quit() + quitCompat() let input = readFile(pyConfig.filepath) runSimpleString(input, pyConfig.filepath).exit0or1 @@ -135,11 +135,16 @@ proc main*(cmdline: string|seq[string] = "", origKey.add p.key errEchoCompat "Unknown option: " & origKey echoUsage() - quit 2 + quitCompat 2 template noLongOption(p: OptParser) = if p.kind == cmdLongOption: p.unknownOption() - + when defined(js) and cmdline is seq[string]: + # fix: initOptParser will call paramCount. + # which is only defined when -d:nodejs + var cmdline = + if cmdline.len == 0: @["-"] + else: cmdline var args: seq[string] versionVerbosity = 0 @@ -157,7 +162,7 @@ proc main*(cmdline: string|seq[string] = "", case p.key: of "help", "h": echoHelp() - quit() + quitCompat() of "version", "V": versionVerbosity.inc of "q": pyConfig.quiet = true @@ -170,6 +175,8 @@ proc main*(cmdline: string|seq[string] = "", else: p.remainingArgs()[0] pyInit(@[]) runSimpleString(code, "").exit0or1 + of "": # allow - + discard else: p.unknownOption() of cmdEnd: break diff --git a/Python/jspython.nim b/Python/jspython.nim index 70494b4..3d19fa4 100644 --- a/Python/jspython.nim +++ b/Python/jspython.nim @@ -1,27 +1,18 @@ import ./neval -import ../Parser/[lexer, parser] import ./cpython -import ../Objects/frameobject import ../Utils/compat import std/jsffi except require -template isMainDefined(m): bool = isMainModule and defined(m) -when isMainDefined(nodejs): - {.emit: """/*INCLUDESECTION*/ - import {existsSync, readFileSync} from 'node:fs'; - import {argv} from 'node:process'; - """.} - let argv{.importc, nodecl.}: JsObject - proc existsSync(fp: cstring): bool {.importc.} - proc readFileSync(fp: cstring): JsObject {.importc.} - proc fileExists(fp: string): bool = - existsSync(fp.cstring) - proc readFile(fp: string): string = - let buf = readFileSync(fp.cstring) +template isMain(b): bool = isMainModule and b +const + nodejs = defined(nodejs) + deno = defined(deno) + dKarax = defined(karax) +when isMain(nodejs or deno): + proc bufferAsString(buf: JsObject): string = let n = buf.length.to int - # without {'encoding': ...} option, Buffer returned when declared(newStringUninit): result = newStringUninit(n) for i in 0..") - var prevF: PyFrameObject +else: + import ./lifecycle + pyInit(@[]) + when isMain(dkarax): + import ../Objects/frameobject + import ../Parser/[lexer, parser] + var finished = true + var rootCst: ParseNode + let lexerInst = newLexer("") + var prevF: PyFrameObject + proc interactivePython(input: string): bool {. exportc, discardable .} = + echo input + if finished: + rootCst = nil + lexerInst.clearIndent + return parseCompileEval(input, lexerInst, rootCst, prevF, finished) - proc interactivePython(input: cstring): bool {. exportc, discardable .} = - echo input - return parseCompileEval($input, lexerInst, rootCst, prevF, finished) + const prompt = ">>> " - include karax/prelude - import karax/kdom + include karax/prelude + import karax/kdom - proc createDom(): VNode = - result = buildHtml(tdiv): - tdiv(class="stream"): - echo stream.len - for line in stream: - let (prompt, content) = line - tdiv(class="line"): - p(class="prompt"): - if prompt.len == 0: - text kstring" " - else: - text prompt - p: - text content - tdiv(class="line editline"): - p(class="prompt"): - text prompt - p(class="edit", contenteditable="true"): - proc onKeydown(ev: Event, n: VNode) = - if KeyboardEvent(ev).keyCode == 13: - let input = n.dom.innerHTML - echo input - interactivePython($input) - n.dom.innerHTML = kstring"" - ev.preventDefault + proc createDom(): VNode = + result = buildHtml(tdiv): + tdiv(class="stream"): + echo stream.len + for line in stream: + let (prompt, content) = line + tdiv(class="line"): + p(class="prompt"): + if prompt.len == 0: + text kstring" " + else: + text prompt + p: + text content + tdiv(class="line editline"): + p(class="prompt"): + text prompt + p(class="edit", contenteditable="true"): + proc onKeydown(ev: Event, n: VNode) = + if KeyboardEvent(ev).keyCode == 13: + let input = n.dom.innerHTML + echo input + interactivePython($input) + n.dom.innerHTML = kstring"" + ev.preventDefault - setRenderer createDom + setRenderer createDom + elif isMainModule: + mayWaitFor interactiveShell() diff --git a/Python/lifecycle.nim b/Python/lifecycle.nim index 8f33c97..4c0a5f3 100644 --- a/Python/lifecycle.nim +++ b/Python/lifecycle.nim @@ -2,17 +2,15 @@ import coreconfig # init bltinmodule import bltinmodule import ../Objects/[pyobject, typeobject] -import ../Utils/utils +import ../Utils/[utils, compat] -import std/os - - -proc outOfMemHandler = - let e = new OutOfMemDefect - raise e +import std/os except getCurrentDir when declared(system.outOfMemHook): # if not JS + proc outOfMemHandler = + let e = new OutOfMemDefect + raise e system.outOfMemHook = outOfMemHandler when not defined(js): @@ -26,9 +24,9 @@ proc pyInit*(args: seq[string]) = t.typeReady if args.len == 0: - pyConfig.path = os.getCurrentDir() + pyConfig.path = getCurrentDir() else: - pyConfig.filepath = joinPath(os.getCurrentDir(), args[0]) + pyConfig.filepath = joinPath(getCurrentDir(), args[0]) pyConfig.filename = pyConfig.filepath.extractFilename() pyConfig.path = pyConfig.filepath.parentDir() when defined(debug): diff --git a/Utils/compat.nim b/Utils/compat.nim index 0aee04f..991377e 100644 --- a/Utils/compat.nim +++ b/Utils/compat.nim @@ -4,8 +4,8 @@ when defined(js): import std/jsconsole template errEchoCompat*(content: string) = console.error cstring content - const karax* = defined(karax) - when karax: + const dKarax = defined(karax) + when dKarax: include karax/prelude var stream*: seq[(kstring, kstring)] proc log*(prompt, info: cstring) {. importc .} @@ -134,7 +134,7 @@ when defined(js): bind errEchoCompat, quitCompat errEchoCompat(e) quitCompat QuitFailure - when karax: + when dKarax: template echoCompat*(content: string) = echo content stream.add((kstring"", kstring(content))) @@ -174,8 +174,11 @@ when not declared(async): template mayWaitFor*(x): untyped = x when not declared(getCurrentDir): - import std/os - export getCurrentDir + when defined(js): + proc getCurrentDir*(): string = "" ## XXX: workaround for pyInit(@[]) + else: + import std/os + export getCurrentDir when not declared(quitCompat): template quitCompat*(e: untyped = 0) = quit(e) From 86a64429c1616bbde3dd32a8c0e7810a37f4cb13 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 16 Jul 2025 15:44:42 +0800 Subject: [PATCH 061/163] chore(nimble): buildKarax --- .gitignore | 4 +++- npython.nimble | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bb14d23..52fba3e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ trash/ ./Objects/test.nim __pycache__/ -/bin/ \ No newline at end of file +/bin/ +/app.html +/app.js \ No newline at end of file diff --git a/npython.nimble b/npython.nimble index 1581356..2a08353 100644 --- a/npython.nimble +++ b/npython.nimble @@ -49,6 +49,11 @@ taskWithArgs test, "test all, assuming after build": exec pyExe & ' ' & i import std/os -taskWithArgs buildJs, "build JS": +taskWithArgs buildJs, "build JS. supported backends: " & + "-d:nodejs|-d:deno|-d:jsAlert": selfExec "js -o:" & binPathWithoutExt & ".js " & args.quoteShellCommand & ' '& srcDir & '/' & srcName + +taskWithArgs buildKarax, "build html page with karax": + exec "karun -d:karax " & + args.quoteShellCommand & ' '& srcDir & '/' & srcName From 9affd0cbee377ccd3df63f0fe53538e6128b1b4d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 16 Jul 2025 18:02:55 +0800 Subject: [PATCH 062/163] refact(karaxpython): split from jspython --- Python/jspython.nim | 52 +++---------------------------------- Python/karaxpython.nim | 53 ++++++++++++++++++++++++++++++++++++++ Python/karaxpython.nim.cfg | 2 ++ 3 files changed, 59 insertions(+), 48 deletions(-) create mode 100644 Python/karaxpython.nim create mode 100644 Python/karaxpython.nim.cfg diff --git a/Python/jspython.nim b/Python/jspython.nim index 3d19fa4..67de23a 100644 --- a/Python/jspython.nim +++ b/Python/jspython.nim @@ -66,53 +66,9 @@ when isMain(nodejs or deno): mayWaitFor main(commandLineParams(), wrap_nPython) else: - import ./lifecycle - pyInit(@[]) - when isMain(dkarax): - import ../Objects/frameobject - import ../Parser/[lexer, parser] - var finished = true - var rootCst: ParseNode - let lexerInst = newLexer("") - var prevF: PyFrameObject - proc interactivePython(input: string): bool {. exportc, discardable .} = - echo input - if finished: - rootCst = nil - lexerInst.clearIndent - return parseCompileEval(input, lexerInst, rootCst, prevF, finished) - - const prompt = ">>> " - - include karax/prelude - import karax/kdom - - proc createDom(): VNode = - result = buildHtml(tdiv): - tdiv(class="stream"): - echo stream.len - for line in stream: - let (prompt, content) = line - tdiv(class="line"): - p(class="prompt"): - if prompt.len == 0: - text kstring" " - else: - text prompt - p: - text content - tdiv(class="line editline"): - p(class="prompt"): - text prompt - p(class="edit", contenteditable="true"): - proc onKeydown(ev: Event, n: VNode) = - if KeyboardEvent(ev).keyCode == 13: - let input = n.dom.innerHTML - echo input - interactivePython($input) - n.dom.innerHTML = kstring"" - ev.preventDefault - - setRenderer createDom + when isMain(dKarax): + import ./karaxpython elif isMainModule: + import ./lifecycle + pyInit(@[]) mayWaitFor interactiveShell() diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim new file mode 100644 index 0000000..aafb361 --- /dev/null +++ b/Python/karaxpython.nim @@ -0,0 +1,53 @@ +{.used.} +import ./cpython + +import ../Utils/compat + +import ./lifecycle +import ../Objects/frameobject +import ../Parser/[lexer, parser] +pyInit(@[]) + +var finished = true +var rootCst: ParseNode +let lexerInst = newLexer("") +var prevF: PyFrameObject +proc interactivePython(input: string): bool {. exportc, discardable .} = + echo input + if finished: + rootCst = nil + lexerInst.clearIndent + return parseCompileEval(input, lexerInst, rootCst, prevF, finished) + +const prompt = ">>> " + +include karax/prelude +import karax/kdom + +proc createDom(): VNode = + result = buildHtml(tdiv): + tdiv(class="stream"): + echo stream.len + for line in stream: + let (prompt, content) = line + tdiv(class="line"): + p(class="prompt"): + if prompt.len == 0: + text kstring" " + else: + text prompt + p: + text content + tdiv(class="line editline"): + p(class="prompt"): + text prompt + p(class="edit", contenteditable="true"): + proc onKeydown(ev: Event, n: VNode) = + if KeyboardEvent(ev).keyCode == 13: + let input = $n.dom.textContent + echo input + interactivePython(input) + n.dom.innerHTML = kstring"" + ev.preventDefault + +setRenderer createDom diff --git a/Python/karaxpython.nim.cfg b/Python/karaxpython.nim.cfg new file mode 100644 index 0000000..a7236d0 --- /dev/null +++ b/Python/karaxpython.nim.cfg @@ -0,0 +1,2 @@ +-d:karax +-d:js \ No newline at end of file From c41c3b535eb7c74f0284b19b9cadb3ad61e1c879 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 16 Jul 2025 18:03:51 +0800 Subject: [PATCH 063/163] fix(js): handle: spaces in browser's contenteditable may become U+00A0 --- Python/karaxpython.nim | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index aafb361..1fa70a1 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -1,4 +1,11 @@ {.used.} + +import std/strutils +proc subsNonbreakingSpace(s: var string) = + ## The leading space, if with following spaces, in on contenteditable element + ## will become `U+00A0` (whose utf-8 encoding is c2a0) + s = s.replace("\xc2\xa0", " ") + import ./cpython import ../Utils/compat @@ -44,8 +51,8 @@ proc createDom(): VNode = p(class="edit", contenteditable="true"): proc onKeydown(ev: Event, n: VNode) = if KeyboardEvent(ev).keyCode == 13: - let input = $n.dom.textContent - echo input + var input = $n.dom.textContent + input.subsNonbreakingSpace interactivePython(input) n.dom.innerHTML = kstring"" ev.preventDefault From af9986acd4c96d781d5dce70e2bb5b46f1c28770 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 16 Jul 2025 21:17:11 +0800 Subject: [PATCH 064/163] feat(js): browser (karax) repl --- Python/cpython.nim | 2 +- Python/karaxpython.nim | 95 ++++++++++++++++++++++++++++++------------ Utils/compat.nim | 3 +- 3 files changed, 70 insertions(+), 30 deletions(-) diff --git a/Python/cpython.nim b/Python/cpython.nim index c05c43f..6fa5f51 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -12,7 +12,7 @@ import ../Parser/[lexer, parser] import ../Objects/bundle import ../Utils/[utils, compat, getplatform] -proc getVersionString(verbose=false): string = +proc getVersionString*(verbose=false): string = result = "NPython " if not verbose: result.add Version diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index 1fa70a1..43c6be3 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -1,11 +1,6 @@ {.used.} import std/strutils -proc subsNonbreakingSpace(s: var string) = - ## The leading space, if with following spaces, in on contenteditable element - ## will become `U+00A0` (whose utf-8 encoding is c2a0) - s = s.replace("\xc2\xa0", " ") - import ./cpython import ../Utils/compat @@ -14,6 +9,11 @@ import ./lifecycle import ../Objects/frameobject import ../Parser/[lexer, parser] pyInit(@[]) +proc subsNonbreakingSpace(s: var string) = + ## The leading space, if with following spaces, in on contenteditable element + ## will become `U+00A0` (whose utf-8 encoding is c2a0) + s = s.replace("\xc2\xa0", " ") + var finished = true var rootCst: ParseNode @@ -26,35 +26,76 @@ proc interactivePython(input: string): bool {. exportc, discardable .} = lexerInst.clearIndent return parseCompileEval(input, lexerInst, rootCst, prevF, finished) -const prompt = ">>> " +let info = getVersionString(verbose=true) include karax/prelude import karax/kdom +import karax/vstyles + +var prompt: kstring + +let + suitHeight = (StyleAttr.height, kstring"wrap-content") # XXX: still too height + preserveSpaces = (whiteSpace, kstring"pre") # ensure spaces remained + monospace = (StyleAttr.font_family, kstring"monospace") +template oneReplLineNode(editable: static[bool]; promptExpr, editExpr): VNode{.dirty.} = + buildHtml: + tdiv(class="line", style=style( + (display, kstring"flex"), # make children within one line + suitHeight, + )): + p(class="prompt" , style=style( + suitHeight, monospace, preserveSpaces, + )): + promptExpr + + p(class="edit", contenteditable=editable, style=style( + (flex, kstring"1"), # without this, it becomes uneditable + (border, kstring"none"), + (outline, kstring"none"), + suitHeight, monospace, preserveSpaces, + )): + editExpr +# TODO: arrow-up / arrow-down for history +const historyContainerId = "history-container" +proc pushHistory(prompt: kstring, exp: string) = + stream.add (prompt, kstring exp) + + # auto scroll down when the inputing line is to go down the view + let historyNode = document.getElementById(historyContainerId) + let last = historyNode.lastChild + if last.isNil: return + last.scrollIntoView(ScrollIntoViewOptions( + `block`: "start", inline: "start", behavior: "instant")) proc createDom(): VNode = result = buildHtml(tdiv): - tdiv(class="stream"): - echo stream.len + tdiv(class="header"): + p(class="info"): + text info + tdiv(class="stream", id=historyContainerId): for line in stream: let (prompt, content) = line - tdiv(class="line"): - p(class="prompt"): - if prompt.len == 0: - text kstring" " - else: - text prompt - p: - text content - tdiv(class="line editline"): - p(class="prompt"): - text prompt - p(class="edit", contenteditable="true"): - proc onKeydown(ev: Event, n: VNode) = - if KeyboardEvent(ev).keyCode == 13: - var input = $n.dom.textContent - input.subsNonbreakingSpace - interactivePython(input) - n.dom.innerHTML = kstring"" - ev.preventDefault + tdiv(class="history"): + oneReplLineNode(false, + text prompt, text content + ) + oneReplLineNode(true, block: + prompt = if finished: + kstring">>> " + else: + kstring"... " + text prompt + , + block: + proc onKeydown(ev: Event, n: VNode) = + if KeyboardEvent(ev).keyCode == 13: + var input = $n.dom.textContent + input.subsNonbreakingSpace + pushHistory(prompt, input) + interactivePython(input) + n.dom.innerHTML = kstring"" + ev.preventDefault + ) setRenderer createDom diff --git a/Utils/compat.nim b/Utils/compat.nim index 991377e..6aca049 100644 --- a/Utils/compat.nim +++ b/Utils/compat.nim @@ -8,8 +8,7 @@ when defined(js): when dKarax: include karax/prelude var stream*: seq[(kstring, kstring)] - proc log*(prompt, info: cstring) {. importc .} - + import std/jsffi when defined(nodejs): From c50a8fc7f0223e1bda092285c0bd2234cafcad6e Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 16 Jul 2025 21:18:01 +0800 Subject: [PATCH 065/163] refine: fixup: use pre over p for line showing, simplfying many things --- Python/karaxpython.nim | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index 43c6be3..77ef03a 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -9,11 +9,6 @@ import ./lifecycle import ../Objects/frameobject import ../Parser/[lexer, parser] pyInit(@[]) -proc subsNonbreakingSpace(s: var string) = - ## The leading space, if with following spaces, in on contenteditable element - ## will become `U+00A0` (whose utf-8 encoding is c2a0) - s = s.replace("\xc2\xa0", " ") - var finished = true var rootCst: ParseNode @@ -36,24 +31,22 @@ var prompt: kstring let suitHeight = (StyleAttr.height, kstring"wrap-content") # XXX: still too height - preserveSpaces = (whiteSpace, kstring"pre") # ensure spaces remained - monospace = (StyleAttr.font_family, kstring"monospace") template oneReplLineNode(editable: static[bool]; promptExpr, editExpr): VNode{.dirty.} = buildHtml: tdiv(class="line", style=style( (display, kstring"flex"), # make children within one line suitHeight, )): - p(class="prompt" , style=style( - suitHeight, monospace, preserveSpaces, + pre(class="prompt" , style=style( + suitHeight, )): promptExpr - p(class="edit", contenteditable=editable, style=style( + pre(class="edit", contenteditable=editable, style=style( (flex, kstring"1"), # without this, it becomes uneditable (border, kstring"none"), (outline, kstring"none"), - suitHeight, monospace, preserveSpaces, + suitHeight, )): editExpr # TODO: arrow-up / arrow-down for history @@ -91,7 +84,6 @@ proc createDom(): VNode = proc onKeydown(ev: Event, n: VNode) = if KeyboardEvent(ev).keyCode == 13: var input = $n.dom.textContent - input.subsNonbreakingSpace pushHistory(prompt, input) interactivePython(input) n.dom.innerHTML = kstring"" From 1cfd3d4468ae658f9bcf6cc7252683361977a8a5 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 16 Jul 2025 22:35:21 +0800 Subject: [PATCH 066/163] chore(nimble): breaks: `chore(nimble): buildKarax`, npython is now appname; chore(nimble): sup jsDir,appName,htmlName arg --- .gitignore | 4 +- Tools/mykarun.nim | 129 ++++++++++++++++++++++++++++++++++++++++++++++ npython.nimble | 2 +- 3 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 Tools/mykarun.nim diff --git a/.gitignore b/.gitignore index 52fba3e..3fcab2f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ trash/ __pycache__/ /bin/ /app.html -/app.js \ No newline at end of file +/app.js +/npython.html +/npython.js \ No newline at end of file diff --git a/Tools/mykarun.nim b/Tools/mykarun.nim new file mode 100644 index 0000000..73a899a --- /dev/null +++ b/Tools/mykarun.nim @@ -0,0 +1,129 @@ + +import std / [os, strutils, browsers, strformat, parseopt] + +const + css = """ + + +""" + html = """ + + + + + + $1 + $2 + + +
$3
+$4 + + +""" + + +proc exec(cmd: string) = + if os.execShellCmd(cmd) != 0: + quit "External command failed: " & cmd + +proc build(ssr: bool, entry: string, rest: string, selectedCss: string, run: bool, + watch: bool, + jsDir, appName: string, htmlName=appName + ) = + echo("Building...") + var cmd: string + var content = "" + var outTempPath: string + var outHtmlName: string + let jsFp = jsDir / appName & ".js" + if ssr: + outHtmlName = changeFileExt(extractFilename(entry),"html") + outTempPath = getTempDir() / outHtmlName + cmd = "nim c -r " & rest & " " & outTempPath + else: + cmd = "nim js --out:" & jsFp & ' ' & rest + if watch: + discard os.execShellCmd(cmd) + else: + exec cmd + let dest = htmlName & ".html" + let script = if ssr:"" else: &"""""" # & (if watch: websocket else: "") + if ssr: + content = readFile(outTempPath) + writeFile(dest, html % [if ssr: outHtmlName else:appName, selectedCss,content, script]) + if run: openDefaultBrowser("http://localhost:8080") + +proc main = + var op = initOptParser() + var rest: string + var + jsDir = "" # root + appName = "app" + htmlName = appName + htmlNameGiven = false + var file = "" + var run = false + var watch = false + var selectedCss = "" + var ssr = false + + template addToRestAux = + rest.add op.key + if op.val != "": + rest.add ':' + rest.add op.val + + while true: + op.next() + case op.kind + of cmdLongOption: + case op.key + of "htmlName": + htmlName = op.val + htmlNameGiven = true + of "appName": + appName = op.val + of "jsDir": + jsDir = op.val + of "run": + run = true + of "css": + if op.val != "": + selectedCss = readFile(op.val) + else: + selectedCss = css + of "ssr": + ssr = true + else: + rest.add " --" + addToRestAux + of cmdShortOption: + if op.key == "r": + run = true + elif op.key == "w": + watch = true + elif op.key == "s": + ssr = true + else: + rest.add " -" + addToRestAux + of cmdArgument: + file = op.key + rest.add ' ' + rest.add file + of cmdEnd: break + + if file.len == 0: quit "filename expected" + # if run: + # spawn serve() + # if watch: + # spawn watchBuild(ssr, file, selectedCss, rest) + if not htmlNameGiven: + htmlName = appName + build(ssr, file, rest, selectedCss, run, watch, + jsDir, appName, htmlName, + ) + # sync() + +main() diff --git a/npython.nimble b/npython.nimble index 2a08353..3c3d506 100644 --- a/npython.nimble +++ b/npython.nimble @@ -55,5 +55,5 @@ taskWithArgs buildJs, "build JS. supported backends: " & args.quoteShellCommand & ' '& srcDir & '/' & srcName taskWithArgs buildKarax, "build html page with karax": - exec "karun -d:karax " & + selfExec "r --hints:off -d:release Tools/mykarun -d:karax " & " --appName=" & namedBin[srcName] & ' ' & args.quoteShellCommand & ' '& srcDir & '/' & srcName From 9bbec5ac9ec0540fe30df5eccd66f71b2c1caa11 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 00:39:39 +0800 Subject: [PATCH 067/163] feat(mykarun): --includeJs for one-file html --- Tools/mykarun.nim | 48 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/Tools/mykarun.nim b/Tools/mykarun.nim index 73a899a..ce9d82e 100644 --- a/Tools/mykarun.nim +++ b/Tools/mykarun.nim @@ -27,9 +27,26 @@ proc exec(cmd: string) = if os.execShellCmd(cmd) != 0: quit "External command failed: " & cmd +const Buf = 1024 +proc extend(outf, inf: File) = + var buf: array[Buf, uint8] + while true: + let n = inf.readBytes(buf, 0, Buf) + if n == 0: break + let wn = outf.writeBytes(buf, 0, n) + assert n == wn +proc extendNJsFileAndCompress(outf, inf: File) = + var line: string + while true: + line = inf.readLine + outf.writeLine line.strip(true, false, {' '}) # we know Nim current only use spaces for indentation + if inf.endOfFile: + break + proc build(ssr: bool, entry: string, rest: string, selectedCss: string, run: bool, watch: bool, - jsDir, appName: string, htmlName=appName + jsDir, appName: string, htmlName=appName, includeJs = false, + release=false, ) = echo("Building...") var cmd: string @@ -48,10 +65,30 @@ proc build(ssr: bool, entry: string, rest: string, selectedCss: string, run: boo else: exec cmd let dest = htmlName & ".html" - let script = if ssr:"" else: &"""""" # & (if watch: websocket else: "") if ssr: content = readFile(outTempPath) - writeFile(dest, html % [if ssr: outHtmlName else:appName, selectedCss,content, script]) + if includeJs: + defer: removeFile jsFp + assert not ssr, "includeJs and ssr are exclusive" + let parts = html.split("$4", 1) + assert parts.len == 2 + var outF = open(dest, fmWrite) + outF.write parts[0] % [appName, selectedCss, ""] + outF.write"""""" + outF.write parts[1] + outF.close() + else: + let script = if ssr:"" else: &"""""" # & (if watch: websocket else: "") + writeFile(dest, html % [if ssr: outHtmlName else:appName, selectedCss,content, script]) if run: openDefaultBrowser("http://localhost:8080") proc main = @@ -62,6 +99,7 @@ proc main = appName = "app" htmlName = appName htmlNameGiven = false + includeJs = false var file = "" var run = false var watch = false @@ -86,6 +124,8 @@ proc main = appName = op.val of "jsDir": jsDir = op.val + of "includeJs": + includeJs = true of "run": run = true of "css": @@ -123,6 +163,8 @@ proc main = htmlName = appName build(ssr, file, rest, selectedCss, run, watch, jsDir, appName, htmlName, + includeJs, + "-d:release" in rest # XXX: not exhaustive ) # sync() From df15dd04a5a2dcb9f20e8c11c3397e3d78ebf034 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 01:05:11 +0800 Subject: [PATCH 068/163] chore(ci): playground --- .github/workflows/playground.yml | 57 ++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/playground.yml diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml new file mode 100644 index 0000000..c172927 --- /dev/null +++ b/.github/workflows/playground.yml @@ -0,0 +1,57 @@ +name: playground-deploy + +# yamllint disable-line rule:truthy +on: + push: + branches: + - master + tags: + - v*.* + workflow_dispatch: + +env: + nim-version: 'stable' + git-url-arg: --git.url:https://github.com/${{ github.repository }} --git.commit:master + deploy-dir: .gh-pages +jobs: + docs: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - name: Cache nimble + id: cache-nimble + uses: actions/cache@v4 + with: + path: ~/.nimble + key: ${{ runner.os }}-nimble + - uses: jiro4989/setup-nim-action@v1 + with: + nim-version: ${{ env.nim-version }} + - name: install karax + run: nimble install karax + - name: buildKarax + run: | + nimble buildKarax -d:homepage="${{ github.event.repository.homepage }}" \ + -d:release --opt:size --includeJs \ + --htmlName=index + - name: mv to deploy dir + run: mv index.html ${{ env.deploy-dir }} + - name: "CNAME" + run: | + cname=$(echo ${{ github.event.repository.homepage }} | grep -oP 'https?://\K[^/]+') + prefix="play." + # py: if not cname.startswith(prefix) + # bash: if [[ "${cname}" != $prefix* ]] + if [ ${cname##$prefix} = $cname ]; then + cname="${prefix}${cname}" + fi + echo -n $cname > ${{ env.deploy-dir }}/CNAME + # We must re-build CNAME as we use 'peaceiris/actions-gh-pages@v4', + # where the old dir (including CNAME) will be purged. + - name: Deploy documents + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ${{ env.deploy-dir }} From be12b790e1a87745b29c73c541b9fa6544ece6a0 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 01:12:26 +0800 Subject: [PATCH 069/163] fixup: need to create deploy-dir --- .github/workflows/playground.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml index c172927..7db0dd6 100644 --- a/.github/workflows/playground.yml +++ b/.github/workflows/playground.yml @@ -35,10 +35,12 @@ jobs: run: | nimble buildKarax -d:homepage="${{ github.event.repository.homepage }}" \ -d:release --opt:size --includeJs \ - --htmlName=index - - name: mv to deploy dir - run: mv index.html ${{ env.deploy-dir }} + --htmlName=index - name: "CNAME" + run: mkdir ${{ env.deploy-dir }} || true + - name: mv to deploy dir + run: mv index.html ${{ env.deploy-dir }}/index.html + - name: create deploy-dir if needed run: | cname=$(echo ${{ github.event.repository.homepage }} | grep -oP 'https?://\K[^/]+') prefix="play." From b971429d56553fc355b1ec88b1ec1159dca677af Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 01:37:17 +0800 Subject: [PATCH 070/163] impr(karaxpython): add simple info line about repo url --- .github/workflows/playground.yml | 5 +++-- Python/karaxpython.nim | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml index 7db0dd6..c71c4de 100644 --- a/.github/workflows/playground.yml +++ b/.github/workflows/playground.yml @@ -11,7 +11,7 @@ on: env: nim-version: 'stable' - git-url-arg: --git.url:https://github.com/${{ github.repository }} --git.commit:master + git-repo-url: https://github.com/${{ github.repository }} deploy-dir: .gh-pages jobs: docs: @@ -33,7 +33,8 @@ jobs: run: nimble install karax - name: buildKarax run: | - nimble buildKarax -d:homepage="${{ github.event.repository.homepage }}" \ + nimble buildKarax \ + -d:homepage="${{ github.event.repository.homepage }}" -d:gitRepoUrl="${{ env.git-repo-url }}" \ -d:release --opt:size --includeJs \ --htmlName=index - name: "CNAME" diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index 77ef03a..d89361d 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -22,7 +22,8 @@ proc interactivePython(input: string): bool {. exportc, discardable .} = return parseCompileEval(input, lexerInst, rootCst, prevF, finished) let info = getVersionString(verbose=true) - +const gitRepoUrl{.strdefine.} = "" +const repoInfoPre = "This website is frontend-only. Open-Source at " include karax/prelude import karax/kdom import karax/vstyles @@ -66,6 +67,10 @@ proc createDom(): VNode = tdiv(class="header"): p(class="info"): text info + when gitRepoUrl.len != 0: + small: italic(class="repo-info"): # TODO: artistic + text repoInfoPre + a(href=gitRepoUrl): text "Github" tdiv(class="stream", id=historyContainerId): for line in stream: let (prompt, content) = line From e22d9796ff6682be1db608156430c123bded89fa Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 01:50:44 +0800 Subject: [PATCH 071/163] doc(readme): update url of github & online playground; detailed "How to use", indicating multiply backends --- README.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1998b02..4d83100 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,14 @@ (Subset of) Python programming language implemented in Nim, from the compiler to the VM. -[Online interactive demo by compiling Nim to Javascript](https://liwt31.github.io/NPython-demo/). +[Online interactive demo by compiling Nim to Javascript][play-npython]. + +[play-npython]: https://play.nimpylib.org/ ### Purpose -Fun and practice. Learn both Python and Nim. +- Fun and practice. Learn both Python and Nim. +- Serve as a altertive small version of CPython + (as of 0.1.1, less than 2MB on release build mode) ### Status @@ -23,13 +27,34 @@ Check out `./tests` to see more examples. ### How to use + +#### prepare + +``` +git clone https://github.com/nimpylib/npython.git +cd npython +``` + +NPython support C backend and multiply JS backends: + +> after build passing `-h` flag to npython and you will +see help message + +#### For a binary executable (C backend) + ``` -git clone https://github.com/liwt31/NPython.git -cd NPython nimble build bin/npython ``` +#### For JS backend + +- NodeJS: `nimble buildJs -d:nodejs` +- Deno: `nimble buildJs -d:deno` +- Browser, prompt&alert-based repl: `nimble buildJs -d:jsAlert` +- single page website: `nimble buildKarax` (requires `nimble install karax`). This is how [online playground][play-npython] runs + + ### Todo * more features on user defined class * builtin compat dict From 7c15c29cd74e5b6706612401239f8cf5293c7686 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 17:23:35 +0800 Subject: [PATCH 072/163] feat(karaxpython/ui): edit line is focused on user entering page --- Python/karaxpython.nim | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index d89361d..409b7b9 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -32,7 +32,9 @@ var prompt: kstring let suitHeight = (StyleAttr.height, kstring"wrap-content") # XXX: still too height -template oneReplLineNode(editable: static[bool]; promptExpr, editExpr): VNode{.dirty.} = + +template oneReplLineNode(editNodeClasses; + editable: static[bool]; promptExpr, editExpr): VNode = buildHtml: tdiv(class="line", style=style( (display, kstring"flex"), # make children within one line @@ -43,25 +45,40 @@ template oneReplLineNode(editable: static[bool]; promptExpr, editExpr): VNode{.d )): promptExpr - pre(class="edit", contenteditable=editable, style=style( + pre(class=editNodeClasses, contenteditable=editable, style=style( (flex, kstring"1"), # without this, it becomes uneditable (border, kstring"none"), (outline, kstring"none"), suitHeight, )): editExpr -# TODO: arrow-up / arrow-down for history + const historyContainerId = "history-container" +var historyNode: Node + +# TODO: arrow-up / arrow-down for history proc pushHistory(prompt: kstring, exp: string) = stream.add (prompt, kstring exp) # auto scroll down when the inputing line is to go down the view - let historyNode = document.getElementById(historyContainerId) let last = historyNode.lastChild if last.isNil: return last.scrollIntoView(ScrollIntoViewOptions( `block`: "start", inline: "start", behavior: "instant")) +const isEditingClass = "isEditing" + +# NOTE: do not use add callback for DOMContentLoaded +# as karax's init is called on windows.load event +# so to set `clientPostRenderCallback` of setRenderer +proc postRenderCallback() = + historyNode = document.getElementById(historyContainerId) + + let nodes = document.getElementsByClassName(isEditingClass) + assert nodes.len == 1, $nodes.len + let edit = nodes[0] + edit.focus() + proc createDom(): VNode = result = buildHtml(tdiv): tdiv(class="header"): @@ -75,10 +92,10 @@ proc createDom(): VNode = for line in stream: let (prompt, content) = line tdiv(class="history"): - oneReplLineNode(false, + oneReplLineNode("expr", false, text prompt, text content ) - oneReplLineNode(true, block: + oneReplLineNode("expr " & isEditingClass, true, block: prompt = if finished: kstring">>> " else: @@ -95,4 +112,5 @@ proc createDom(): VNode = ev.preventDefault ) -setRenderer createDom +setRenderer createDom, clientPostRenderCallback=postRenderCallback + From e38f9da2e5ddb3c4b38494854428db6e13f97bbf Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 18:10:17 +0800 Subject: [PATCH 073/163] feat(karaxpython/ui): history (bug: cursor not to end on restoring) --- Python/karaxpython.nim | 50 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index 409b7b9..2b1fa57 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -24,6 +24,7 @@ proc interactivePython(input: string): bool {. exportc, discardable .} = let info = getVersionString(verbose=true) const gitRepoUrl{.strdefine.} = "" const repoInfoPre = "This website is frontend-only. Open-Source at " + include karax/prelude import karax/kdom import karax/vstyles @@ -56,10 +57,49 @@ template oneReplLineNode(editNodeClasses; const historyContainerId = "history-container" var historyNode: Node +# === history input track === +type HistoryTrackPos = object + offset: Natural ## neg order + +proc reset(self: var HistoryTrackPos) = self.offset = 0 + +proc stepToPastImpl(self: var HistoryTrackPos) = + let hi = stream.high + self.offset = + if self.offset == hi: hi + else: self.offset + 1 +proc stepToNowImpl(self: var HistoryTrackPos) = + self.offset = + if self.offset == 0: 0 + else: self.offset - 1 + +template getHistoryRecord(self: HistoryTrackPos): untyped = + stream[stream.high - self.offset] + +template genStep(pastOrNext){.dirty.} = + proc `stepTo pastOrNext`*(self: var HistoryTrackPos, n: var Node) = + self.`stepTo pastOrNext Impl` + var tup: tuple[prompt, info: kstring] + + tup = self.getHistoryRecord + if tup.prompt == "": # is output over input + # skip this one + self.`stepTo pastOrNext Impl` + tup = self.getHistoryRecord + + n.innerHtml = tup.info + +genStep Past +genStep Now + +var historyInputPos: HistoryTrackPos + # TODO: arrow-up / arrow-down for history proc pushHistory(prompt: kstring, exp: string) = stream.add (prompt, kstring exp) + historyInputPos.reset + # auto scroll down when the inputing line is to go down the view let last = historyNode.lastChild if last.isNil: return @@ -104,12 +144,18 @@ proc createDom(): VNode = , block: proc onKeydown(ev: Event, n: VNode) = - if KeyboardEvent(ev).keyCode == 13: + case KeyboardEvent(ev).key # .keyCode is deprecated + of "Enter": var input = $n.dom.textContent pushHistory(prompt, input) interactivePython(input) n.dom.innerHTML = kstring"" - ev.preventDefault + of "ArrowUp": + historyInputPos.stepToPast n.dom + of "ArrowDown": + historyInputPos.stepToNow n.dom + else: return + ev.preventDefault ) setRenderer createDom, clientPostRenderCallback=postRenderCallback From 8a959cdca4f31c54dbba800bce4ad55f6edae2cf Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 19:14:28 +0800 Subject: [PATCH 074/163] fixup: bug: cursor not to end on restoring --- Python/karaxpython.nim | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index 2b1fa57..7ab5471 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -61,7 +61,7 @@ var historyNode: Node type HistoryTrackPos = object offset: Natural ## neg order -proc reset(self: var HistoryTrackPos) = self.offset = 0 +proc reset*(self: var HistoryTrackPos) = self.offset = 0 proc stepToPastImpl(self: var HistoryTrackPos) = let hi = stream.high @@ -76,8 +76,37 @@ proc stepToNowImpl(self: var HistoryTrackPos) = template getHistoryRecord(self: HistoryTrackPos): untyped = stream[stream.high - self.offset] +{.push noconv.} +proc createRange(doc: Document): Range{.importcpp.} +proc setStart(rng: Range, node: Node, pos: int) {.importcpp.} +proc collapse(rng: Range, b: bool) {.importcpp.} +proc addRange(s: Selection, rng: Range){.importcpp.} +{.pop.} + +proc setCursorPos(element: Node, position: int) = + ## .. note:: position is starting from 1, not 0 + # from JS code: + # Create a new range + let range = document.createRange() + + # Get the text node + let textNode = element.firstChild + + # Set the position + range.setStart(textNode, position) + range.collapse(true) + + # Apply the selection + let selection = document.getSelection() + selection.removeAllRanges() + selection.addRange(range) + + # Focus the element + element.focus(); + + template genStep(pastOrNext){.dirty.} = - proc `stepTo pastOrNext`*(self: var HistoryTrackPos, n: var Node) = + proc `stepTo pastOrNext`*(self: var HistoryTrackPos, input: var Node) = self.`stepTo pastOrNext Impl` var tup: tuple[prompt, info: kstring] @@ -87,14 +116,18 @@ template genStep(pastOrNext){.dirty.} = self.`stepTo pastOrNext Impl` tup = self.getHistoryRecord - n.innerHtml = tup.info + let hisInp = tup.info + input.innerHTML = hisInp + # set cursor to end (otherwise it'll just be at the begining) + let le = hisInp.len # XXX: suitable for Unicode? + input.setCursorPos(le) genStep Past genStep Now var historyInputPos: HistoryTrackPos -# TODO: arrow-up / arrow-down for history + proc pushHistory(prompt: kstring, exp: string) = stream.add (prompt, kstring exp) From 8fdea036e6f7aaf10d6919c5c69ca1aa7253d192 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 17 Jul 2025 19:14:52 +0800 Subject: [PATCH 075/163] fixup: the last incomplete input cannot be restored --- Python/karaxpython.nim | 46 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index 7ab5471..ff43183 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -58,23 +58,44 @@ const historyContainerId = "history-container" var historyNode: Node # === history input track === -type HistoryTrackPos = object - offset: Natural ## neg order +import std/options +type + Historyinfo = tuple[prompt, info: kstring] + HistoryTrackPos = object + offset: Natural ## neg order + incomplete: Option[HistoryInfo] + useInComplete: bool + +proc pushInCompleteHistory*(self: var HistoryTrackPos, ps, inp: kstring) = + self.incomplete = some (ps, inp) + +proc popInCompleteHistory*(self: var HistoryTrackPos) = + if self.incomplete.isSome: + self.incomplete = none HistoryInfo proc reset*(self: var HistoryTrackPos) = self.offset = 0 proc stepToPastImpl(self: var HistoryTrackPos) = let hi = stream.high + if self.useInComplete: + self.useInComplete = false + return self.offset = if self.offset == hi: hi else: self.offset + 1 proc stepToNowImpl(self: var HistoryTrackPos) = self.offset = - if self.offset == 0: 0 + if self.offset == 0: + if self.incomplete.isSome: + self.useInComplete = true + 0 else: self.offset - 1 template getHistoryRecord(self: HistoryTrackPos): untyped = - stream[stream.high - self.offset] + if self.useInComplete: + assert self.incomplete.isSome + self.incomplete.unsafeGet() + else: stream[stream.high - self.offset] {.push noconv.} proc createRange(doc: Document): Range{.importcpp.} @@ -83,6 +104,7 @@ proc collapse(rng: Range, b: bool) {.importcpp.} proc addRange(s: Selection, rng: Range){.importcpp.} {.pop.} +import std/jsconsole proc setCursorPos(element: Node, position: int) = ## .. note:: position is starting from 1, not 0 # from JS code: @@ -93,6 +115,9 @@ proc setCursorPos(element: Node, position: int) = let textNode = element.firstChild # Set the position + if textNode.isNil: + # happend if last incomplete history input is empty + return range.setStart(textNode, position) range.collapse(true) @@ -134,9 +159,9 @@ proc pushHistory(prompt: kstring, exp: string) = historyInputPos.reset # auto scroll down when the inputing line is to go down the view - let last = historyNode.lastChild - if last.isNil: return - last.scrollIntoView(ScrollIntoViewOptions( + let incomplete = historyNode.lastChild + if incomplete.isNil: return + incomplete.scrollIntoView(ScrollIntoViewOptions( `block`: "start", inline: "start", behavior: "instant")) const isEditingClass = "isEditing" @@ -177,13 +202,18 @@ proc createDom(): VNode = , block: proc onKeydown(ev: Event, n: VNode) = + template getCurInput: kstring = n.dom.textContent case KeyboardEvent(ev).key # .keyCode is deprecated of "Enter": - var input = $n.dom.textContent + historyInputPos.popInCompleteHistory() + let kInput = getCurInput() + let input = $kInput pushHistory(prompt, input) interactivePython(input) n.dom.innerHTML = kstring"" of "ArrowUp": + let kInput = getCurInput() + historyInputPos.pushIncompleteHistory(prompt, kInput) historyInputPos.stepToPast n.dom of "ArrowDown": historyInputPos.stepToNow n.dom From c34fa878c586b4fae62997ca3cc36d52bc37d99c Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 04:05:52 +0800 Subject: [PATCH 076/163] feat(magic): iXX (e.g. `__iadd__`) --- Objects/pyobjectBase.nim | 14 ++++++++ Python/asdl.nim | 2 +- Python/ast.nim | 73 ++++++++++++++++++++++++++++++---------- Python/compile.nim | 22 +++++++++--- Python/neval.nim | 31 +++++++++++++++-- Python/symtable.nim | 5 +++ 6 files changed, 121 insertions(+), 26 deletions(-) diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index f75edc8..5fa0dc8 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -57,6 +57,20 @@ type Mod: BinaryMethod pow: BinaryMethod + iadd, + isub, + imul, + itrueDiv, + ifloorDiv, + # use uppercase to avoid conflict with nim keywords + # backquoting is a less clear solution + iMod, + ipow, + # note: these 3 are all bitwise operations, nothing to do with keywords `and` or `or` + iAnd, + iXor, + iOr: BinaryMethod + Not: UnaryMethod negative: UnaryMethod positive: UnaryMethod diff --git a/Python/asdl.nim b/Python/asdl.nim index 48567d1..9d84825 100644 --- a/Python/asdl.nim +++ b/Python/asdl.nim @@ -360,7 +360,7 @@ genAsdlTypes: Delete(expr* targets), Assign(expr* targets, expr value), - AugAssign(expr target, operator op, expr value), + AugAssign(expr target, operator op, expr value), # augmented asgn (in-place op) # 'simple' indicates that we annotate simple name without parens AnnAssign(expr target, expr annotation, expr? value, int simple), diff --git a/Python/ast.nim b/Python/ast.nim index 40f25ce..613424a 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -463,7 +463,13 @@ ast expr_stmt, [AsdlStmt]: node.value = testlistStarExpr2 result = node of Token.augassign: # `x += 1` like - raiseSyntaxError("Inplace operation not implemented", middleChild) + let testlistStarExpr2 = astTestlist(parseNode.children[2]) + let node = newAstAugAssign() + setNo(node, middleChild.children[0]) + node.target = testlistStarExpr1 + node.op = astAugAssign(middleChild) + node.value = testlistStarExpr2 + result = node else: raiseSyntaxError("Only support simple assignment like a=1", middleChild) assert result != nil @@ -486,13 +492,50 @@ ast testlist_star_expr, [AsdlExpr]: result = newTuple(elms) copyNo(result, elms[0]) assert result != nil - + +#[ + var op: AsdlAugAssign + case token + of Token.Plusequal: + op = newAstPlusequal() + of Token.Minequal: + op = newAstMinequal() + else: + unreachable + var nodeSeq = @[firstAstNode] + for idx in 1..parseNode.children.len div 2: + let nextChild = parseNode.children[2 * idx] + let nextAstNode = childAstFunc(nextChild) + nodeSeq.add(nextAstNode) + result = newAugAssign(op, nodeSeq) + copyNo(result, firstAstNode) +]# # augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | # '<<=' | '>>=' | '**=' | '//=') ast augassign, [AsdlOperator]: - raiseSyntaxError("Inplace operator not implemented") - + assert parseNode.children.len == 1 + let augassignNode = parseNode.children[0] + let token = augassignNode.tokenNode.token + result = AsdlOperator(case token + of Token.Plusequal: newAstAdd() + of Token.Minequal:newAstSub() + of Token.Starequal: newAstMult() + of Token.Slashequal:newAstDiv() + of Token.Percentequal: newAstMod() + of Token.DoubleSlashequal: newAstFloorDiv() + else: + let msg = fmt"Complex augumented assign operation not implemented: " & $token + raiseSyntaxError(msg) + ) + + +#[ +Amperequal +Vbarequal + Circumflexequal + ]# + proc astDelStmt(parseNode: ParseNode): AsdlStmt = raiseSyntaxError("del not implemented") @@ -840,24 +883,18 @@ template astForBinOp(childAstFunc: untyped) = result = firstAstNode for idx in 1..parseNode.children.len div 2: let opParseNode = parseNode.children[2 * idx - 1] - var op: AsdlOperator let token = opParseNode.tokenNode.token - case token - of Token.Plus: - op = newAstAdd() - of Token.Minus: - op = newAstSub() - of Token.Star: - op = newAstMult() - of Token.Slash: - op = newAstDiv() - of Token.Percent: - op = newAstMod() - of Token.DoubleSlash: - op = newAstFloorDiv() + let op = AsdlOperator(case token + of Token.Plus: newAstAdd() + of Token.Minus:newAstSub() + of Token.Star: newAstMult() + of Token.Slash:newAstDiv() + of Token.Percent: newAstMod() + of Token.DoubleSlash: newAstFloorDiv() else: let msg = fmt"Complex binary operation not implemented: " & $token raiseSyntaxError(msg) + ) let secondChild = parseNode.children[2 * idx] let secondAstNode = childAstFunc(secondChild) diff --git a/Python/compile.nim b/Python/compile.nim index 29308b5..5cb0ac8 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -360,6 +360,20 @@ genMapMethod toOpCode: FloorDiv: BinaryFloorDivide } +method toInplaceOpCode(op: AsdlOperator): OpCode {.base.} = + echo "inplace ",op + assert false +genMapMethod toInplaceOpCode: + { + Add: InplaceAdd, + Sub: InplaceSubtract, + Mult: InplaceMultiply, + Div: InplaceTrueDivide, + Mod: InplaceModulo, + Pow: InplacePower, + FloorDiv: InplaceFloorDivide + } + method toOpCode(op: AsdlUnaryop): OpCode {.base.} = unreachable @@ -468,11 +482,11 @@ compileMethod Assign: c.compile(astNode.value) c.compile(astNode.targets[0]) - compileMethod AugAssign: - # don't do augassign as it's complicated and not necessary - unreachable # should be blocked by ast - + c.compile(astNode.target) + c.compile(astNode.value) + let opCode = astNode.op.toInplaceOpCode + c.addOp(newInstr(opCode, astNode.lineNo.value)) compileMethod For: assert astNode.orelse.len == 0 diff --git a/Python/neval.nim b/Python/neval.nim index 6ff3e1c..116c992 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -46,6 +46,18 @@ template doUnary(opName: untyped) = let res = top.callMagic(opName, handleExcp=true) sSetTop res +macro callInplaceMagic(op1, instr, op2): untyped = + let iMagic = ident 'i' & instr.strVal + quote do: + `op1`.callMagic(`iMagic`, `op2`, handleExcp=true) + +template doInplace(opName: untyped) = + bind callInplaceMagic + let op2 = sPop() + let op1 = sTop() + let res = op1.callInplaceMagic(opName, op2) + sSetTop res + template doBinary(opName: untyped) = let op2 = sPop() let op1 = sTop() @@ -217,21 +229,34 @@ proc evalFrame*(f: PyFrameObject): PyObject = let value = sPop() discard obj.callMagic(setitem, idx, value, handleExcp=true) + of OpCode.BinarySubscr: + doBinary(getitem) + + of OpCode.BinaryAdd: doBinary(add) of OpCode.BinarySubtract: doBinary(sub) - of OpCode.BinarySubscr: - doBinary(getitem) - of OpCode.BinaryFloorDivide: doBinary(floorDiv) of OpCode.BinaryTrueDivide: doBinary(trueDiv) + of OpCode.InplaceAdd: + doInplace(add) + + of OpCode.InplaceSubtract: + doInplace(sub) + + of OpCode.InplaceFloorDivide: + doInplace(floorDiv) + + of OpCode.InplaceTrueDivide: + doInplace(trueDiv) + of OpCode.GetIter: let top = sTop() let (iterObj, _) = getIterableWithCheck(top) diff --git a/Python/symtable.nim b/Python/symtable.nim index e137696..9a74375 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -218,6 +218,11 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = visit assignNode.targets[0] visit assignNode.value + of AsdlStmtTk.AugAssign: + let binOpNode = AstAugAssign(astNode) + visit binOpNode.target + visit binOpNode.value + of AsdlStmtTk.For: let forNode = AstFor(astNode) if not (forNode.target.kind == AsdlExprTk.Name): From b431380b96f391adeb72969ab29c0499ed48bfb1 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 04:14:12 +0800 Subject: [PATCH 077/163] feat(builtins): NotImplemented --- Objects/notimplementedobject.nim | 18 ++++++++++++++++++ Objects/pyobjectBase.nim | 1 + Python/bltinmodule.nim | 3 ++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Objects/notimplementedobject.nim diff --git a/Objects/notimplementedobject.nim b/Objects/notimplementedobject.nim new file mode 100644 index 0000000..6b33653 --- /dev/null +++ b/Objects/notimplementedobject.nim @@ -0,0 +1,18 @@ +import ./pyobject +import ./stringobject + +declarePyType NotImplemented(tpToken): + discard + +let pyNotImplemented* = newPyNotImplementedSimple() ## singleton + +proc isNotImplemented*(obj: PyObject): bool = + obj.id == pyNotImplemented.id + +proc dollar(self: PyNotImplementedObject): string = "NotImplemented" +method `$`*(self: PyNotImplementedObject): string = + self.dollar + +implNotImplementedMagic repr: + newPyString self.dollar + diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index 5fa0dc8..74826d5 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -10,6 +10,7 @@ type NULL, Object, None, + NotImplemented, BaseError, # exception Int, Float, diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 2fcad88..312a62b 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -2,7 +2,7 @@ import strformat {.used.} # this module contains toplevel code, so never `importButNotUsed` import neval import builtindict -import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject] +import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject, notimplementedobject] import ../Utils/[utils, macroutils, compat] @@ -106,6 +106,7 @@ implBltinFunc buildClass(funcObj: PyFunctionObject, name: PyStrObject), "__build tpMagic(Type, new)(@[pyTypeObjectType, name, newPyTuple(@[]), f.toPyDict()]) +registerBltinObject("NotImplemented", pyNotImplemented) registerBltinObject("None", pyNone) registerBltinObject("type", pyTypeObjectType) registerBltinObject("range", pyRangeObjectType) From eed6d9cdc08957702f97e0fd6c3a030244aca4d9 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 04:16:18 +0800 Subject: [PATCH 078/163] feat(augassign): impl `a.__iXX__(b)` fallback to `a=a.__XX__(b)` --- Objects/pyobject.nim | 53 ++++++++++++++++++++++++++++++++------------ Python/neval.nim | 41 +++++++++++++++++++++++++++++----- 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 23b3eae..77013e6 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -21,11 +21,13 @@ export pyobjectBase template getMagic*(obj: PyObject, methodName): untyped = obj.pyType.magicMethods.methodName -template getFun*(obj: PyObject, methodName: untyped, handleExcp=false): untyped = - if obj.pyType.isNil: - unreachable("Py type not set") - let fun = getMagic(obj, methodName) - if fun.isNil: + +template checkTypeNotNil(obj) = + when not defined(release): + if obj.pyType.isNil: + unreachable("Py type not set") + +template handleNilFunOfGetFun(obj, methodName, handleExcp) = let objTypeStr = $obj.pyType.name let methodStr = astToStr(methodName) let msg = "No " & methodStr & " method for " & objTypeStr & " defined" @@ -34,16 +36,20 @@ template getFun*(obj: PyObject, methodName: untyped, handleExcp=false): untyped handleException(excp) else: return excp + +template getFun*(obj: PyObject, methodName: untyped, handleExcp=false): untyped = + bind checkTypeNotNil, handleNilFunOfGetFun + obj.checkTypeNotNil + let fun = getMagic(obj, methodName) + if fun.isNil: + handleNilFunOfGetFun(obj, methodName, handleExcp) fun # XXX: `obj` is used twice so it better be a simple identity # if it's a function then the function is called twice! -# is there any ways to reduce the repetition? simple template won't work -template callMagic*(obj: PyObject, methodName: untyped, handleExcp=false): PyObject = - let fun = obj.getFun(methodName, handleExcp) - let res = fun(obj) +template checkExcAndRet[T](res: T, handleExcp): T = when handleExcp: if res.isThrownException: handleException(res) @@ -51,16 +57,35 @@ template callMagic*(obj: PyObject, methodName: untyped, handleExcp=false): PyObj else: res +# is there any ways to reduce the repetition? simple template won't work +template callMagic*(obj: PyObject, methodName: untyped, handleExcp=false): PyObject = + let fun = obj.getFun(methodName, handleExcp) + let res = fun(obj) + bind checkExcAndRet + res.checkExcAndRet handleExcp + template callMagic*(obj: PyObject, methodName: untyped, arg1: PyObject, handleExcp=false): PyObject = let fun = obj.getFun(methodName, handleExcp) let res = fun(obj, arg1) - when handleExcp: - if res.isThrownException: - handleException(res) - res + bind checkExcAndRet + res.checkExcAndRet handleExcp + + +template callInplaceMagic*(obj: PyObject, methodName1: untyped, + arg1: PyObject, handleExcp=false): PyObject = + bind checkTypeNotNil, handleNilFunOfGetFun + bind checkExcAndRet + obj.checkTypeNotNil + var fun = obj.getMagic(methodName1) + if fun.isNil: + pyNotImplemented else: - res + if fun.isNil: + handleNilFunOfGetFun(obj, methodName1, handleExcp) + else: + let res = fun(obj, arg1) + res.checkExcAndRet handleExcp template callMagic*(obj: PyObject, methodName: untyped, arg1, arg2: PyObject, handleExcp=false): PyObject = diff --git a/Python/neval.nim b/Python/neval.nim index 116c992..943bd2f 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -10,7 +10,7 @@ import builtindict import traceback import ../Objects/[pyobject, baseBundle, tupleobject, listobject, dictobject, sliceobject, codeobject, frameobject, funcobject, cellobject, - setobject, + setobject, notimplementedobject, exceptionsImpl, moduleobject, methodobject] import ../Utils/utils @@ -46,17 +46,46 @@ template doUnary(opName: untyped) = let res = top.callMagic(opName, handleExcp=true) sSetTop res -macro callInplaceMagic(op1, instr, op2): untyped = - let iMagic = ident 'i' & instr.strVal +macro tryCallInplaceMagic(op1, opName, op2): untyped = + let iMagic = ident 'i' & opName.strVal quote do: - `op1`.callMagic(`iMagic`, `op2`, handleExcp=true) + `op1`.callInplaceMagic(`iMagic`, `op2`, handleExcp=false) template doInplace(opName: untyped) = bind callInplaceMagic let op2 = sPop() let op1 = sTop() - let res = op1.callInplaceMagic(opName, op2) - sSetTop res + let res = op1.tryCallInplaceMagic(opName, op2) + if res.isNotImplemented: + var nres = op1.callMagic(opName, op2, handleExcp=true) + if nres.isNotImplemented: + let + opStr{.inject.} = "i" & astToStr(opName) + # PY-DIFF: not iadd, but += + typ1{.inject.} = op1.pyType.name + typ2{.inject.} = op2.pyType.name + nres = newTypeError( + &"unsupported operand type(s) for '{opStr}': '{typ1}' and '{typ2}'" + ) + sSetTop nres + else: + sSetTop nres + let stIdx = lastI - 2 + let (opCode, opArg) = f.code.code[stIdx] + var st: OpCode + case opCode + of OpCode.LoadFast: st=StoreFast + of OpCode.LoadGlobal: st=StoreGlobal + of OpCode.LoadAttr: st=StoreAttr + of OpCode.LoadName: st=StoreName + of OpCode.LoadDeref: st=StoreDeref + of {OpCode.LoadClosure, LoadMethod, LoadClassDeref}: + raiseAssert( + &"augumented assignment for {opCode} is not implemented yet") + else: unreachable() + f.code.code.insert (st, opArg), lastI+1 + else: + sSetTop res template doBinary(opName: untyped) = let op2 = sPop() From fe923efee926970ac9c4fd945483baf80aa75fd6 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 04:48:13 +0800 Subject: [PATCH 079/163] fix(lexer): `""` and `''` crash lexer (TODO: multiline string) --- Parser/lexer.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 2ced3fe..9b4ff36 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -58,7 +58,6 @@ proc newTokenNode*(token: Token, else: case token of contentTokenSet: - assert content != "" result = TokenNode(token: token, content: content) else: assert content == "" From 35a0f3a3d526d6bba9dfc24bb617318b70bccccf Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 05:41:27 +0800 Subject: [PATCH 080/163] feat(implMagic): for binary magic allow typed params --- Objects/pyobject.nim | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 77013e6..2a9c5ea 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -215,6 +215,8 @@ template checkTypeTmpl(obj, tp, tpObj, methodName) = let msg = fmt"{expected} is requred for {mName} (got {got})" return newTypeError(msg) +proc isSeqObject(n: NimNode): bool = + n.kind == nnkBracketExpr and n[0].eqIdent"seq" and n[1].eqIdent"PyObject" macro checkArgTypes*(nameAndArg, code: untyped): untyped = let methodName = nameAndArg[0] @@ -226,30 +228,32 @@ macro checkArgTypes*(nameAndArg, code: untyped): untyped = assert varargs[0].strVal == "*" varargName = varargs[1].strVal let argNum = argTypes.len - if varargName == "": - # return `checkArgNum(1, "append")` like - body.add newCall(ident("checkArgNum"), - newIntLitNode(argNum), - newStrLitNode(methodName.strVal) - ) - else: - body.add newCall(ident("checkArgNumAtLeast"), - newIntLitNode(argNum - 1), - newStrLitNode(methodName.strVal) - ) - let remainingArgNode = ident(varargname) - body.add(quote do: - let `remainingArgNode` = args[`argNum`-1..^1] - ) - + let oriParams = code.params + let multiArg = argNum > 1 or oriParams[^1][1].isSeqObject + if multiArg: + if varargName == "": + # return `checkArgNum(1, "append")` like + body.add newCall(ident("checkArgNum"), + newIntLitNode(argNum), + newStrLitNode(methodName.strVal) + ) + else: + body.add newCall(ident("checkArgNumAtLeast"), + newIntLitNode(argNum - 1), + newStrLitNode(methodName.strVal) + ) + let remainingArgNode = ident(varargname) + body.add(quote do: + let `remainingArgNode` = args[`argNum`-1..^1] + ) for idx, child in argTypes: if child.kind == nnkPrefix: continue - let obj = nnkBracketExpr.newTree( + let obj = if multiArg: nnkBracketExpr.newTree( ident("args"), newIntLitNode(idx), - ) + ) else: oriParams[idx+1][0] let name = child[0] let tp = child[1] if tp.strVal == "PyObject": # won't bother checking From 75cc8ce151f20ce0b4bdebfcb0fc317452c748ab Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 05:41:40 +0800 Subject: [PATCH 081/163] feat(str): `__add__` --- Objects/stringobjectImpl.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index b9063be..a00e6fe 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -50,3 +50,6 @@ implStrMagic New(tp: PyObject, obj: PyObject): return newTypeError( &"__str__ returned non-string (type {result.pyType.name:.200s})") + +implStrMagic add(i: PyStrObject): + newPyStr self.str & i.str From d02caf53d0be61c2f7e25ebd153704b59492ace8 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 08:15:57 +0800 Subject: [PATCH 082/163] feat(karaxpython/ui): wrap line if overflow --- Python/karaxpython.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index ff43183..730c5ad 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -50,6 +50,10 @@ template oneReplLineNode(editNodeClasses; (flex, kstring"1"), # without this, it becomes uneditable (border, kstring"none"), (outline, kstring"none"), + (wordBreak, kstring"break-all"), # break anywhere, excluding CJK + #(lineBreak, kstring"anywhere"), # break anywhere, for CJK, not sup by karax + #(wordWrap, kstring"anywhere"), # alias of overflow-wrap, not sup by karax + (whiteSpace, kstring"pre-wrap"), # Preserve spaces and allow wrapping suitHeight, )): editExpr From bcb7d9e3e3cc486ba339be143533d3a4a9bafcb0 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 08:41:59 +0800 Subject: [PATCH 083/163] impr(ast): parse float/int and string were too slow! --- Python/ast.nim | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index 613424a..e13ca47 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -1028,21 +1028,20 @@ ast atom, [AsdlExpr]: of Token.NUMBER: # float - for c in child1.tokenNode.content: - if not (c in '0'..'9'): - let f = parseFloat(child1.tokenNode.content) - let pyFloat = newPyFloat(f) - result = newAstConstant(pyFloat) + if not child1.tokenNode.content.allCharsInSet({'0'..'9'}): + let f = parseFloat(child1.tokenNode.content) + let pyFloat = newPyFloat(f) + result = newAstConstant(pyFloat) # int - if result.isNil: + else: let pyInt = newPyInt(child1.tokenNode.content) result = newAstConstant(pyInt) of Token.STRING: - var strSeq: seq[string] + var str: string for child in parseNode.children: - strSeq.add(child.tokenNode.content) - let pyString = newPyString(strSeq.join()) + str.add(child.tokenNode.content) + let pyString = newPyString(str) result = newAstConstant(pyString) of Token.True: From 511e0f232457e903b19d4a11cc518986688133e8 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 09:03:46 +0800 Subject: [PATCH 084/163] fixup(NotImplemented): `type(NotImplemented)() is not NotImplemented` --- Objects/notimplementedobject.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Objects/notimplementedobject.nim b/Objects/notimplementedobject.nim index 6b33653..2ad063e 100644 --- a/Objects/notimplementedobject.nim +++ b/Objects/notimplementedobject.nim @@ -1,7 +1,8 @@ import ./pyobject import ./stringobject +import ./exceptions -declarePyType NotImplemented(tpToken): +declarePyType NotImplemented(): discard let pyNotImplemented* = newPyNotImplementedSimple() ## singleton @@ -16,3 +17,5 @@ method `$`*(self: PyNotImplementedObject): string = implNotImplementedMagic repr: newPyString self.dollar +implNotImplementedMagic New(tp: PyObject): + return pyNotImplemented From b3585395e92d0c7626e7f341706097abfada0640 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Fri, 18 Jul 2025 09:12:18 +0800 Subject: [PATCH 085/163] feat(builtin): Ellipsis (ast syntax `...` is sup too) --- Objects/pyobjectBase.nim | 2 +- Objects/sliceobject.nim | 15 +++++++++++++++ Python/ast.nim | 9 +++++++-- Python/bltinmodule.nim | 1 + 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index 74826d5..d95a1b6 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -10,7 +10,7 @@ type NULL, Object, None, - NotImplemented, + Ellipsis, BaseError, # exception Int, Float, diff --git a/Objects/sliceobject.nim b/Objects/sliceobject.nim index 35c8f57..9167968 100644 --- a/Objects/sliceobject.nim +++ b/Objects/sliceobject.nim @@ -70,3 +70,18 @@ proc getSliceItems*(slice: PySliceObject, src, dest: ptr seq[PyObject]): PyObjec dest[].add(src[][start]) start += step pyNone + +declarePyType Ellipsis(tpToken): + discard + +let pyEllipsis* = newPyEllipsisSimple() + +proc dollar(self: PyEllipsisObject): string = "Ellipsis" +method `$`*(self: PyEllipsisObject): string = + self.dollar + +implEllipsisMagic repr: + newPyString self.dollar + +implEllipsisMagic New(tp: PyObject): + return pyEllipsis diff --git a/Python/ast.nim b/Python/ast.nim index e13ca47..a4493e0 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -7,7 +7,10 @@ import strformat import asdl import ../Parser/[token, parser] -import ../Objects/[pyobject, noneobject, numobjects, boolobjectImpl, stringobjectImpl] +import ../Objects/[pyobject, noneobject, + numobjects, boolobjectImpl, stringobjectImpl, + sliceobject # pyEllipsis + ] import ../Utils/[utils, compat] @@ -1053,8 +1056,10 @@ ast atom, [AsdlExpr]: of Token.None: result = newAstConstant(pyNone) + of Token.Ellipsis: + result = newAstConstant(pyEllipsis) else: - raiseSyntaxError("ellipsis not implemented") + unreachable() assert result != nil setNo(result, parseNode.children[0]) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 312a62b..fa04c39 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -107,6 +107,7 @@ implBltinFunc buildClass(funcObj: PyFunctionObject, name: PyStrObject), "__build registerBltinObject("NotImplemented", pyNotImplemented) +registerBltinObject("Ellipsis", pyEllipsis) registerBltinObject("None", pyNone) registerBltinObject("type", pyTypeObjectType) registerBltinObject("range", pyRangeObjectType) From fcfe141540e46f4eb4b62665a912bcba87ceccb3 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sat, 19 Jul 2025 00:41:35 +0800 Subject: [PATCH 086/163] fix(windows): lexer init not run; buildinfo contains newline --- Modules/getbuildinfo.nim | 8 +++++++- Parser/grammar.nim | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Modules/getbuildinfo.nim b/Modules/getbuildinfo.nim index 41588a0..165fb38 100644 --- a/Modules/getbuildinfo.nim +++ b/Modules/getbuildinfo.nim @@ -7,6 +7,8 @@ when NimMajor > 1: else: from std/os import `/../`, parentDir import ./os_findExe_patch +when defined(windows): + from std/strutils import stripLineEnd ## see CPython/configure.ac @@ -20,7 +22,11 @@ else: bind srcdir_git let res = gorgeEx(git.exe & " --git-dir " & srcdir_git & " " & sub) assert res.exitCode == 0, res.output - res.output + when defined(windows): + var outp = res.output + outp.stripLineEnd + outp + else: res.output const version = git.exec"rev-parse --short HEAD" diff --git a/Parser/grammar.nim b/Parser/grammar.nim index 5148da3..65ceaa5 100644 --- a/Parser/grammar.nim +++ b/Parser/grammar.nim @@ -41,7 +41,7 @@ type cursor: int -const grammarLines = slurp("Grammar").split("\n") +const grammarLines = slurp("Grammar").splitLines() var From 99c34b4a14278565bb4dd28041c24583a6caa823 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sat, 19 Jul 2025 00:59:00 +0800 Subject: [PATCH 087/163] fixup! feat(implMagic): for binary magic allow typed params --- Objects/pyobject.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 2a9c5ea..2b5cb85 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -253,7 +253,7 @@ macro checkArgTypes*(nameAndArg, code: untyped): untyped = let obj = if multiArg: nnkBracketExpr.newTree( ident("args"), newIntLitNode(idx), - ) else: oriParams[idx+1][0] + ) else: oriParams[idx+2][0] let name = child[0] let tp = child[1] if tp.strVal == "PyObject": # won't bother checking From 5f8bad7ef9af610555cedd081637dd01d7f921d8 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sat, 19 Jul 2025 16:22:07 +0800 Subject: [PATCH 088/163] fix(Parser.grammar): not correct err msg --- Parser/grammar.nim | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Parser/grammar.nim b/Parser/grammar.nim index 65ceaa5..6474c5c 100644 --- a/Parser/grammar.nim +++ b/Parser/grammar.nim @@ -41,7 +41,9 @@ type cursor: int -const grammarLines = slurp("Grammar").splitLines() +const + grammarFilepath = "Grammar" + grammarLines = slurp(grammarFilepath).splitLines() var @@ -485,7 +487,12 @@ proc matchH(grammar: Grammar): GrammarNode = proc lexGrammar = - let lines = grammarLines + template lines: untyped = grammarLines + template badGrammar(msg) = + quit( + &"lexerGrammer: Unknown syntax at {grammarFilepath}:{lineIdx}:{line}. " & + msg + ) var lineIdx = 0 while lineIdx < lines.len(): @@ -495,7 +502,7 @@ proc lexGrammar = continue let colonIdx = line.find(':') if colonIdx == -1: - quit("Unknown syntax at {lineIdx}: {line}") + badGrammar"expect colon but found none" let name = line[0.. Date: Mon, 21 Jul 2025 20:59:35 +0800 Subject: [PATCH 089/163] feat(versionInfo): add PyMajor,PyMinor,PyPatch --- Python/versionInfo.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/versionInfo.nim b/Python/versionInfo.nim index 06db3e6..a98a903 100644 --- a/Python/versionInfo.nim +++ b/Python/versionInfo.nim @@ -4,6 +4,10 @@ const Minor* = 1 Patch* = 1 + PyMajor*{.intdefine.} = 3 + PyMinor*{.intdefine.} = 13 + PyPatch*{.intdefine.} = 0 + const sep = '.' template asVersion(major, minor, patch: int): string = $major & sep & $minor & sep & $patch From 89823ce4dea4ff16d4b5db982e19ebedc7f2f25b Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 21 Jul 2025 21:00:23 +0800 Subject: [PATCH 090/163] refact(lexer): mv types to lexerTypes --- Parser/lexer.nim | 12 ++---------- Parser/lexerTypes.nim | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 10 deletions(-) create mode 100644 Parser/lexerTypes.nim diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 9b4ff36..a4baba5 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -8,6 +8,8 @@ import tables import parseutils import token +import ./lexerTypes +export lexerTypes except lineNo, indentStack import ../Utils/[utils, compat] type @@ -15,16 +17,6 @@ type Source = ref object lines: seq[string] - Mode* {.pure.} = enum - Single - File - Eval - - Lexer* = ref object - indentStack: seq[int] # Stack to track indentation levels - lineNo: int - tokenNodes*: seq[TokenNode] # might be consumed by parser - fileName*: string template indentLevel(lexer: Lexer): int = lexer.indentStack[^1] diff --git a/Parser/lexerTypes.nim b/Parser/lexerTypes.nim new file mode 100644 index 0000000..6dce8b1 --- /dev/null +++ b/Parser/lexerTypes.nim @@ -0,0 +1,18 @@ + +import token + +type + Mode* {.pure.} = enum + Single + File + Eval + + Lexer* = ref object + indentStack: seq[int] # Stack to track indentation levels + lineNo: int + tokenNodes*: seq[TokenNode] # might be consumed by parser + fileName*: string + +proc lineNo*(lexer: Lexer): var int{.inline.} = lexer.lineNo + +proc indentStack*(lexer: Lexer): var seq[int]{.inline.} = lexer.indentStack From 8ee5bfb747602eb9644b16bb3b16ce23c80a7141 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 21 Jul 2025 21:01:42 +0800 Subject: [PATCH 091/163] feat(warnings): nim api: warnExplicit (not complete, a simpler impl) --- Objects/warningobject.nim | 84 +++++++++++++++++++++++++++++++++++++++ Python/warnings.nim | 47 ++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 Objects/warningobject.nim create mode 100644 Python/warnings.nim diff --git a/Objects/warningobject.nim b/Objects/warningobject.nim new file mode 100644 index 0000000..82cd56e --- /dev/null +++ b/Objects/warningobject.nim @@ -0,0 +1,84 @@ + +import std/options +from std/strutils import `%` +import ./[ + pyobject, + stringobject, + numobjects, + exceptions, + noneobject, +] + +declarePyType Warning(base(Exception)): discard + +declarePyType WarningMessage(): + message: PyStrObject #PyStrObject + category: PyObject #Warning + filename: PyStrObject #PyStrObject + lineno: PyIntObject #PyIntObject + file: PyObject + line: PyObject #Option PyStrObject + source: PyObject + private_category_name: Option[string] + +proc newPyWarningMessage*( + message: PyStrObject, category: PyTypeObject, filename: PyStrObject, lineno: PyIntObject, + file: PyObject = pyNone, line: PyObject = pyNone, source: PyObject = pyNone + ): PyWarningMessageObject = + ## Create a new `WarningMessage` object. + let self = newPyWarningMessageSimple() + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.file = file + self.line = line + self.source = source + self.private_category_name = + if category.ofPyNoneObject: none(string) + else: some(category.name) + result = self + +proc categoryName*(self: PyWarningMessageObject): string = + ## Get the name of the warning category. + if self.private_category_name.isSome: + self.private_category_name.unsafeGet + else: "None" + +#[ +implWarningMessageMagic init: + # (message, category, filename, lineno: PyObject, + # file, line, source: PyObject = pyNone): + if args.len < 3 or args.len > 7: + return newTypeError( # Fixed typo from 'retun' to 'return' + "WarningMessage.__init__() takes 3 to 7 positional arguments but $# were given" % $args.len) + else: # Added else to handle the case when the argument count is valid + template nArgOrNone(n): PyObject = + if args.len > n: args[n] else: pyNone + newPyWarningMessage(args[0], + args[1], + args[2], + args[3], + nArgOrNone(4), + nArgOrNone(5), + nArgOrNone(6), + ) +]# + +template callReprOrStr(obj: PyObject, reprOrStr): string = + obj.getMagic(reprOrStr)(obj).PyStrObject.str + +proc `$`*(self: PyWarningMessageObject): string = + ("{message : $#, category : $#, filename : $#, lineno : $#, " & + "line : $#}") % [self.message.callReprOrStr(repr), self.categoryName, + self.filename.callReprOrStr(repr), self.lineno.callReprOrStr(str), self.line.callReprOrStr(repr)] + +implWarningMessageMagic str: + newPyStr $self + +template declWarning(w){.dirty.} = + declarePyType w(base(Warning)): discard + +declWarning SyntaxWarning + +declWarning DeprecationWarning diff --git a/Python/warnings.nim b/Python/warnings.nim new file mode 100644 index 0000000..db6e01f --- /dev/null +++ b/Python/warnings.nim @@ -0,0 +1,47 @@ +## `_warnings` + +import std/strformat +import ../utils/compat +import ../Objects/[warningobject, stringobject, numobjects, pyobjectBase] +export warningobject + +proc formatwarnmsg_impl_nonewline(msg: PyWarningMessageObject): string{.raises: [].} = + ## `_formatwarnmsg_impl` of `Lib/_py_warnings.py` + ## but without the tailing newline + let category = msg.categoryName + let + filename = msg.filename.str + linenoObj = msg.lineno + message = msg.message.str + let linenoS = + try: $linenoObj + except Exception: "" ## Handle potential exceptions + result = fmt"{filename}:{linenoS}: {category}: {message}" ## Use the new message variable + + # TODO: linecache.getline(msg.filename, msg.lineno) + # TODO: tracemalloc.get_object_traceback(msg.source) + + +proc showwarnmsg_impl(msg: PyWarningMessageObject){.raises: [].} = + ## `_showwarnmsg_impl` of `Lib/_py_warnings.py` + # _formatwarnmsg + ## TODO: sys.stderr + try: + errEchoCompat( + # XXX: TODO: `_formatwarnmsg_impl` + msg.formatwarnmsg_impl_nonewline + ) + except IOError: discard + except Exception: discard ## workaround for NIM-BUG about `$`'s callMagic has `Exception` exception + +proc warnExplicit*(category: PyTypeObject#[typedesc[Warning]]#, message: string, filename: string, lineno: int, + #module: string, registry: PyObject + ){.raises: [].} = + ## `warn_explicit` of `_warnings.c` + showwarnmsg_impl newPyWarningMessage( + message.newPyStr, category, filename.newPyStr, lineno.newPyInt, + #source + ) + +# TODO: another overload for `warnExplicit` with `PyWarningMessageObject` + From 7c7043977ca289ca08e379ed47b9a45a79ba84d3 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 21 Jul 2025 21:07:58 +0800 Subject: [PATCH 092/163] feat(Utils): translateEscape from nimpylib@c0a6f744ca9c7787d38e78ecaef0a6478cbe0859 --- Utils/translateEscape.nim | 280 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 Utils/translateEscape.nim diff --git a/Utils/translateEscape.nim b/Utils/translateEscape.nim new file mode 100644 index 0000000..765f039 --- /dev/null +++ b/Utils/translateEscape.nim @@ -0,0 +1,280 @@ + +import std/macros +import std/unicode +import std/strformat + +const allowNimEscape*{.booldefine: "nimpylibTranslateEscapeAllowNimExt" +.} = false ## - if false(default), use extract Python's format; \ +## - if true, allow Nim's escape format like `\u{...}`, `\p`, `\N` + +when allowNimEscape: + template `?+`(bothC, onlyNimC): untyped = {bothC, onlyNimC} + template `?+*`(bothC; onlyNimChars): untyped = {bothC} + onlyNimChars +else: + template `?+`(bothC, _): untyped = bothC + template `?+*`(bothC; _): untyped = bothC + +# compiler/lineinfos.nim +type + TranslateEscapeErr* = enum + teeExtBad_uCurly = (-1, "bad hex digit in \\u{...}") ## Nim's EXT + teeBadEscape = "invalid escape sequence" + teeBadOct = "invalid octal escape sequence" ## SyntaxWarning in Python + teeUniOverflow = "illegal Unicode character" + teeTrunc_x2 = "truncated \\xXX escape" + teeTrunc_u4 = "truncated \\uXXXX escape" + teeTrunc_U8 = "truncated \\UXXXXXXXX escape" + +type + Token = object # a Nim token + literal: string # the parsed (string) literal + +type + LexMessage* = proc(info: LineInfo, kind: TranslateEscapeErr, nimStyleErrMsg: string) + Lexer*[U: static[bool], M] = object + ## `U` means whether supporting escape about unicode; + ## `M` is a `proc(LineInfo, TranslateEscapeErr, string)`, + ## leaving as a generic to allow custom pragma like `{.raises: [].}` + bufLen: int + bufpos: int + buf: string + + lineInfo*: LineInfo + lexMessageImpl: M # static method no supported by Nim + + +proc newLexerNoMessager[U: static[bool], M](s: string): Lexer[U, M] = + result.buf = s + result.bufLen = s.len + + +proc newLexerImpl[U: static[bool], M](s: string, messager: M): Lexer[U, M] = + result = newLexerNoMessager[U, M](s) + result.lexMessageImpl = messager + +template newLexer*[U: static[bool]](s: string, messager): Lexer = + ## create a new lexer with a messager + bind newLexerImpl + newLexerImpl[U, typeof(messager)](s, messager) + +template allow_unicode[U: static[bool], M](L: Lexer[U, M]): bool = U + + +proc staticLexMessageImpl(info: LineInfo, kind: TranslateEscapeErr, nimStyleErrMsg: string){.compileTime.} = + let errMsg = '\n' & fmt""" +File "{info.filename}", line {info.line}, col {info.column} + {nimStyleErrMsg}""" # Updated to use nimStyleErrMsg + case kind + of teeBadEscape: + warning errMsg + else: + error errMsg + #else: debugEcho errMsg + +proc newStaticLexer*[U: static[bool]](s: string): Lexer[U, LexMessage]{.compileTime.} = + ## use Nim-Like error message + result = newLexerNoMessager[U, LexMessage](s) + result.lexMessageImpl = staticLexMessageImpl + +proc lexMessage[U: static[bool], M](L: Lexer[U, M], kind: TranslateEscapeErr, nimStyleErrMsg: string) = + var info = L.lineInfo + # XXX: when is multiline string, we cannot know where the position is, + # as Nim has been translated multiline as single-line. + info.column += L.bufpos + 1 # plus 1 to become 1-based + L.lexMessageImpl(info, kind, nimStyleErrMsg) + +func handleOctChars(L: var Lexer, xi: var int) = + ## parse at most 3 chars + for _ in 0..2: + let c = L.buf[L.bufpos] + if c notin {'0'..'7'}: break + xi = (xi * 8) + (ord(c) - ord('0')) + inc(L.bufpos) + if L.bufpos == L.bufLen: break + +proc handleHexChar(L: var Lexer, xi: var int; position: int, eKind: TranslateEscapeErr) = + ## parseHex in std/parseutils allows `_` and prefix `0x`, which shall not be allowed here + template invalid(c) = + lexMessage(L, eKind, + "expected a hex digit, but found: " & c & + "; maybe prepend with 0") + if L.bufpos == L.bufLen: invalid("END") + let c = L.buf[L.bufpos] + case c + of '0'..'9': + xi = (xi shl 4) or (ord(c) - ord('0')) + inc(L.bufpos) + of 'a'..'f': + xi = (xi shl 4) or (ord(c) - ord('a') + 10) + inc(L.bufpos) + of 'A'..'F': + xi = (xi shl 4) or (ord(c) - ord('A') + 10) + inc(L.bufpos) + of '"', '\'': + if position <= 1: invalid(c) + # do not progress the bufpos here. + elif position == 0: inc(L.bufpos) + else: + invalid(c) + +const + CR = '\r' + LF = '\n' + FF = '\f' + BACKSPACE = '\b' + ESC = '\e' + +template uncheckedAddUnicodeCodePoint(s: var string, i: int) = + ## add a Unicode codepoint to the string, assuming `i` is a valid codepoint + s.add cast[Rune](i) + + +proc getEscapedChar(L: var Lexer, tok: var Token) = + inc(L.bufpos) # skip '\' + when L.allow_unicode: + template uniOverErr(curVal: string) = + lexMessage(L, teeUniOverflow, + "Unicode codepoint must be lower than 0x10FFFF, but was: " & curVal) + + template invalidEscape() = + lexMessage(L, teeBadEscape, "invalid character constant") + + template doIf(cond, body) = + when cond: body + else: invalidEscape() + + template addTokLitOnAllowNim(cOrS) = + doIf allowNimEscape: + tok.literal.add(cOrS) + inc(L.bufpos) + let c = L.buf[L.bufpos] + case c + of 'n' ?+ 'N': + tok.literal.add('\L') + inc(L.bufpos) + of 'p', 'P': + addTokLitOnAllowNim("\p") + of 'r' ?+* {'R', 'c', 'C'}: + tok.literal.add(CR) + inc(L.bufpos) + of 'l' ?+ 'L': + tok.literal.add(LF) + inc(L.bufpos) + of 'f' ?+ 'F': + tok.literal.add(FF) + inc(L.bufpos) + of 'e', 'E': + addTokLitOnAllowNim(ESC) + of 'a' ?+ 'A': + tok.literal.add('\a') + inc(L.bufpos) + of 'b' ?+ 'B': + tok.literal.add(BACKSPACE) + inc(L.bufpos) + of 'v' ?+ 'V': + tok.literal.add('\v') + inc(L.bufpos) + of 't' ?+ 'T': + tok.literal.add('\t') + inc(L.bufpos) + of '\'', '\"': + tok.literal.add(c) + inc(L.bufpos) + of '\\': + tok.literal.add('\\') + inc(L.bufpos) + of 'x' ?+ 'X': + inc(L.bufpos) + var xi = 0 + handleHexChar(L, xi, 1, teeTrunc_x2) + handleHexChar(L, xi, 2, teeTrunc_x2) + tok.literal.add(chr(xi)) + of 'U': + doIf L.allow_unicode: + # \Uhhhhhhhh + inc(L.bufpos) + var xi = 0 + let start = L.bufpos + for i in 0..7: + handleHexChar(L, xi, i, teeTrunc_U8) + if xi > 0x10FFFF: + uniOverErr L.buf[start..L.bufpos-2] + uncheckedAddUnicodeCodePoint(tok.literal, xi) + of 'u': + doIf L.allow_unicode: + inc(L.bufpos) + var xi = 0 + template handle4Hex = + for i in 1..4: + handleHexChar(L, xi, i, teeTrunc_u4) + when allowNimEscape: + if L.buf[L.bufpos] == '{': + inc(L.bufpos) + let start = L.bufpos + while L.buf[L.bufpos] != '}': + handleHexChar(L, xi, 0, teeExtBad_uCurly) + if start == L.bufpos: + lexMessage(L, teeExtBad_uCurly, + "Unicode codepoint cannot be empty") + inc(L.bufpos) + if xi > 0x10FFFF: + uniOverErr L.buf[start..L.bufpos-2] + else: handle4Hex + else: handle4Hex + uncheckedAddUnicodeCodePoint(tok.literal, xi) + of '0'..'7': + var xi = 0 + handleOctChars(L, xi) + tok.literal.add(chr(xi)) + else: + invalidEscape() + tok.literal.add('\\') + inc(L.bufpos) + tok.literal.add(c) + inc(L.bufpos) + +proc getString(L: var Lexer, tok: var Token) = + var pos = L.bufpos + + while pos < L.bufLen: + let c = L.buf[pos] + if c == '\\': + L.bufpos = pos + getEscapedChar(L, tok) + pos = L.bufpos + else: + tok.literal.add(c) + pos.inc + L.bufpos = pos + +proc getString(L: var Lexer): Token = + L.getString result + +proc translateEscape*(L: var Lexer): string = + L.getString().literal + +proc translateEscape*(pattern: static[string], + allow_unicode: static[bool] = true, +): string{.compileTime.} = + ## like `translateEscapeWithErr` but without lineInfo error msg + var L = newStaticLexer[allow_unicode](pattern) + L.translateEscape + +macro getLineInfoObj(n): LineInfo = + ## get the line info from a node + let linfo = n.lineInfoObj + result = nnkObjConstr.newTree(bindSym"LineInfo", + nnkExprColonExpr.newTree(ident"filename", newLit linfo.filename), + nnkExprColonExpr.newTree(ident"line", newLit linfo.line), + nnkExprColonExpr.newTree(ident"column", newLit linfo.column) + ) + +template translateEscapeWithErr*(pattern: static[string], + allow_unicode: static[bool] = true, +): string = + bind newStaticLexer, getLineInfoObj, translateEscape + const res = block: + var L = newStaticLexer[allow_unicode](pattern) + L.lineInfo = pattern.getLineInfoObj + L.translateEscape + res From 06ab264fb645df5671a55e84832dbc31a6127c51 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 21 Jul 2025 21:11:00 +0800 Subject: [PATCH 093/163] feat(parser/strlit),fix(py): add decode_unicode_with_escapes, escape in strlit is handled now --- Parser/lexer.nim | 4 +++- Parser/string_parser.nim | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 Parser/string_parser.nim diff --git a/Parser/lexer.nim b/Parser/lexer.nim index a4baba5..3b06340 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -8,6 +8,7 @@ import tables import parseutils import token +import ./string_parser import ./lexerTypes export lexerTypes except lineNo, indentStack import ../Utils/[utils, compat] @@ -143,7 +144,8 @@ proc getNextToken( if idx + l + 1 == line.len: # pairing `"` not found raiseSyntaxError("Invalid string syntax") else: - result = newTokenNode(Token.String, lexer.lineNo, idx, line[idx+1..idx+l]) + let s = lexer.decode_unicode_with_escapes(line[idx+1..idx+l]) + result = newTokenNode(Token.String, lexer.lineNo, idx, s) idx += l + 2 of '\n': diff --git a/Parser/string_parser.nim b/Parser/string_parser.nim new file mode 100644 index 0000000..1e002a3 --- /dev/null +++ b/Parser/string_parser.nim @@ -0,0 +1,35 @@ + + +import ../Python/[ + warnings, versionInfo, +] +import ../Objects/[ + pyobject, +] +import ../Utils/[ + utils, + translateEscape, +] +import lexerTypes + +proc lexMessage(info: LineInfo, kind: TranslateEscapeErr, _: string){.raises: [SyntaxError].} = + let arg = $kind + case kind + of teeBadEscape: + warnExplicit( + when PyMinor >= 12: pySyntaxWarningObjectType + else: pyDeprecationWarningObjectType + , + arg, info.fileName, info.line + ) + else: + raiseSyntaxError(arg, info.fileName, info.line, info.column) + +proc decode_unicode_with_escapes*(L: lexerTypes.Lexer, s: string): string{. + raises: [SyntaxError].} = + var lex = newLexer[true](s, lexMessage) + lex.lineInfo.fileName = L.fileName + lex.lineInfo.line = L.lineNo + lex.translateEscape + + From 8bd66ba1772e038503fe03c4a3e82acdd812fd65 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 00:19:49 +0800 Subject: [PATCH 094/163] feat(str): UCS-4 or UCS-1 based over only UCS-1(ascii) --- Objects/boolobjectImpl.nim | 4 +- Objects/descrobject.nim | 2 +- Objects/dictobject.nim | 22 ++-- Objects/exceptions.nim | 30 +++--- Objects/exceptionsImpl.nim | 8 +- Objects/listobject.nim | 6 +- Objects/notimplementedobject.nim | 2 +- Objects/numobjects.nim | 35 +++---- Objects/pyobject.nim | 10 +- Objects/rangeobject.nim | 8 +- Objects/setobject.nim | 8 +- Objects/sliceobject.nim | 6 +- Objects/stringobject.nim | 170 ++++++++++++++++++++++++++++++- Objects/stringobjectImpl.nim | 17 ++-- Objects/tupleobject.nim | 20 ++-- Objects/typeobject.nim | 14 +-- Objects/warningobject.nim | 2 +- Python/bltinmodule.nim | 6 +- Python/compile.nim | 6 +- Python/cpython.nim | 2 +- Python/neval.nim | 34 +++---- Python/symtable.nim | 6 +- Python/traceback.nim | 6 +- 23 files changed, 292 insertions(+), 132 deletions(-) diff --git a/Objects/boolobjectImpl.nim b/Objects/boolobjectImpl.nim index 08bbf89..24ef8e4 100644 --- a/Objects/boolobjectImpl.nim +++ b/Objects/boolobjectImpl.nim @@ -61,9 +61,9 @@ implBoolMagic eq: implBoolMagic repr: if self.b: - return newPyString("True") + return newPyAscii("True") else: - return newPyString("False") + return newPyAscii("False") implBoolMagic hash: newPyInt(Hash(self.b)) diff --git a/Objects/descrobject.nim b/Objects/descrobject.nim index 11bf9ef..d47402d 100644 --- a/Objects/descrobject.nim +++ b/Objects/descrobject.nim @@ -48,7 +48,7 @@ implMethodDescrMagic get: if other.pyType != self.dType: let msg = fmt"descriptor {self.name} for {self.dType.name} objects " & fmt"doesn't apply to {other.pyType.name} object" - return newTypeError(msg) + return newTypeError(newPyStr msg) let owner = other case self.kind of NFunc.BltinFunc: diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index 952ace3..08b51cc 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -39,7 +39,7 @@ template checkHashableTmpl(obj) = if hashFunc.isNil: let tpName = obj.pyType.name let msg = "unhashable type: " & tpName - return newTypeError(msg) + return newTypeError newPyStr(msg) implDictMagic contains, [mutable: read]: @@ -48,21 +48,23 @@ implDictMagic contains, [mutable: read]: result = self.table.getOrDefault(other, nil) except DictError: let msg = "__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" - return newTypeError(msg) + return newTypeError newPyAscii(msg) if result.isNil: return pyFalseObj else: return pyTrueObj implDictMagic repr, [mutable: read, reprLockWithMsg"{...}"]: - var ss: seq[string] + var ss: seq[UnicodeVariant] for k, v in self.table.pairs: let kRepr = k.callMagic(repr) let vRepr = v.callMagic(repr) errorIfNotString(kRepr, "__str__") errorIfNotString(vRepr, "__str__") - ss.add fmt"{PyStrObject(kRepr).str}: {PyStrObject(vRepr).str}" - return newPyString("{" & ss.join(", ") & "}") + ss.add newUnicodeUnicodeVariant PyStrObject(kRepr).str.toRunes & + toRunes": " & + PyStrObject(vRepr).str.toRunes + return newPyString(toRunes"{" & ss.joinAsRunes(", ") & toRunes"}") implDictMagic len, [mutable: read]: @@ -78,16 +80,16 @@ implDictMagic getitem, [mutable: read]: result = self.table.getOrDefault(other, nil) except DictError: let msg = "__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" - return newTypeError(msg) + return newTypeError newPyAscii(msg) if not (result.isNil): return result - var msg: string + var msg: PyStrObject let repr = other.pyType.magicMethods.repr(other) if repr.isThrownException: - msg = "exception occured when generating key error msg calling repr" + msg = newPyAscii"exception occured when generating key error msg calling repr" else: - msg = PyStrObject(repr).str + msg = PyStrObject(repr) return newKeyError(msg) @@ -97,7 +99,7 @@ implDictMagic setitem, [mutable: write]: self.table[arg1] = arg2 except DictError: let msg = "__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" - return newTypeError(msg) + return newTypeError newPyAscii(msg) pyNone implDictMethod copy(), [mutable: read]: diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 5d17415..376e41a 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -102,12 +102,11 @@ template newProcTmpl(excpName) = excp.thrown = true excp - - proc `new excpName Error`*(msgStr:string): PyBaseErrorObject{.inline.} = + proc `new excpName Error`*(msgStr: PyStrObject): PyBaseErrorObject{.inline.} = let excp = `newPy excpName ErrorSimple`() excp.tk = ExceptionToken.`excpName` excp.thrown = true - excp.msg = newPyString(msgStr) + excp.msg = msgStr excp @@ -121,16 +120,17 @@ macro genNewProcs: untyped = genNewProcs - -template newAttributeError*(tpName, attrName: string): PyExceptionObject = - let msg = tpName & " has no attribute " & attrName +template newAttributeError*(tpName, attrName: PyStrObject): PyExceptionObject = + let msg = tpName & newPyAscii" has no attribute " & attrName newAttributeError(msg) +template newAttributeError*(tpName, attrName: string): PyExceptionObject = + newAttributeError(tpName.newPyStr, attrName.newPyStr) -template newIndexTypeError*(typeName:string, obj:PyObject): PyExceptionObject = - let name = $obj.pyType.name - let msg = typeName & " indices must be integers or slices, not " & name +template newIndexTypeError*(typeName: PyStrObject, obj:PyObject): PyExceptionObject = + let name = obj.pyType.name + let msg = typeName & newPyAscii(" indices must be integers or slices, not ") & newPyStr name newTypeError(msg) @@ -159,13 +159,13 @@ template errorIfNotString*(pyObj: untyped, methodName: string) = if not pyObj.ofPyStrObject: let typeName {. inject .} = pyObj.pyType.name let msg = methodName & fmt" returned non-string (type {typeName})" - return newTypeError(msg) + return newTypeError newPyStr(msg) template errorIfNotBool*(pyObj: untyped, methodName: string) = if not pyObj.ofPyBoolObject: let typeName {. inject .} = pyObj.pyType.name let msg = methodName & fmt" returned non-bool (type {typeName})" - return newTypeError(msg) + return newTypeError(newPyStr msg) template getIterableWithCheck*(obj: PyObject): (PyObject, UnaryMethod) = @@ -174,13 +174,13 @@ template getIterableWithCheck*(obj: PyObject): (PyObject, UnaryMethod) = let iterFunc = obj.getMagic(iter) if iterFunc.isNil: let msg = obj.pyType.name & " object is not iterable" - retTuple = (newTypeError(msg), nil) + retTuple = (newTypeError(newPyStr msg), nil) break body let iterObj = iterFunc(obj) let iternextFunc = iterObj.getMagic(iternext) if iternextFunc.isNil: let msg = fmt"iter() returned non-iterator of type " & iterObj.pyType.name - retTuple = (newTypeError(msg), nil) + retTuple = (newTypeError(newPyStr msg), nil) break body retTuple = (iterobj, iternextFunc) retTuple @@ -193,7 +193,7 @@ template checkArgNum*(expected: int, name="") = msg = name & " takes exactly " & $expected & fmt" argument ({args.len} given)" else: msg = "expected " & $expected & fmt" argument ({args.len} given)" - return newTypeError(msg) + return newTypeError(newPyStr msg) template checkArgNumAtLeast*(expected: int, name="") = @@ -203,4 +203,4 @@ template checkArgNumAtLeast*(expected: int, name="") = msg = name & " takes at least " & $expected & fmt" argument ({args.len} given)" else: msg = "expected at least " & $expected & fmt" argument ({args.len} given)" - return newTypeError(msg) + return newTypeError(newPyStr msg) diff --git a/Objects/exceptionsImpl.nim b/Objects/exceptionsImpl.nim index 976991b..562c41d 100644 --- a/Objects/exceptionsImpl.nim +++ b/Objects/exceptionsImpl.nim @@ -24,16 +24,16 @@ template newMagicTmpl(excpName: untyped, excpNameStr: string) = if self.msg.isNil: msg = "" # could be improved elif self.msg.ofPyStrObject: - msg = PyStrObject(self.msg).str + msg = $PyStrObject(self.msg) else: # ensure this is either an throwned exception or string for user-defined type let msgObj = self.msg.callMagic(repr) if msgObj.isThrownException: msg = "evaluating __repr__ failed" else: - msg = PyStrObject(msgObj).str + msg = $PyStrObject(msgObj) let str = $self.tk & "Error: " & msg - newPyString(str) + newPyAscii(str) # this is for initialization at Python level `impl excpName ErrorMagic` New: @@ -70,7 +70,7 @@ proc isExceptionType*(obj: PyObject): bool = proc fromBltinSyntaxError*(e: SyntaxError, fileName: PyStrObject): PyExceptionObject = - let excpObj = newSyntaxError(e.msg) + let excpObj = newSyntaxError newPyStr(e.msg) # don't have code name excpObj.traceBacks.add (PyObject fileName, PyObject nil, e.lineNo, e.colNo) excpObj diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 9f48687..6a78a77 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -34,8 +34,8 @@ implListMagic setitem, [mutable: write]: self.items[idx] = arg2 return pyNone if arg1.ofPySliceObject: - return newTypeError("store to slice not implemented") - return newIndexTypeError("list", arg1) + return newTypeError newPyAscii("store to slice not implemented") + return newIndexTypeError(newPyAscii"list", arg1) implListMethod append(item: PyObject), [mutable: write]: @@ -104,7 +104,7 @@ implListMethod insert(idx: PyIntObject, item: PyObject), [mutable: write]: implListMethod pop(), [mutable: write]: if self.items.len == 0: let msg = "pop from empty list" - return newIndexError(msg) + return newIndexError newPyAscii(msg) self.items.pop implListMethod remove(target: PyObject), [mutable: write]: diff --git a/Objects/notimplementedobject.nim b/Objects/notimplementedobject.nim index 2ad063e..0641d18 100644 --- a/Objects/notimplementedobject.nim +++ b/Objects/notimplementedobject.nim @@ -15,7 +15,7 @@ method `$`*(self: PyNotImplementedObject): string = self.dollar implNotImplementedMagic repr: - newPyString self.dollar + newPyAscii self.dollar implNotImplementedMagic New(tp: PyObject): return pyNotImplemented diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index 52c23da..2b614d9 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -470,8 +470,9 @@ proc lMod(v, w: PyIntObject, modRes: var PyIntObject): bool = (modRes.sign == Positive and w.sign == Negative): modRes = modRes + w +let divZeroError = newPyAscii"division by zero" template retZeroDiv = - return newZeroDivisionError"division by zero" + return newZeroDivisionError divZeroError proc `%`*(a, b: PyIntObject): PyObject = var res: PyIntObject @@ -734,9 +735,9 @@ proc newPyInt(i: int): PyIntObject = result.digits.add uint32(ii shr 32) result.normalize ]# -proc fromStr(s: string): PyIntObject = +proc fromStr[C: char|Rune](s: openArray[C]): PyIntObject = result = newPyIntSimple() - var sign = s[0] == '-' + var sign = s[0] == C'-' # assume s not empty result.digits.add 0 for i in (sign.int).. other.b.len: return false + for i, c in self.a: + if uint32(c) != uint32(other.b[i]): + return false + return true + + case ((self.ascii.uint8 shl 1) or other.ascii.uint8) + of 0: + return self.unicodeStr == other.unicodeStr + of 1: + cmpAttr(unicodeStr, asciiStr) + of 2: + cmpAttr(asciiStr, unicodeStr) + else: # of 3 + return self.asciiStr == other.asciiStr + declarePyType Str(tpToken): - str: string + str: UnicodeVariant + +proc `==`*(self, other: PyStrObject): bool {. inline, cdecl .} = + self.str == other.str + +proc hash*(self: PyStrObject): Hash {. inline, cdecl .} = + result = hash(self.str) # don't write as self.str.hash as that returns attr method `$`*(strObj: PyStrObject): string = - "\"" & $strObj.str & "\"" + $strObj.str + +proc repr*(strObj: PyStrObject): string = + '\'' & $strObj.str & '\'' # TODO -proc newPyString*(str: string): PyStrObject = +proc newPyString*(str: UnicodeVariant): PyStrObject{.inline.} = result = newPyStrSimple() result.str = str -let newPyStr* = newPyString +proc newPyString*(str: string, ensureAscii=false): PyStrObject = + newPyString str.newUnicodeVariant(ensureAscii) +proc newPyString*(str: seq[Rune]): PyStrObject = + newPyString newUnicodeUnicodeVariant(str) +proc newPyAscii*(str: string): PyStrObject = + newPyString newAsciiUnicodeVariant(str) + +# TODO: make them faster +proc newPyString*(r: Rune): PyStrObject{.inline.} = newPyString @[r] +proc newPyString*(c: char): PyStrObject{.inline.} = newPyString $c + +proc `&`*(self: PyStrObject, i: PyStrObject): PyStrObject {. cdecl .} = + newPyString self.str & i.str + +proc len*(strObj: PyStrObject): int {. inline, cdecl .} = + strObj.str.doBothKindOk(len) + +template newPyStr*(s: string; ensureAscii=false): PyStrObject = + bind newPyString + newPyString(s, ensureAscii) +template newPyStr*(s: seq[Rune]|UnicodeVariant): PyStrObject = + bind newPyString + newPyString(s) diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index a00e6fe..a06c3a0 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -1,4 +1,4 @@ -import hashes + import std/strformat import pyobject import baseBundle @@ -6,13 +6,6 @@ import stringobject export stringobject -proc hash*(self: PyStrObject): Hash {. inline, cdecl .} = - hash(self.str) - - -proc `==`*(self, other: PyStrObject): bool {. inline, cdecl .} = - self.str == other.str - # redeclare this for these are "private" macros @@ -31,9 +24,11 @@ implStrMagic eq: implStrMagic str: self +implStrMagic len: + newPyInt self.len implStrMagic repr: - newPyString($self) + newPyString(repr self) implStrMagic hash: @@ -47,9 +42,9 @@ implStrMagic New(tp: PyObject, obj: PyObject): return obj.callMagic(repr) result = fun(obj) if not result.ofPyStrObject: - return newTypeError( + return newTypeError newPyStr( &"__str__ returned non-string (type {result.pyType.name:.200s})") implStrMagic add(i: PyStrObject): - newPyStr self.str & i.str + self & i diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 07a5b47..b0dde4d 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -34,7 +34,7 @@ template genCollectMagics*(items, implNameMagic repr, mutReadRepr: - var ss: seq[string] + var ss: seq[UnicodeVariant] for item in self.items: var itemRepr: PyStrObject let retObj = item.callMagic(repr) @@ -91,7 +91,7 @@ template genSequenceMagics*(nameStr, implNameMagic init: if 1 < args.len: let msg = nameStr & fmt" expected at most 1 args, got {args.len}" - return newTypeError(msg) + return newTypeError newPyAscii(msg) if self.items.len != 0: self.items.setLen(0) if args.len == 1: @@ -117,7 +117,7 @@ template genSequenceMagics*(nameStr, else: return newObj - return newIndexTypeError(nameStr, other) + return newIndexTypeError(newPyStr nameStr, other) implNameMethod index(target: PyObject), mutRead: for idx, item in self.items: @@ -127,7 +127,7 @@ template genSequenceMagics*(nameStr, if retObj == pyTrueObj: return newPyInt(idx) let msg = fmt"{target} is not in " & nameStr - newValueError(msg) + newValueError(newPyStr msg) implNameMethod count(target: PyObject), mutRead: var count: int @@ -139,17 +139,17 @@ template genSequenceMagics*(nameStr, inc count newPyInt(count) -proc tupleSeqToString(ss: openArray[string]): string = +proc tupleSeqToString(ss: openArray[UnicodeVariant]): UnicodeVariant = ## one-element tuple must be out as "(1,)" - result = "(" + result = newUnicodeUnicodeVariant "(" case ss.len of 0: discard of 1: - result.add ss[0] - result.add ',' + result.unicodeStr.add ss[0].toRunes + result.unicodeStr.add ',' else: - result.add ss.join", " - result.add ')' + result.unicodeStr.add ss.joinAsRunes", " + result.unicodeStr.add ')' genSequenceMagics "tuple", implTupleMagic, implTupleMethod, diff --git a/Objects/typeobject.nim b/Objects/typeobject.nim index a826d71..7ec8d2d 100644 --- a/Objects/typeobject.nim +++ b/Objects/typeobject.nim @@ -37,7 +37,7 @@ implTypeGetter dict: newPyDictProxy(self.dict) implTypeSetter dict: - newTypeError(fmt"can't set attributes of built-in/extension type {self.name}") + newTypeError(newPyStr fmt"can't set attributes of built-in/extension type {self.name}") pyTypeObjectType.getsetDescr["__dict__"] = (tpGetter(Type, dict), tpSetter(Type, dict)) @@ -75,7 +75,7 @@ proc getAttr(self: PyObject, nameObj: PyObject): PyObject {. cdecl .} = if not nameObj.ofPyStrObject: let typeStr = nameObj.pyType.name let msg = fmt"attribute name must be string, not {typeStr}" - return newTypeError(msg) + return newTypeError(newPyStr msg) let name = PyStrObject(nameObj) let typeDict = self.getTypeDict if typeDict.isNil: @@ -95,14 +95,14 @@ proc getAttr(self: PyObject, nameObj: PyObject): PyObject {. cdecl .} = if not descr.isNil: return descr - return newAttributeError($self.pyType.name, $name) + return newAttributeError(self.pyType.name, $name) # generic getattr proc setAttr(self: PyObject, nameObj: PyObject, value: PyObject): PyObject {. cdecl .} = if not nameObj.ofPyStrObject: let typeStr = nameObj.pyType.name let msg = fmt"attribute name must be string, not {typeStr}" - return newTypeError(msg) + return newTypeError(newPyStr msg) let name = PyStrObject(nameObj) let typeDict = self.getTypeDict if typeDict.isNil: @@ -169,7 +169,7 @@ proc initTypeDict(tp: PyTypeObject) = # bltin methods for name, meth in tp.bltinMethods.pairs: - let namePyStr = newPyString(name) + let namePyStr = newPyAscii(name) d[namePyStr] = newPyMethodDescr(tp, meth, namePyStr) tp.dict = d @@ -192,7 +192,7 @@ implTypeMagic call: let newFunc = self.magicMethods.New if newFunc.isNil: let msg = fmt"cannot create '{self.name}' instances because __new__ is not set" - return newTypeError(msg) + return newTypeError(newPyStr msg) let newObj = newFunc(@[PyObject(self)] & args) if newObj.isThrownException: return newObj @@ -290,7 +290,7 @@ implTypeMagic New(metaType: PyTypeObject, name: PyStrObject, bases: PyTupleObject, dict: PyDictObject): assert metaType == pyTypeObjectType assert bases.len == 0 - let tp = newPyType(name.str) + let tp = newPyType($name.str) tp.kind = PyTypeToken.Type tp.magicMethods.New = tpMagic(Instance, new) updateSlots(tp, dict) diff --git a/Objects/warningobject.nim b/Objects/warningobject.nim index 82cd56e..6b5e28f 100644 --- a/Objects/warningobject.nim +++ b/Objects/warningobject.nim @@ -66,7 +66,7 @@ implWarningMessageMagic init: ]# template callReprOrStr(obj: PyObject, reprOrStr): string = - obj.getMagic(reprOrStr)(obj).PyStrObject.str + $obj.getMagic(reprOrStr)(obj).PyStrObject.str proc `$`*(self: PyWarningMessageObject): string = ("{message : $#, category : $#, filename : $#, lineno : $#, " & diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index fa04c39..ab13a89 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -7,13 +7,13 @@ import ../Utils/[utils, macroutils, compat] proc registerBltinFunction(name: string, fun: BltinFunc) = - let nameStr = newPyString(name) + let nameStr = newPyAscii(name) assert (not bltinDict.hasKey(nameStr)) bltinDict[nameStr] = newPyNimFunc(fun, nameStr) proc registerBltinObject(name: string, obj: PyObject) = - let nameStr = newPyString(name) + let nameStr = newPyAscii(name) assert (not bltinDict.hasKey(nameStr)) bltinDict[nameStr] = obj @@ -68,7 +68,7 @@ proc builtinPrint*(args: seq[PyObject]): PyObject {. cdecl .} = for obj in args: let objStr = obj.callMagic(str) errorIfNotString(objStr, "__str__") - echoCompat PyStrObject(objStr).str + echoCompat $PyStrObject(objStr).str pyNone registerBltinFunction("print", builtinPrint) diff --git a/Python/compile.nim b/Python/compile.nim index 5cb0ac8..2eda71a 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -601,7 +601,7 @@ compileMethod Assert: var ending = newBasicBlock() c.compile(astNode.test) c.addOp(newJumpInstr(OpCode.PopJumpIfTrue, ending, lineNo)) - c.addLoadOp(newPyString("AssertionError"), lineNo) + c.addLoadOp(newPyAscii("AssertionError"), lineNo) if not astNode.msg.isNil: c.compile(astNode.msg) c.addOp(newArgInstr(OpCode.CallFunction, 1, lineNo)) @@ -714,7 +714,7 @@ compileMethod ListComp: let body = newBasicBlock() let ending = newBasicBlock() c.addOp(newArgInstr(OpCode.BuildList, 0, lineNo)) - c.addLoadOp(newPyString(".0"), astNode.lineNo.value) # the implicit iterator argument + c.addLoadOp(newPyAscii(".0"), astNode.lineNo.value) # the implicit iterator argument c.addBlock(body) c.addOp(newJumpInstr(OpCode.ForIter, ending, lineNo)) c.compile(genNode.target) @@ -725,7 +725,7 @@ compileMethod ListComp: c.addBlock(ending) c.addOp(OpCode.ReturnValue, lineNo) - c.makeFunction(c.units.pop, newPyString("listcomp"), lineNo) + c.makeFunction(c.units.pop, newPyAscii("listcomp"), lineNo) # prepare the first arg of the function c.compile(genNode.iter) c.addOp(OpCode.GetIter, lineNo) diff --git a/Python/cpython.nim b/Python/cpython.nim index 6fa5f51..133ba1e 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -62,7 +62,7 @@ proc parseCompileEval*(input: string, lexer: Lexer, globals = prevF.globals else: globals = newPyDict() - let fun = newPyFunc(newPyString("Bla"), co, globals) + let fun = newPyFunc(newPyAscii("Bla"), co, globals) let f = newPyFrame(fun) var retObj = f.evalFrame if retObj.isThrownException: diff --git a/Python/neval.nim b/Python/neval.nim index 943bd2f..96fa5db 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -64,7 +64,7 @@ template doInplace(opName: untyped) = # PY-DIFF: not iadd, but += typ1{.inject.} = op1.pyType.name typ2{.inject.} = op2.pyType.name - nres = newTypeError( + nres = newTypeError newPyStr( &"unsupported operand type(s) for '{opStr}': '{typ1}' and '{typ2}'" ) sSetTop nres @@ -331,7 +331,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = template incompatibleLengthError(gotLen: int) = let got {. inject .} = $gotLen let msg = fmt"not enough values to unpack (expected {oparg}, got {got})" - let excp = newValueError(msg) + let excp = newValueError newPyStr(msg) handleException(excp) let s = sPop() if s.ofPyTupleObject(): @@ -464,7 +464,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = if not targetExcp.isExceptionType: let msg = "catching classes that do not inherit " & "from BaseException is not allowed" - handleException(newTypeError(msg)) + handleException(newTypeError newPyAscii(msg)) let currentExcp = PyExceptionObject(sTop()) sPush matchExcp(PyTypeObject(targetExcp), currentExcp) else: @@ -513,7 +513,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = obj = bltinDict[name] else: let msg = fmt"name '{name.str}' is not defined" - handleException(newNameError(msg)) + handleException(newNameError newPyStr(msg)) sPush obj of OpCode.SetupFinally: @@ -527,7 +527,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = if obj.isNil: let name = f.code.localVars[opArg] let msg = fmt"local variable {name} referenced before assignment" - let excp = newUnboundLocalError(msg) + let excp = newUnboundLocalError(newPyStr msg) handleException(excp) sPush obj @@ -538,7 +538,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = case opArg of 0: if (not hasTryBlock) or getTopBlock.context.isNil: - let excp = newRunTimeError("No active exception to reraise") + let excp = newRunTimeError(newPyAscii"No active exception to reraise") handleException(excp) else: handleException(getTopBlock.context) @@ -579,7 +579,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = let callFunc = funcObjNoCast.pyType.magicMethods.call if callFunc.isNil: let msg = fmt"{funcObjNoCast.pyType.name} is not callable" - retObj = newTypeError(msg) + retObj = newTypeError(newPyStr msg) else: retObj = callFunc(funcObjNoCast, args) if retObj.isThrownException: @@ -618,7 +618,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = if c.refObj.isNil: let name = f.code.cellVars[opArg] let msg = fmt"local variable {name} referenced before assignment" - let excp = newUnboundLocalError(msg) + let excp = newUnboundLocalError(newPyStr(msg)) handleException(excp) sPush c.refObj @@ -633,12 +633,12 @@ proc evalFrame*(f: PyFrameObject): PyObject = else: let msg = fmt"!!! NOT IMPLEMENTED OPCODE {opCode} IN EVAL FRAME !!!" - return newNotImplementedError(msg) # no need to handle + return newNotImplementedError(newPyAscii msg) # no need to handle except OutOfMemDefect: - excpObj = newMemoryError("Out of Memory") + excpObj = newMemoryError(newPyAscii"Out of Memory") handleException(excpObj) except InterruptError: - excpObj = newKeyboardInterruptError("") + excpObj = newKeyboardInterruptError(newPyAscii"Keyboard Interrupt") handleException(excpObj) # exception handler, return exception or re-enter the loop with new instruction index @@ -674,14 +674,14 @@ proc evalFrame*(f: PyFrameObject): PyObject = when defined(js): proc pyImport*(name: PyStrObject): PyObject = - newRunTimeError("Can't import in js mode") + newRunTimeError(newPyAscii"Can't import in js mode") else: import os proc pyImport*(name: PyStrObject): PyObject = - let filepath = pyConfig.path.joinPath(name.str).addFileExt("py") + let filepath = pyConfig.path.joinPath($name.str).addFileExt("py") if not filepath.fileExists: let msg = fmt"File {filepath} not found" - return newImportError(msg) + return newImportError(newPyStr(msg)) let input = readFile(filepath) let compileRes = compile(input, filepath) if compileRes.isThrownException: @@ -714,12 +714,12 @@ proc newPyFrame*(fun: PyFunctionObject, # handle wrong number of args if code.argScopes.len < args.len: let msg = fmt"{fun.name.str}() takes {code.argScopes.len} positional arguments but {args.len} were given" - return newTypeError(msg) + return newTypeError(newPyStr msg) elif args.len < code.argScopes.len: let diff = code.argScopes.len - args.len let msg = fmt"{fun.name.str}() missing {diff} required positional argument: " & fmt"{code.argNames[^diff..^1]}. {args.len} args are given." - return newTypeError(msg) + return newTypeError(newPyStr(msg)) let frame = newPyFrame() frame.back = back frame.code = code @@ -757,7 +757,7 @@ proc newPyFrame*(fun: PyFunctionObject, proc runCode*(co: PyCodeObject): PyObject = when defined(debug): echo co - let fun = newPyFunc(newPyString(""), co, newPyDict()) + let fun = newPyFunc(newPyAscii(""), co, newPyDict()) let f = newPyFrame(fun) f.evalFrame diff --git a/Python/symtable.nim b/Python/symtable.nim index 9a74375..d8a30cc 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -185,8 +185,8 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = let genNode = AstComprehension(gen) toVisitPerSte.add(genNode.target) # the iterator. Need to add here to let symbol table make room for the localVar - ste.addDeclaration(newPyString(".0")) - ste.argVars[newPyString(".0")] = 0 + ste.addDeclaration(newPyAscii(".0")) + ste.argVars[newPyAscii(".0")] = 0 else: unreachable @@ -251,7 +251,7 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = of AsdlStmtTk.Assert: let assertNode = AstAssert(astNode) - ste.addUsed(newPyString("AssertionError")) + ste.addUsed(newPyAscii("AssertionError")) visit assertNode.test visit assertNode.msg diff --git a/Python/traceback.nim b/Python/traceback.nim index d5cceb7..fe4af15 100644 --- a/Python/traceback.nim +++ b/Python/traceback.nim @@ -11,13 +11,13 @@ proc fmtTraceBack(tb: TraceBack): string = assert tb.fileName.ofPyStrObject # lineNo should starts from 1. 0 means not initialized properly assert tb.lineNo != 0 - let fileName = PyStrObject(tb.fileName).str + let fileName = $PyStrObject(tb.fileName).str var atWhere: string if tb.funName.isNil: atWhere = "" else: assert tb.funName.ofPyStrObject - atWhere = ", in " & PyStrObject(tb.funName).str + atWhere = $(newPyAscii", in " & PyStrObject(tb.funName)) result &= fmt(" File \"{fileName}\", line {tb.lineNo}{atWhere}\n") result &= " " & getSource(fileName, tb.lineNo).strip(chars={' '}) if tb.colNo != -1: @@ -32,7 +32,7 @@ proc printTb*(excp: PyExceptionObject) = singleExcpStrs.add "Traceback (most recent call last):" for tb in cur.traceBacks.reversed: singleExcpStrs.add tb.fmtTraceBack - singleExcpStrs.add PyStrObject(tpMagic(BaseError, repr)(cur)).str + singleExcpStrs.add $PyStrObject(tpMagic(BaseError, repr)(cur)).str excpStrs.add singleExcpStrs.join("\n") cur = cur.context let joinMsg = "\n\nDuring handling of the above exception, another exception occured\n\n" From 94931af87081b8f7fecc776102c4a751a33dcd1a Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 01:26:06 +0800 Subject: [PATCH 095/163] feat(Utils): rangeLen from nimpylib@aca732bcf09561a6ef3a6d4957af55ec854b5045 --- Utils/rangeLen.nim | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Utils/rangeLen.nim diff --git a/Utils/rangeLen.nim b/Utils/rangeLen.nim new file mode 100644 index 0000000..82e1ebb --- /dev/null +++ b/Utils/rangeLen.nim @@ -0,0 +1,11 @@ +# nimpylib src/pylib/builtins/private/mathutils.nim +# @aca732bcf09561a6ef3a6d4957af55ec854b5045 +func iPosCeil*[I: SomeInteger](x: float): I = + ## I(ceil(x)) if x > 0 else 0 + if x > 0: + let more = (x - float(I(x)) > 0) + I(x) + I(more) + else: I(0) + +func rangeLen*[I](start, stop, step: I): I = + iPosCeil[I]((stop - start) / step) From 30263eeda2d5d0937616e45c1909008bf44f727e Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 01:30:28 +0800 Subject: [PATCH 096/163] refine(slice): getSliceItems used ptr over var --- Objects/sliceobject.nim | 28 +++++++++++++++++++++------- Objects/tupleobject.nim | 2 +- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Objects/sliceobject.nim b/Objects/sliceobject.nim index 1a07115..01e79c4 100644 --- a/Objects/sliceobject.nim +++ b/Objects/sliceobject.nim @@ -1,6 +1,6 @@ import pyobject import baseBundle - +import ../Utils/rangeLen declarePyType Slice(tpToken): start: PyObject @@ -34,8 +34,22 @@ proc newPySlice*(start, stop, step: PyObject): PyObject = slice +proc calLen*(self: PySliceObject): int = + ## Get the length of the slice. + ## .. note:: python's slice has no `__len__`. + ## this is just a convenience method for internal use. + template intOrNone(obj: PyObject, defaultValue: int): int = + if obj.ofPyIntObject: + PyIntObject(obj).toInt + else: + defaultValue + rangeLen[int]( + intOrNone(self.start, 0), + intOrNone(self.stop, 0), + intOrNone(self.step, 1) + ) -proc getSliceItems*(slice: PySliceObject, src, dest: ptr seq[PyObject]): PyObject = +proc getSliceItems*[T](slice: PySliceObject, src: openArray[T], dest: var (seq[T]|string)): PyObject = var start, stop, step: int let stepObj = slice.step if stepObj.ofPyIntObject: @@ -47,27 +61,27 @@ proc getSliceItems*(slice: PySliceObject, src, dest: ptr seq[PyObject]): PyObjec template setIndex(name: untyped, defaultValue: int) = let `name Obj` = slice.`name` if `name Obj`.ofPyIntObject: - name = getIndex(PyIntObject(`name Obj`), src[].len) + name = getIndex(PyIntObject(`name Obj`), src.len) else: assert `name Obj`.ofPyNoneObject name = defaultValue var startDefault, stopDefault: int if 0 < step: startDefault = 0 - stopDefault = src[].len + stopDefault = src.len else: - startDefault = src[].len - 1 + startDefault = src.len - 1 stopDefault = -1 setIndex(start, startDefault) setIndex(stop, stopDefault) if 0 < step: while start < stop: - dest[].add(src[][start]) + dest.add(src[start]) start += step else: while stop < start: - dest[].add(src[][start]) + dest.add(src[start]) start += step pyNone diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index b0dde4d..aca705c 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -111,7 +111,7 @@ template genSequenceMagics*(nameStr, if other.ofPySliceObject: let slice = PySliceObject(other) let newObj = newPyNameSimple() - let retObj = slice.getSliceItems(self.items.addr, newObj.items.addr) + let retObj = slice.getSliceItems(self.items, newObj.items) if retObj.isThrownException: return retObj else: From e3bf5565e348cd7aaeef98fc30b7e1910493e2e6 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 01:31:16 +0800 Subject: [PATCH 097/163] fix(py): [1][0:1] raised IndexError --- Objects/numobjects.nim | 4 ++-- Objects/sliceobject.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index 2b614d9..81aa8af 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -1044,12 +1044,12 @@ implFloatMagic hash: # used in list and tuple -template getIndex*(obj: PyIntObject, size: int): int = +template getIndex*(obj: PyIntObject, size: int, sizeOpIdx: untyped = `<=`): int = # todo: if overflow, then thrown indexerror var idx = obj.toInt if idx < 0: idx = size + idx - if (idx < 0) or (size <= idx): + if (idx < 0) or (sizeOpIdx(size, idx)): let msg = "index out of range. idx: " & $idx & ", len: " & $size return newIndexError newPyAscii(msg) idx diff --git a/Objects/sliceobject.nim b/Objects/sliceobject.nim index 01e79c4..6236e6e 100644 --- a/Objects/sliceobject.nim +++ b/Objects/sliceobject.nim @@ -61,7 +61,7 @@ proc getSliceItems*[T](slice: PySliceObject, src: openArray[T], dest: var (seq[T template setIndex(name: untyped, defaultValue: int) = let `name Obj` = slice.`name` if `name Obj`.ofPyIntObject: - name = getIndex(PyIntObject(`name Obj`), src.len) + name = getIndex(PyIntObject(`name Obj`), src.len, `<`) else: assert `name Obj`.ofPyNoneObject name = defaultValue From 00393460da2f3048579ff73cdfdc2e785674c332 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 03:23:01 +0800 Subject: [PATCH 098/163] feat(str): sequence methods and .find --- Objects/stringobject.nim | 64 +++++++++++++------- Objects/stringobjectImpl.nim | 109 +++++++++++++++++++++++++++++++++++ Utils/sequtils.nim | 48 +++++++++++++++ 3 files changed, 199 insertions(+), 22 deletions(-) create mode 100644 Utils/sequtils.nim diff --git a/Objects/stringobject.nim b/Objects/stringobject.nim index 08c8782..caf6cb2 100644 --- a/Objects/stringobject.nim +++ b/Objects/stringobject.nim @@ -60,16 +60,41 @@ template cat[A, B](a: openArray[A], b: openArray[B]): seq[Rune] = s.add(b) s +macro doKindsWith2It*(self, other: UnicodeVariant, do0_3): untyped = + result = nnkCaseStmt.newTree(quote do: + ((`self`.ascii.uint8 shl 1) or `other`.ascii.uint8) + ) + do0_3.expectLen 4 + template alias(name, val): NimNode = + newProc(name, @[ident"untyped"], val, procType=nnkTemplateDef) + template ofI(i: int, attr1, attr2) = + let iNode = newIntLitNode(i) + let doSth = do0_3[i] + var ofStmt = nnkOfBranch.newTree iNode + var blk = newStmtList() + blk.add alias(ident"it1", self.newDotExpr(attr1)) + blk.add alias(ident"it2", other.newDotExpr(attr2)) + blk.add doSth + ofStmt.add blk + result.add ofStmt + let + u = ident"unicodeStr" + a = ident"asciiStr" + ofI 0, u, u + ofI 1, u, a + ofI 2, a, u + ofI 3, a, a + result.add nnkElse.newTree( + quote do: raiseAssert"unreachable" + ) + + proc `&`*(a, b: UnicodeVariant): UnicodeVariant = - case (a.ascii.uint8 shl 1) or b.ascii.uint8 - of 0: - newUnicodeUnicodeVariant(a.unicodeStr & b.unicodeStr) - of 1: - newUnicodeUnicodeVariant(cat(a.unicodeStr, b.asciiStr)) - of 2: - newUnicodeUnicodeVariant(cat(a.asciiStr, b.unicodeStr)) - else: # of 3 - newAsciiUnicodeVariant(a.asciiStr & b.asciiStr) + doKindsWith2It(a, b): + newUnicodeUnicodeVariant(it1 & it2) + newUnicodeUnicodeVariant(cat(it1, it2)) + newUnicodeUnicodeVariant(cat(it1, it2)) + newAsciiUnicodeVariant(it1 & it2) proc toRunes*(self: UnicodeVariant): seq[Rune] = if self.ascii: self.asciiStr.toRunes @@ -117,22 +142,17 @@ proc hash*(self: UnicodeVariant): Hash {. inline, cdecl .} = proc `==`*(self, other: UnicodeVariant): bool {. inline, cdecl .} = template cmpAttr(a, b) = - if self.a.len > other.b.len: return false - for i, c in self.a: - if uint32(c) != uint32(other.b[i]): + if a.len > b.len: return false + for i, c in a: + if uint32(c) != uint32(b[i]): return false return true - case ((self.ascii.uint8 shl 1) or other.ascii.uint8) - of 0: - return self.unicodeStr == other.unicodeStr - of 1: - cmpAttr(unicodeStr, asciiStr) - of 2: - cmpAttr(asciiStr, unicodeStr) - else: # of 3 - return self.asciiStr == other.asciiStr - + doKindsWith2It(self, other): + return it1 == it2 + cmpAttr(it1, it2) + cmpAttr(it1, it2) + return it1 == it2 declarePyType Str(tpToken): str: UnicodeVariant diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index a06c3a0..994c444 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -3,6 +3,8 @@ import std/strformat import pyobject import baseBundle import stringobject +import ./sliceobject +import ../Utils/sequtils export stringobject @@ -48,3 +50,110 @@ implStrMagic New(tp: PyObject, obj: PyObject): implStrMagic add(i: PyStrObject): self & i + +template itemsAt(self: PyStrObject, i: int): PyStrObject = + # this is used in `getitem` magic + #{.push boundChecks: off.} + if self.str.ascii: + newPyString self.str.asciiStr[i] + else: + newPyString self.str.unicodeStr[i] + #{.pop.} +type Getter = proc(i: int): PyStrObject +declarePyType StrIter(): + ascii: bool + len: int + items: UnicodeVariant + idx: int + getItem: Getter + +implStrIterMagic iter: + self + +implStrIterMagic iternext: + if self.idx == self.len: + return newStopIterError() + result = self.getItem self.idx + inc self.idx + + +proc newPyStrIter*(s: PyStrObject): PyStrIterObject = + result = newPyStrIterSimple() + result.items = s.str + result.ascii = s.str.ascii + result.len = s.len + result.getItem = if result.ascii: + proc(i: int): PyStrObject = newPyString s.str.asciiStr[i] + else: + proc(i: int): PyStrObject = newPyString s.str.unicodeStr[i] + +template findExpanded(it1, it2): int = it1.findWithoutMem(it2, key=uint32) +template findAllExpanded(it1, it2): untyped = it1.findAllWithoutMem(it2, key=uint32) + +when true: + # copied and modified from ./tupleobject.nim + implStrMagic iter: + newPyStrIter(self) + + implStrMagic contains(o: PyStrObject): + let res = doKindsWith2It(self.str, o.str): + it1.contains it2 + it1.findExpanded(it2) > 0 + it1.findExpanded(it2) > 0 + it1.contains it2 + if res: pyTrueObj + else: pyFalseObj + + + implStrMagic getitem: + if other.ofPyIntObject: + let idx = getIndex(PyIntObject(other), self.len) + return self.itemsAt idx + if other.ofPySliceObject: + let slice = PySliceObject(other) + template tgetSliceItems(attr): untyped = + slice.getSliceItems(self.str.attr, newObj.str.attr) + var + newObj: PyStrObject + retObj: PyObject + if self.str.ascii: + newObj = newPyString newAsciiUnicodeVariantOfCap slice.calLen + retObj = tgetSliceItems(asciiStr) + else: + newObj = newPyString newUnicodeUnicodeVariantOfCap slice.calLen + retObj = tgetSliceItems(unicodeStr) + if retObj.isThrownException: + return retObj + else: + return newObj + + + implStrMethod index(target: PyStrObject): + let res = doKindsWith2It(self.str, target.str): + sequtils.find(it1, it2) + it1.findExpanded(it2) + it1.findExpanded(it2) + sequtils.find(it1, it2) + if res >= 0: + return newPyInt(res) + let msg = "substring not found" + newValueError(newPyAscii msg) + + implStrMethod count(target: PyStrObject): + var count: int + template cntAll(it) = + for _ in it: count.inc + doKindsWith2It(self.str, target.str): + cntAll it1.findAll(it2) + cntAll it1.findAllExpanded(it2) + cntAll it1.findAllExpanded(it2) + cntAll it1.findAll(it2) + newPyInt(count) + +implStrMethod find(target: PyStrObject): + let res = doKindsWith2It(self.str, target.str): + sequtils.find(it1, it2) + it1.findExpanded(it2) + it1.findExpanded(it2) + sequtils.find(it1, it2) + newPyInt(res) diff --git a/Utils/sequtils.nim b/Utils/sequtils.nim new file mode 100644 index 0000000..0c5d8de --- /dev/null +++ b/Utils/sequtils.nim @@ -0,0 +1,48 @@ + +# TODO: KMP + +iterator findAllWithoutMem*[A, B](self: openArray[A], sub: openArray[B], + start=0, key: typedesc): int = + for i in start..(self.len - sub.len): + var j = 0 + {.push hint[ConvFromXtoItselfNotNeeded]: off.} + while j < sub.len and self[i + j].key == sub[j].key: + j += 1 + {.pop.} + if j == sub.len: + yield i + +proc findWithoutMem*[A, B](self: openArray[A], sub: openArray[B], + start=0, key: typedesc): int = + result = -1 + for i in findAllWithoutMem(self, sub, start, key): + return i + +iterator findAll*[T](s, sub: openArray[T], start=0): int = + when declared(memmem): + var start = start + const N = sizeof(T) + let subLen = sub.len + if subLen != 0: + while start < s.len: + let found = memmem(s[start].addr, csize_t(s.len - start)*N, sub[0].addr, csize_t(subLen)*N) + if found.isNil: + break + else: + let n = (cast[int](found) -% cast[int](s[start].addr)) div N + start += n + yield n + else: + for i in findAllWithoutMem(s, sub, start, T): + yield i + + +proc find*[T](s, sub: openArray[T], start=0): int = + when declared(memmem): + result = -1 + for i in findAll(s, sub, start): + return i + else: + return findWithoutMem(s, sub, start, T) + +proc contains*[T](s, sub: openArray[T]): bool{.inline.} = s.find(sub) > 0 From ea5a2cd905913eb128eb4d33df0fbb0a3dbe8540 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 05:38:01 +0800 Subject: [PATCH 099/163] feat(syntax): DeleteSubScr op (`__delitem__`) --- Objects/listobject.nim | 9 +++++++++ Objects/pyobjectBase.nim | 1 + Python/ast.nim | 22 +++++++++++++++++++--- Python/compile.nim | 10 ++++++++++ Python/neval.nim | 4 ++++ Python/symtable.nim | 4 ++++ 6 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 6a78a77..cd0abea 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -37,6 +37,15 @@ implListMagic setitem, [mutable: write]: return newTypeError newPyAscii("store to slice not implemented") return newIndexTypeError(newPyAscii"list", arg1) +implListMagic delitem, [mutable: write]: + if other.ofPyIntObject: + let idx = getIndex(PyIntObject(other), self.items.len) + self.items.delete idx + return pyNone + if other.ofPySliceObject: + return newTypeError newPyAscii("delete slice not implemented") + return newIndexTypeError(newPyAscii"list", other) + implListMethod append(item: PyObject), [mutable: write]: self.items.add(item) diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index d95a1b6..2493558 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -107,6 +107,7 @@ type # subscription getitem: BinaryMethod setitem: TernaryMethod + delitem: BinaryMethod # descriptor protocol # what to do when getting or setting attributes of its intances diff --git a/Python/ast.nim b/Python/ast.nim index a4493e0..6cafe9d 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -264,6 +264,13 @@ method setStore(astNode: AstTuple) = for elm in astNode.elts: elm.setStore() +method setDelete(astNode: AstNodeBase) {.base.} = + if not (astNode of AsdlExpr): + unreachable + raiseSyntaxError("can't delete", AsdlExpr(astNode)) +method setDelete(astNode: AstSubscript) = + astnode.ctx = newAstDel() + # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE ast single_input, [AstInteractive]: result = newAstInteractive() @@ -539,9 +546,18 @@ Vbarequal Circumflexequal ]# -proc astDelStmt(parseNode: ParseNode): AsdlStmt = - raiseSyntaxError("del not implemented") - +# del_stmt: 'del' exprlist +ast del_stmt, [AsdlStmt]: + var node = newAstDelete() + setNo(node, parseNode.children[0]) + let exprlist = parseNode.children[1] + + let ls = astExprList(exprlist) + setDelete ls + node.targets.add(ls) + node + + # pass_stmt: 'pass' ast pass_stmt, [AstPass]: result = newAstPass() diff --git a/Python/compile.nim b/Python/compile.nim index 2eda71a..6dfa35e 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -488,6 +488,10 @@ compileMethod AugAssign: let opCode = astNode.op.toInplaceOpCode c.addOp(newInstr(opCode, astNode.lineNo.value)) +compileMethod Delete: + for i in astNode.targets: + c.compile(i) + compileMethod For: assert astNode.orelse.len == 0 let start = newBasicBlock(BlockType.For) @@ -789,6 +793,10 @@ compileMethod Subscript: c.compile(astNode.value) c.compile(astNode.slice) c.addOp(OpCode.StoreSubscr, lineNo) + elif astNode.ctx of AstDel: + c.compile(astNode.value) + c.compile(astNode.slice) + c.addOp(OpCode.DeleteSubscr, lineNo) else: unreachable @@ -803,6 +811,8 @@ compileMethod Name: c.addLoadOp(astNode.id, lineNo) elif astNode.ctx of AstStore: c.addStoreOp(astNode.id, lineNo) + #elif astNode.ctx of AstDel: + # c.addOp(newArgInstr(OpCode.DeleteName, astNode.id, lineNo)) else: unreachable # no other context implemented diff --git a/Python/neval.nim b/Python/neval.nim index 96fa5db..946cf06 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -257,6 +257,10 @@ proc evalFrame*(f: PyFrameObject): PyObject = let obj = sPop() let value = sPop() discard obj.callMagic(setitem, idx, value, handleExcp=true) + of OpCode.DeleteSubscr: + let idx = sPop() + let obj = sPop() + discard obj.callMagic(delitem, idx, handleExcp=true) of OpCode.BinarySubscr: doBinary(getitem) diff --git a/Python/symtable.nim b/Python/symtable.nim index d8a30cc..175fd96 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -223,6 +223,10 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = visit binOpNode.target visit binOpNode.value + of AsdlStmtTk.Delete: + let binOpNode = AstDelete(astNode) + visitSeq binOpNode.targets + of AsdlStmtTk.For: let forNode = AstFor(astNode) if not (forNode.target.kind == AsdlExprTk.Name): From dc67a3e7f48190e4852d5d6ac707fef817543d8d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 06:44:57 +0800 Subject: [PATCH 100/163] feat(list): setitem,deliem accept slice --- Objects/listobject.nim | 38 +++++++++++++++++++++++++++++++++----- Objects/sliceobject.nim | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index cd0abea..9d607c5 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -1,4 +1,4 @@ - +import std/sequtils import strformat import strutils @@ -28,22 +28,50 @@ genSequenceMagics "list", newPyListSimple, [mutable: read], [reprLockWithMsg"[...]", mutable: read], lsSeqToStr +proc len*(self: PyListObject): int{.inline.} = self.items.len + implListMagic setitem, [mutable: write]: if arg1.ofPyIntObject: - let idx = getIndex(PyIntObject(arg1), self.items.len) + let idx = getIndex(PyIntObject(arg1), self.len) self.items[idx] = arg2 return pyNone if arg1.ofPySliceObject: - return newTypeError newPyAscii("store to slice not implemented") + let slice = arg1.PySliceObject + let iterableToLoop = arg2 + case slice.stepAsInt + of 1, -1: + var ls: seq[PyObject] + pyForIn it, iterableToLoop: + ls.add it + self.items[slice.toNimSlice(self.len)] = ls + else: + let (iterable, nextMethod) = getIterableWithCheck(iterableToLoop) + if iterable.isThrownException: + return iterable + for i in iterInt(slice, self.len): + let it = nextMethod(iterable) + if it.isStopIter: + break + if it.isThrownException: + return it + self.items[i] = it + return pyNone return newIndexTypeError(newPyAscii"list", arg1) implListMagic delitem, [mutable: write]: if other.ofPyIntObject: - let idx = getIndex(PyIntObject(other), self.items.len) + let idx = getIndex(PyIntObject(other), self.len) self.items.delete idx return pyNone if other.ofPySliceObject: - return newTypeError newPyAscii("delete slice not implemented") + let slice = PySliceObject(other) + case slice.stepAsInt: + of 1, -1: + self.items.delete slice.toNimSlice(self.len) + else: + for i in iterInt(slice, self.len): + self.items.delete i + return pyNone return newIndexTypeError(newPyAscii"list", other) diff --git a/Objects/sliceobject.nim b/Objects/sliceobject.nim index 6236e6e..a9544b8 100644 --- a/Objects/sliceobject.nim +++ b/Objects/sliceobject.nim @@ -34,6 +34,41 @@ proc newPySlice*(start, stop, step: PyObject): PyObject = slice +template I(obj: PySliceObject; attr; defVal: int): int = + if obj.attr == pyNone: defVal + else: + obj.attr.PyIntObject.toInt # TODO: overflow +template CI(obj: PySliceObject; attr; defVal, size: int): int = + var res = obj.I(attr, defVal) + if res < 0: res.inc size + res + +proc stepAsInt*(slice: PySliceObject): int = slice.I(step, 1) +proc stopAsInt*(slice: PySliceObject, size: int): int = slice.CI(stop, size, size) +proc startAsInt*(slice: PySliceObject, size: int): int = slice.CI(start, 0, size) + +proc toNimSlice*(sliceStep1: PySliceObject, size: int): Slice[int] = + let step = sliceStep1.stepAsInt + let n1 = step < 0 + let + stop = sliceStep1.stopAsInt size + start = sliceStep1.startAsInt size + assert step.abs == 1 + if n1: stop+1 .. start + else: start .. stop-1 + +iterator iterInt*(slice: PySliceObject, size: int): int = + let + step = slice.stepAsInt + start = slice.startAsInt size + stop = slice.stopAsInt size + let neg = step < 0 + if neg: + for i in countdown(start, stop+1, step): yield i + else: + for i in countup(start, stop-1, step): yield i + + proc calLen*(self: PySliceObject): int = ## Get the length of the slice. ## .. note:: python's slice has no `__len__`. From c8964e01f14be9dc82814fac4afeb93ea79afc7a Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 07:23:17 +0800 Subject: [PATCH 101/163] fix(None): `type(None)() is not None`; no `__repr__` (followup 511e0f2) --- Objects/noneobject.nim | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Objects/noneobject.nim b/Objects/noneobject.nim index 0caf2f6..05623b5 100644 --- a/Objects/noneobject.nim +++ b/Objects/noneobject.nim @@ -1,13 +1,18 @@ import pyobject -import boolobject +import ./stringobject +import ./exceptions declarePyType None(tpToken): discard -let pyNone* = newPyNoneSimple() +let pyNone* = newPyNoneSimple() ## singleton -implNoneMagic eq: - if other.ofPyNoneObject: - return pyTrueObj - else: - return pyFalseObj +proc isPyNone*(o: PyObject): bool = o == pyNone + +const sNone = "None" +method `$`*(_: PyNoneObject): string = sNone + +implNoneMagic repr: newPyAscii sNone + +implNoneMagic New(tp: PyObject): + return pyNone From 4aea40ef56722c12dc8daf11eaea0195e4d1172e Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 07:26:49 +0800 Subject: [PATCH 102/163] feat(slice): .indices --- Objects/listobject.nim | 2 +- Objects/sliceobject.nim | 35 +---------------- Objects/sliceobjectImpl.nim | 76 +++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 35 deletions(-) create mode 100644 Objects/sliceobjectImpl.nim diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 9d607c5..a7585ab 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -4,7 +4,7 @@ import strutils import pyobject import baseBundle -import sliceobject +import ./sliceobjectImpl import iterobject import ./tupleobject import ../Utils/[utils, compat] diff --git a/Objects/sliceobject.nim b/Objects/sliceobject.nim index a9544b8..7a70684 100644 --- a/Objects/sliceobject.nim +++ b/Objects/sliceobject.nim @@ -34,40 +34,7 @@ proc newPySlice*(start, stop, step: PyObject): PyObject = slice -template I(obj: PySliceObject; attr; defVal: int): int = - if obj.attr == pyNone: defVal - else: - obj.attr.PyIntObject.toInt # TODO: overflow -template CI(obj: PySliceObject; attr; defVal, size: int): int = - var res = obj.I(attr, defVal) - if res < 0: res.inc size - res - -proc stepAsInt*(slice: PySliceObject): int = slice.I(step, 1) -proc stopAsInt*(slice: PySliceObject, size: int): int = slice.CI(stop, size, size) -proc startAsInt*(slice: PySliceObject, size: int): int = slice.CI(start, 0, size) - -proc toNimSlice*(sliceStep1: PySliceObject, size: int): Slice[int] = - let step = sliceStep1.stepAsInt - let n1 = step < 0 - let - stop = sliceStep1.stopAsInt size - start = sliceStep1.startAsInt size - assert step.abs == 1 - if n1: stop+1 .. start - else: start .. stop-1 - -iterator iterInt*(slice: PySliceObject, size: int): int = - let - step = slice.stepAsInt - start = slice.startAsInt size - stop = slice.stopAsInt size - let neg = step < 0 - if neg: - for i in countdown(start, stop+1, step): yield i - else: - for i in countup(start, stop-1, step): yield i - +# slice.indices defined in ./sliceobjectImpl proc calLen*(self: PySliceObject): int = ## Get the length of the slice. diff --git a/Objects/sliceobjectImpl.nim b/Objects/sliceobjectImpl.nim new file mode 100644 index 0000000..99a3cc5 --- /dev/null +++ b/Objects/sliceobjectImpl.nim @@ -0,0 +1,76 @@ + +import std/strformat +import ./pyobject +import ./tupleobject +import ./[numobjects, noneobject] +import ./[exceptions, stringobject] +import ./sliceobject +export sliceobject + +template longOr[T](a, b): T = + when T is PyIntObject: a + else: b + +template I[T](obj: PySliceObject; attr; defVal: T): T = + if obj.attr == pyNone: defVal + else: + let res = obj.attr.PyIntObject + longOr[T](res, res.toInt) # TODO: overflow + +template negative(i: int): bool = i < 0 + +template CI[T](obj: PySliceObject; attr; defVal, size: T): T = + var res = obj.I(attr, defVal) + if res.negative: res = res + size + res + +proc stepAsLong*(slice: PySliceObject): PyIntObject = slice.I(step, pyIntOne) +proc stepAsInt*(slice: PySliceObject): int = slice.I(step, 1) +proc stopAsInt*[I: PyIntObject|int](slice: PySliceObject, size: I): I = slice.CI(stop, size, size) +proc startAsInt*[I: PyIntObject|int](slice: PySliceObject, size: I): I = + slice.CI(start, longOr[I](pyIntZero, 0), size) + +proc indices*(slice: PySliceObject, size: int): tuple[start, stop, stride: int] = + let + step = slice.stepAsInt + stop = slice.stopAsInt size + start = slice.startAsInt size + (start, stop, step) + +# redeclare this for these are "private" macros + +methodMacroTmpl(Slice) + +implSliceMethod indices(size: PyIntObject): + let + step = self.stepAsLong + stop = self.stopAsInt size + start = self.startAsInt size + newPyTuple(@[start.PyObject, stop, step]) + +proc dollar(self: PySliceObject): string = + &"slice({$self.start}, {$self.stop}, {$self.step})" + +method `$`*(self: PySliceObject): string = self.dollar + +implSliceMagic repr: + newPyAscii self.dollar + + +proc toNimSlice*(sliceStep1: PySliceObject, size: int): Slice[int] = + let (start, stop, step) = sliceStep1.indices size + let n1 = step < 0 + assert step.abs == 1 + if n1: stop+1 .. start + else: start .. stop-1 + +iterator iterInt*(slice: PySliceObject, size: int): int = + let + step = slice.stepAsInt + start = slice.startAsInt size + stop = slice.stopAsInt size + let neg = step < 0 + if neg: + for i in countdown(start, stop+1, step): yield i + else: + for i in countup(start, stop-1, step): yield i From 4866a19c8bcc302037f6c4a8e2d22ec5cbfbf364 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 07:27:05 +0800 Subject: [PATCH 103/163] feat(builtins): slice --- Python/bltinmodule.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index ab13a89..dd8e2e1 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -2,7 +2,8 @@ import strformat {.used.} # this module contains toplevel code, so never `importButNotUsed` import neval import builtindict -import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject, notimplementedobject] +import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject, + notimplementedobject, sliceobjectImpl] import ../Utils/[utils, macroutils, compat] @@ -111,6 +112,7 @@ registerBltinObject("Ellipsis", pyEllipsis) registerBltinObject("None", pyNone) registerBltinObject("type", pyTypeObjectType) registerBltinObject("range", pyRangeObjectType) +registerBltinObject("slice", pySliceObjectType) registerBltinObject("list", pyListObjectType) registerBltinObject("tuple", pyTupleObjectType) registerBltinObject("dict", pyDictObjectType) From 045eb6e64893b389a706146ba67c13a72374bf50 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 08:31:20 +0800 Subject: [PATCH 104/163] fix(bool): type(bool)() is not None (followup 511e0f2); bool [] is True --- Objects/boolobject.nim | 10 ++++++--- Objects/boolobjectImpl.nim | 46 +++++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/Objects/boolobject.nim b/Objects/boolobject.nim index 98236df..ebaf742 100644 --- a/Objects/boolobject.nim +++ b/Objects/boolobject.nim @@ -3,10 +3,14 @@ import pyobject declarePyType Bool(tpToken): b: bool -proc newPyBool(b: bool): PyBoolObject = +proc newPyBoolInner(b: bool): PyBoolObject = result = newPyBoolSimple() result.b = b -let pyTrueObj* = newPyBool(true) -let pyFalseObj* = newPyBool(false) +let pyTrueObj* = newPyBoolInner(true) ## singleton +let pyFalseObj* = newPyBoolInner(false) ## singleton + +proc newPyBool*(b: bool): PyBoolObject = + if b: pyTrueObj + else: pyFalseObj diff --git a/Objects/boolobjectImpl.nim b/Objects/boolobjectImpl.nim index 24ef8e4..b421800 100644 --- a/Objects/boolobjectImpl.nim +++ b/Objects/boolobjectImpl.nim @@ -7,6 +7,7 @@ import exceptions import stringobject import boolobject import numobjects +import ./noneobject export boolobject @@ -16,11 +17,7 @@ method `$`*(obj: PyBoolObject): string = methodMacroTmpl(Bool) implBoolMagic Not: - if self == pyTrueObj: - pyFalseObj - else: - pyTrueObj - + newPyBool self != pyTrueObj implBoolMagic bool: self @@ -29,35 +26,23 @@ implBoolMagic bool: implBoolMagic And: let otherBoolObj = other.callMagic(bool) errorIfNotBool(otherBoolObj, "__bool__") - if self.b and PyBoolObject(otherBoolObj).b: - return pyTrueObj - else: - return pyFalseObj + newPyBool self.b and PyBoolObject(otherBoolObj).b implBoolMagic Xor: let otherBoolObj = other.callMagic(bool) errorIfNotBool(otherBoolObj, "__bool__") - if self.b xor PyBoolObject(otherBoolObj).b: - return pyTrueObj - else: - return pyFalseObj + newPyBool self.b xor PyBoolObject(otherBoolObj).b implBoolMagic Or: let otherBoolObj = other.callMagic(bool) errorIfNotBool(otherBoolObj, "__bool__") - if self.b or PyBoolObject(otherBoolObj).b: - return pyTrueObj - else: - return pyFalseObj + newPyBool self.b or PyBoolObject(otherBoolObj).b implBoolMagic eq: let otherBoolObj = other.callMagic(bool) errorIfNotBool(otherBoolObj, "__bool__") let otherBool = PyBoolObject(otherBoolObj).b - if self.b == otherBool: - return pyTrueObj - else: - return pyFalseObj + newPyBool self.b == otherBool implBoolMagic repr: if self.b: @@ -67,3 +52,22 @@ implBoolMagic repr: implBoolMagic hash: newPyInt(Hash(self.b)) + + +proc PyObject_IsTrue*(v: PyObject): bool = + if v == pyTrueObj: return true + if v == pyFalseObj: return false + if v == pyNone: return false + let boolMag = v.getMagic(bool) + if not boolMag.isNil: + return boolMag(v).PyBoolObject.b + elif not v.getMagic(len).isNil: + return v.getMagic(len)(v).PyIntObject.positive + # We currently don't define: + # as_sequence + # as_mapping + return true + +implBoolMagic New(tp: PyObject, obj: PyObject): + newPyBool PyObject_IsTrue obj + From 33ecc77353d210da5e59fd09a4ae192d83cf29b1 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 08:35:49 +0800 Subject: [PATCH 105/163] feat(builtins): bool; fix(py): neval regarded user-define instance as false --- Python/bltinmodule.nim | 1 + Python/neval.nim | 10 ++-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index dd8e2e1..9d072da 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -108,6 +108,7 @@ implBltinFunc buildClass(funcObj: PyFunctionObject, name: PyStrObject), "__build registerBltinObject("NotImplemented", pyNotImplemented) +registerBltinObject("bool", pyBoolObjectType) registerBltinObject("Ellipsis", pyEllipsis) registerBltinObject("None", pyNone) registerBltinObject("type", pyTypeObjectType) diff --git a/Python/neval.nim b/Python/neval.nim index 946cf06..2d86dbb 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -10,7 +10,7 @@ import builtindict import traceback import ../Objects/[pyobject, baseBundle, tupleobject, listobject, dictobject, sliceobject, codeobject, frameobject, funcobject, cellobject, - setobject, notimplementedobject, + setobject, notimplementedobject, boolobjectImpl, exceptionsImpl, moduleobject, methodobject] import ../Utils/utils @@ -102,13 +102,7 @@ template doBinaryContain: PyObject = # "fast" because check if it's a bool object first and save the callMagic(bool) template getBoolFast(obj: PyObject): bool = - var ret: bool - if obj.ofPyBoolObject: - ret = PyBoolObject(obj).b - # if user defined class tried to return non bool, - # the magic method will return an exception - let boolObj = top.callMagic(bool, handleExcp=true) - PyBoolObject(boolObj).b + PyObject_IsTrue(obj) # if declared as a local variable, js target will fail. See gh-10651 when defined(js): From ca49d9790afdbb34d036362c132cab5f33435d5d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 12:49:35 +0800 Subject: [PATCH 106/163] fixup! feat(warnings): nim api: warnExplicit (not complete, a simpler impl) Utils was spelt as utils --- Python/warnings.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/warnings.nim b/Python/warnings.nim index db6e01f..101607f 100644 --- a/Python/warnings.nim +++ b/Python/warnings.nim @@ -1,7 +1,7 @@ ## `_warnings` import std/strformat -import ../utils/compat +import ../Utils/compat import ../Objects/[warningobject, stringobject, numobjects, pyobjectBase] export warningobject From 4b28a22cd42859dc4c53c7c059eddadcc9a56aff Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 14:10:26 +0800 Subject: [PATCH 107/163] feat(syntax): multi-targets in del_stmt (e.g. `del ls[0], ls[1]`) followup ea5a2cd905913eb128eb4d33df0fbb0a3dbe8540 --- Python/ast.nim | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index 6cafe9d..72eced4 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -151,7 +151,7 @@ proc astTestlistComp(parseNode: ParseNode): seq[AsdlExpr] proc astTrailer(parseNode: ParseNode, leftExpr: AsdlExpr): AsdlExpr proc astSubscriptlist(parseNode: ParseNode): AsdlSlice proc astSubscript(parseNode: ParseNode): AsdlSlice -proc astExprList(parseNode: ParseNode): AsdlExpr +proc astExprList(parseNode: ParseNode): seq[AsdlExpr] proc astTestList(parseNode: ParseNode): AsdlExpr proc astDictOrSetMaker(parseNode: ParseNode): AsdlExpr proc astClassDef(parseNode: ParseNode): AstClassDef @@ -552,9 +552,9 @@ ast del_stmt, [AsdlStmt]: setNo(node, parseNode.children[0]) let exprlist = parseNode.children[1] - let ls = astExprList(exprlist) - setDelete ls - node.targets.add(ls) + for i in astExprList(exprlist): + node.targets.add(i) + setDelete i node @@ -715,14 +715,23 @@ ast while_stmt, [AstWhile]: if not (parseNode.children.len == 4): raiseSyntaxError("Else clause in while not implemented", parseNode.children[4]) +proc astExprListInFor(e: var AsdlExpr, node: ParseNode) = + if not (node.children.len == 1): + raiseSyntaxError("unpacking in for loop not implemented", node) + let child = node.children[0] + if not (child.tokenNode.token == Token.expr): + raiseSyntaxError("unpacking in for loop not implemented", child) + e = astExprList(node)[0] + e.setStore + # e=newTuple(astExprList(node)) + # for_stmt 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] ast for_stmt, [AsdlStmt]: if not (parseNode.children.len == 6): raiseSyntaxError("for with else not implemented", parseNode.children[6]) let forNode = newAstFor() setNo(forNode, parseNode.children[0]) - forNode.target = astExprList(parseNode.children[1]) - forNode.target.setStore + forNode.target.astExprListInFor(parseNode.children[1]) forNode.iter = astTestlist(parseNode.children[3]) forNode.body = astSuite(parseNode.children[5]) result = forNode @@ -1184,15 +1193,12 @@ ast subscript, [AsdlSlice]: # exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] -# currently only used in `for` stmt, so assume only one child -ast exprlist, [AsdlExpr]: - if not (parseNode.children.len == 1): - raiseSyntaxError("unpacking in for loop not implemented", parseNode.children[1]) - let child = parseNode.children[0] - if not (child.tokenNode.token == Token.expr): - raiseSyntaxError("unpacking in for loop not implemented", child) - astExpr(child) - +ast exprlist, [seq[AsdlExpr]]: + ## current `ast star_expr` not impl + for i in 0..<((parseNode.children.len + 1) div 2): + let child = parseNode.children[2*i] + result.add astExpr(child) + # testlist: test (',' test)* [','] ast testlist, [AsdlExpr]: var elms: seq[AsdlExpr] @@ -1284,8 +1290,7 @@ ast sync_comp_for, [seq[AsdlComprehension]]: if parseNode.children.len == 5: raiseSyntaxError("Complex comprehension not implemented", parseNode.children[5]) let comp = newAstComprehension() - comp.target = astExprList(parseNode.children[1]) - comp.target.setStore() + comp.target.astExprListInFor(parseNode.children[1]) comp.iter = astOrTest(parseNode.children[3]) result.add comp From 2be4169eb558508c0cad29026c813fce3a4353ae Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 15:47:01 +0800 Subject: [PATCH 108/163] feat(builtins): dict methods except keys,values,fromkeys,update; `__delitem__` --- Objects/dictobject.nim | 104 ++++++++++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 23 deletions(-) diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index 08b51cc..98d488c 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -1,6 +1,6 @@ import strformat -import strutils + import tables import macros @@ -29,17 +29,26 @@ proc hasKey*(dict: PyDictObject, key: PyObject): bool = proc `[]`*(dict: PyDictObject, key: PyObject): PyObject = return dict.table[key] +proc del*(dict: PyDictObject, key: PyObject) = + ## do nothing if key not exists + dict.table.del key + +proc clear*(dict: PyDictObject) = dict.table.clear proc `[]=`*(dict: PyDictObject, key, value: PyObject) = dict.table[key] = value -template checkHashableTmpl(obj) = +template checkHashableTmpl(res; obj) = let hashFunc = obj.pyType.magicMethods.hash if hashFunc.isNil: let tpName = obj.pyType.name let msg = "unhashable type: " & tpName - return newTypeError newPyStr(msg) + res = newTypeError newPyStr(msg) + return + +template checkHashableTmpl(obj) = + result.checkHashableTmpl(obj) implDictMagic contains, [mutable: read]: @@ -49,10 +58,7 @@ implDictMagic contains, [mutable: read]: except DictError: let msg = "__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" return newTypeError newPyAscii(msg) - if result.isNil: - return pyFalseObj - else: - return pyTrueObj + return newPyBool(not result.isNil) implDictMagic repr, [mutable: read, reprLockWithMsg"{...}"]: var ss: seq[UnicodeVariant] @@ -73,35 +79,87 @@ implDictMagic len, [mutable: read]: implDictMagic New: newPyDict() - -implDictMagic getitem, [mutable: read]: - checkHashableTmpl(other) - try: - result = self.table.getOrDefault(other, nil) - except DictError: - let msg = "__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" - return newTypeError newPyAscii(msg) - if not (result.isNil): - return result - +template keyError(other: PyObject): PyObject = var msg: PyStrObject let repr = other.pyType.magicMethods.repr(other) if repr.isThrownException: msg = newPyAscii"exception occured when generating key error msg calling repr" else: msg = PyStrObject(repr) - return newKeyError(msg) + newKeyError(msg) +let badHashMsg = + newPyAscii"__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" + +template handleBadHash(res; body){.dirty.} = + try: + body + except DictError: + res = newTypeError badHashMsg + return + +proc getitemImpl(self: PyDictObject, other: PyObject): PyObject = + checkHashableTmpl(other) + result.handleBadHash: + result = self.table.getOrDefault(other, nil) + if not (result.isNil): + return result + return keyError other +implDictMagic getitem, [mutable: read]: self.getitemImpl other implDictMagic setitem, [mutable: write]: checkHashableTmpl(arg1) - try: + result.handleBadHash: self.table[arg1] = arg2 - except DictError: - let msg = "__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" - return newTypeError newPyAscii(msg) pyNone +proc pop*(self: PyDictObject, other: PyObject, res: var PyObject): bool = + ## - if `other` not in `self`, `res` is set to KeyError; + ## - if in, set to value of that key; + ## - if `DictError`_ raised, `res` is set to TypeError + res.checkHashableTmpl(other) + res.handleBadHash: + if self.table.pop(other, res): + return true + res = keyError other + return false + +proc delitemImpl*(self: PyDictObject, other: PyObject): PyObject = + ## internal use. (in typeobject) + if self.pop(other, result): + return pyNone + assert not result.isNil + +implDictMagic delitem, [mutable: write]: + self.delitemImpl other + +implDictMethod get, [mutable: write]: + checkargnumatleast 1 + let key = args[0] + checkhashabletmpl(key) + if args.len == 1: + return self.getitemimpl key + checkargnum 2 + let defval = args[1] + result.handleBadHash: + return self.table.getOrDefault(key, defVal) + # XXX: Python's dict.get(k, v) doesn't discard TypeError + +implDictMethod pop, [mutable: write]: + checkargnumatleast 1 + let key = args[0] + checkhashabletmpl(key) + if args.len == 1: + return self.delitemimpl key + checkargnum 2 + let defval = args[1] + if self.pop(key, result): + return + # XXX: Python's dict.pop(k, v) discard TypeError, KeyError + return defval + +implDictMethod clear(), [mutable: write]: self.clear() + implDictMethod copy(), [mutable: read]: let newT = newPyDict() newT.table = self.table From dfe28ce6fe5f72cce5b485e599d928c677c74fd0 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 15:48:21 +0800 Subject: [PATCH 109/163] feat(syntax): DeleteAttr op (`__delattr__`) --- Objects/pyobjectBase.nim | 1 + Objects/typeobject.nim | 16 +++++++++++++--- Python/ast.nim | 4 +++- Python/compile.nim | 2 ++ Python/neval.nim | 5 +++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index 2493558..6b8ee41 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -101,6 +101,7 @@ type init: BltinMethod getattr: BinaryMethod setattr: TernaryMethod + delattr: BinaryMethod hash: UnaryMethod call: BltinMethod diff --git a/Objects/typeobject.nim b/Objects/typeobject.nim index 7ec8d2d..7eb6eb7 100644 --- a/Objects/typeobject.nim +++ b/Objects/typeobject.nim @@ -114,13 +114,22 @@ proc setAttr(self: PyObject, nameObj: PyObject, value: PyObject): PyObject {. cd if not descrSet.isNil: return descr.descrSet(self, value) + template retAttributeError = + return newAttributeError($self.pyType.name, $name) if self.hasDict: let instDict = PyDictObject(self.getDict) - instDict[name] = value + if value.isNil: + let res = instDict.delitemImpl(name) + if res != pyNone: + assert res.pyType != pyTypeErrorObjectType + retAttributeError + else: + instDict[name] = value return pyNone + retAttributeError - return newAttributeError($self.pyType.name, $name) - +proc delAttr(self: PyObject, nameObj: PyObject): PyObject {. cdecl .} = + setAttr(self, nameObj, nil) proc addGeneric(t: PyTypeObject) = template nilMagic(magicName): bool = @@ -139,6 +148,7 @@ proc addGeneric(t: PyTypeObject) = trySetSlot(eq, defaultEq) trySetSlot(getattr, getAttr) trySetSlot(setattr, setAttr) + trySetSlot(delattr, delAttr) trySetSlot(repr, reprDefault) trySetSlot(hash, hashDefault) trySetSlot(str, t.magicMethods.repr) diff --git a/Python/ast.nim b/Python/ast.nim index 72eced4..d847687 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -268,7 +268,9 @@ method setDelete(astNode: AstNodeBase) {.base.} = if not (astNode of AsdlExpr): unreachable raiseSyntaxError("can't delete", AsdlExpr(astNode)) -method setDelete(astNode: AstSubscript) = +method setDelete(astNode: AstSubscript) = + astnode.ctx = newAstDel() +method setDelete(astNode: AstAttribute) = astnode.ctx = newAstDel() # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE diff --git a/Python/compile.nim b/Python/compile.nim index 6dfa35e..7b036f2 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -780,6 +780,8 @@ compileMethod Attribute: c.addOp(newArgInstr(OpCode.LoadAttr, opArg, lineNo)) elif astNode.ctx of AstStore: c.addOp(newArgInstr(OpCode.StoreAttr, opArg, lineNo)) + elif astNode.ctx of AstDel: + c.addOp(newArgInstr(OpCode.DeleteAttr, opArg, lineNo)) else: unreachable diff --git a/Python/neval.nim b/Python/neval.nim index 2d86dbb..c9f93a5 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -256,6 +256,11 @@ proc evalFrame*(f: PyFrameObject): PyObject = let obj = sPop() discard obj.callMagic(delitem, idx, handleExcp=true) + of OpCode.DeleteAttr: + let name = names[opArg] + let obj = sPop() + discard obj.callMagic(delattr, name, handleExcp=true) + of OpCode.BinarySubscr: doBinary(getitem) From c1c98013f0256497d489bcc0e2cc5061d8c11687 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 16:13:37 +0800 Subject: [PATCH 110/163] fix(py): hash() was id-based over content-based --- Objects/tupleobject.nim | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index aca705c..aaa1b1e 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -1,4 +1,5 @@ -import strutils +import std/hashes +import ./hash import strformat import pyobject @@ -9,6 +10,8 @@ import sliceobject declarePyType Tuple(reprLock, tpToken): items: seq[PyObject] + setHash: bool + hash: Hash proc newPyTuple*(items: seq[PyObject]): PyTupleObject = @@ -157,14 +160,25 @@ genSequenceMagics "tuple", newPyTupleSimple, [], [reprLock], tupleSeqToString -template hashImpl*(items) = - var h = self.id - for item in self.items: - h = h xor item.id - return newPyInt(h) +template hashCollectionImpl*(items; hashForEmpty): Hash = + var result: Hash + if items.len == 0: + result = hashForEmpty + else: + for item in items: + result = result !& hash(item) + !$result + +proc hashCollection*[T: PyObject](self: T): Hash = + if self.setHash: return self.hash + result = self.items.hashCollectionImpl Hash self.pyType.id + self.hash = result + self.setHash = true + +proc hash*(self: PyTupleObject): Hash = self.hashCollection implTupleMagic hash: - hashImpl items + newPyInt hash(self) proc len*(t: PyTupleObject): int {. cdecl inline .} = t.items.len From be0b272d9d01c3b44df4d328af13fec565efef49 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 16:15:17 +0800 Subject: [PATCH 111/163] fix(py): hash worked for dict,list,set --- Objects/dictobject.nim | 2 ++ Objects/hash.nim | 4 ++++ Objects/listobject.nim | 2 ++ Objects/setobject.nim | 10 +++++++--- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index 98d488c..fa64c45 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -76,6 +76,8 @@ implDictMagic repr, [mutable: read, reprLockWithMsg"{...}"]: implDictMagic len, [mutable: read]: newPyInt(self.table.len) +implDictMagic hash: unhashable self + implDictMagic New: newPyDict() diff --git a/Objects/hash.nim b/Objects/hash.nim index 1243261..83c5dfa 100644 --- a/Objects/hash.nim +++ b/Objects/hash.nim @@ -3,6 +3,10 @@ import ./pyobject import ./baseBundle import ../Utils/utils +proc unhashable*(obj: PyObject): PyObject = newTypeError newPyAscii( + "unhashable type '" & obj.pyType.name & '\'' +) + proc rawHash*(obj: PyObject): Hash = ## for type.__hash__ hash(obj.id) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index a7585ab..f3295f6 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -5,6 +5,7 @@ import strutils import pyobject import baseBundle import ./sliceobjectImpl +import ./hash import iterobject import ./tupleobject import ../Utils/[utils, compat] @@ -74,6 +75,7 @@ implListMagic delitem, [mutable: write]: return pyNone return newIndexTypeError(newPyAscii"list", other) +implListMagic hash: unhashable self implListMethod append(item: PyObject), [mutable: write]: self.items.add(item) diff --git a/Objects/setobject.nim b/Objects/setobject.nim index ab1feb9..18a8e2f 100644 --- a/Objects/setobject.nim +++ b/Objects/setobject.nim @@ -3,6 +3,8 @@ import strformat import strutils import std/sets +import std/hashes + import pyobject import baseBundle import ./iterobject @@ -16,6 +18,8 @@ declarePyType Set(reprLock, mutable, tpToken): declarePyType FrozenSet(reprLock, tpToken): items: HashSet[PyObject] + setHash: bool + hash: Hash template setSeqToStr(ss): string = if ss.len == 0: @@ -77,9 +81,6 @@ template genSet(S, mutRead, mutReadRepr){.dirty.} = setSeqToStr - `impl S Magic` hash: - hashImpl items - `impl S Magic` init: if 1 < args.len: let msg = $S & fmt" expected at most 1 args, got {args.len}" @@ -109,6 +110,9 @@ template genSet(S, mutRead, mutReadRepr){.dirty.} = genSet Set, [mutable: read], [mutable: read, reprLock] genSet FrozenSet, [], [reprLock] +proc hash*(self: PyFrozenSetObject): Hash = self.hashCollection +implFrozenSetMagic hash: newPyInt hash(self) +implSetMagic hash: unhashable self implSetMethod update(args), [mutable: write]: for other in args: From 481d38758acdd356382f1432ff6352f41ec8b74b Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 16:46:10 +0800 Subject: [PATCH 112/163] feat(iterobject): NimIteratorIter --- Objects/iterobject.nim | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Objects/iterobject.nim b/Objects/iterobject.nim index c846857..76f176a 100644 --- a/Objects/iterobject.nim +++ b/Objects/iterobject.nim @@ -31,3 +31,25 @@ template pyForIn*(it; iterableToLoop: PyObject; doWithIt) = if it.isThrownException: return it doWithIt + + +type ItorPy = iterator(): PyObject +declarePyType NimIteratorIter(): + itor: ItorPy + +implNimIteratorIterMagic iter: + self + +implNimIteratorIterMagic iternext: + result = self.itor() + if self.itor.finished(): + return newStopIterError() + +proc newPyNimIteratorIter*(itor: ItorPy): PyNimIteratorIterObject = + result = newPyNimIteratorIterSimple() + result.itor = itor + +template genPyNimIteratorIter*(iterable): PyNimIteratorIterObject = + bind newPyNimIteratorIter + newPyNimIteratorIter iterator(): PyObject = + for i in iterable: yield i From 4242ad27b09924f74448f49e698ee48567962f54 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 16:46:38 +0800 Subject: [PATCH 113/163] fix(py): frozenset repr was the same as set's --- Objects/setobject.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Objects/setobject.nim b/Objects/setobject.nim index 18a8e2f..60739b6 100644 --- a/Objects/setobject.nim +++ b/Objects/setobject.nim @@ -26,6 +26,9 @@ template setSeqToStr(ss): string = self.pyType.name & "()" else: '{' & ss.join", " & '}' +template frozensetSeqToStr(ss): string = + bind setSeqToStr + "frozenset(" & setSeqToStr(ss) & ')' template getItems(s: PyObject, elseDo): HashSet = if s.ofPySetObject: PySetObject(s).items @@ -61,7 +64,7 @@ template genBMe(S, mutRead, pyMethod, nop){.dirty.} = if nop(self.items, other.getItems): pyTrueObj else: pyFalseObj -template genSet(S, mutRead, mutReadRepr){.dirty.} = +template genSet(S, setSeqToStr, mutRead, mutReadRepr){.dirty.} = proc `newPy S`*: `Py S Object` = `newPy S Simple`() @@ -92,6 +95,9 @@ template genSet(S, mutRead, mutReadRepr){.dirty.} = self.items.incl i pyNone + `impl S Magic` iter: + genPyNimIteratorIter self.items + genOp S, mutRead, Or, `+` genMe S, mutRead, union, `+` @@ -107,8 +113,8 @@ template genSet(S, mutRead, mutReadRepr){.dirty.} = genBMe S, mutRead, issubset, `<=` genBMe S, mutRead, issuperset, `>=` -genSet Set, [mutable: read], [mutable: read, reprLock] -genSet FrozenSet, [], [reprLock] +genSet Set, setSeqToStr, [mutable: read], [mutable: read, reprLock] +genSet FrozenSet, frozensetSeqToStr, [], [reprLock] proc hash*(self: PyFrozenSetObject): Hash = self.hashCollection implFrozenSetMagic hash: newPyInt hash(self) From ff768cb197bc98b5868acc9232e67773b2d27990 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 16:46:51 +0800 Subject: [PATCH 114/163] feat(builtins): next --- Python/bltinmodule.nim | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 9d072da..c496ac6 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -3,7 +3,7 @@ import strformat import neval import builtindict import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject, - notimplementedobject, sliceobjectImpl] + notimplementedobject, sliceobjectImpl, exceptions] import ../Utils/[utils, macroutils, compat] @@ -93,6 +93,20 @@ implBltinFunc len(obj: PyObject): implBltinFunc hash(obj: PyObject): obj.callMagic(hash) implBltinFunc iter(obj: PyObject): obj.callMagic(iter) +proc builtinNext*(args: seq[PyObject]): PyObject {. cdecl .} = + template callNext(obj): PyObject = + obj.callMagic(iternext) + checkArgNumAtLeast 1 + let obj = args[0] + if args.len == 1: + return callNext obj + checkArgNum 2 + let defVal = args[1] + result = obj.callNext + if result.isStopIter: + return defVal + +registerBltinFunction("next", builtinNext) implBltinFunc repr(obj: PyObject): obj.callMagic(repr) From cca93bc34e5e6471eda172bcd6e9ee6c28131a60 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 17:38:04 +0800 Subject: [PATCH 115/163] feat(inner/pyobject): declarePyType accepts custom typeName --- Objects/pyobject.nim | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index be6987b..6eb3239 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -512,18 +512,30 @@ template methodMacroTmpl(name: untyped, nameStr: string) = macro methodMacroTmpl*(name: untyped): untyped = getAst(methodMacroTmpl(name, name.strVal)) -macro declarePyType*(prototype, fields: untyped): untyped = +macro declarePyType*(prototype, fields: untyped): untyped = + ## `prototype` is of nnkCall format, + ## whose arguments call contains: + ## - base: BASE + ## - typeName: TYPE_NAME; + ## TYPE_NAME defaults to lowerAscii of `prototype[0]` + ## - tpToken, dict, mutable, reprLock prototype.expectKind(nnkCall) fields.expectKind(nnkStmtList) var tpToken, mutable, dict, reprLock: bool var baseTypeStr = "PyObject" + var typeName: string # parse options the silly way for i in 1.. Date: Tue, 22 Jul 2025 17:39:17 +0800 Subject: [PATCH 116/163] fix(py): exceptions type names was like "stopitererror" --- Objects/exceptions.nim | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 376e41a..789270f 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -5,7 +5,7 @@ # let alone using global variables so # we return exception object directly with a thrown flag inside # This might be a bit slower but we are not pursueing ultra performance anyway - +import std/enumutils import strformat import pyobject @@ -32,6 +32,12 @@ type ExceptionToken* {. pure .} = enum Memory, KeyboardInterrupt, +proc getTokenName*(excp: ExceptionToken): string = excp.symbolName +proc getBltinName*(excp: ExceptionToken): string = + case excp + of Base: "Exception" + of StopIter: "StopIteration" + else: excp.getTokenName & "Error" type TraceBack* = tuple fileName: PyObject # actually string @@ -40,7 +46,7 @@ type TraceBack* = tuple colNo: int # optional, for syntax error -declarePyType BaseError(tpToken): +declarePyType BaseError(tpToken, typeName("Exception")): tk: ExceptionToken thrown: bool msg: PyObject # could be nil @@ -59,18 +65,22 @@ proc ofPyExceptionObject*(obj: PyObject): bool {. cdecl, inline .} = macro declareErrors: untyped = result = newStmtList() - var tokenStr: string for i in 1..int(ExceptionToken.high): - let tokenStr = $ExceptionToken(i) + let tok = ExceptionToken(i) + let tokenStr = tok.getTokenName let typeNode = nnkStmtList.newTree( nnkCommand.newTree( newIdentNode("declarePyType"), nnkCall.newTree( newIdentNode(tokenStr & "Error"), - nnkCall.newTree( + newCall( newIdentNode("base"), - newIdentNode("BaseError") + bindSym("BaseError") + ), + newCall( + ident"typeName", + ident tok.getBltinName # or it'll be e.g. "stopitererror" ) ), nnkStmtList.newTree( @@ -112,9 +122,8 @@ template newProcTmpl(excpName) = macro genNewProcs: untyped = result = newStmtList() - var tokenStr: string for i in ExceptionToken.low..ExceptionToken.high: - let tokenStr = $ExceptionToken(i) + let tokenStr = ExceptionToken(i).getTokenName result.add(getAst(newProcTmpl(ident(tokenStr)))) @@ -187,6 +196,7 @@ template getIterableWithCheck*(obj: PyObject): (PyObject, UnaryMethod) = template checkArgNum*(expected: int, name="") = + bind fmt, newTypeError, newPyStr if args.len != expected: var msg: string if name != "": @@ -197,6 +207,7 @@ template checkArgNum*(expected: int, name="") = template checkArgNumAtLeast*(expected: int, name="") = + bind fmt, newTypeError, newPyStr if args.len < expected: var msg: string if name != "": From da03c48973678d7f5094ff7d7ca42b1d74114fc6 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 17:39:51 +0800 Subject: [PATCH 117/163] feat(builtins),fix(py): ren as Exception, StopIteration --- Python/bltinmodule.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index c496ac6..5ca1dda 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -144,9 +144,10 @@ macro registerErrors: untyped = result = newStmtList() template registerTmpl(name:string, tp:PyTypeObject) = registerBltinObject(name, tp) - for i in 1..int(ExceptionToken.high): - let tokenStr = $ExceptionToken(i) - let excpName = tokenStr & "Error" + for i in 0..int(ExceptionToken.high): + let tok = ExceptionToken(i) + let tokenStr = tok.getTokenName + let excpName = tok.getBltinName let typeName = fmt"py{tokenStr}ErrorObjectType" result.add getAst(registerTmpl(excpName, ident(typeName))) From af6b3d1882b5f384eab8f532896d923fb1a70328 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 18:20:49 +0800 Subject: [PATCH 118/163] feat(inner/exceptions): obj.isExceptionOf(ExceptionToken) --- Objects/exceptions.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 789270f..439ea51 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -143,12 +143,13 @@ template newIndexTypeError*(typeName: PyStrObject, obj:PyObject): PyExceptionObj newTypeError(msg) -proc isStopIter*(obj: PyObject): bool = +proc isExceptionOf*(obj: PyObject, tk: ExceptionToken): bool = if not obj.ofPyExceptionObject: return false let excp = PyExceptionObject(obj) - return (excp.tk == ExceptionToken.StopIter) and (excp.thrown) + return (excp.tk == tk) and (excp.thrown) +proc isStopIter*(obj: PyObject): bool = obj.isExceptionOf StopIter method `$`*(e: PyExceptionObject): string = result = "Error: " & $e.tk & " " From 10860668dc4017e69412aa08e31541dd7cdcc4e9 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 18:21:37 +0800 Subject: [PATCH 119/163] feat(builtins): getattr, setattr, delattr --- Python/bltinmodule.nim | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 5ca1dda..270464f 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -93,20 +93,36 @@ implBltinFunc len(obj: PyObject): implBltinFunc hash(obj: PyObject): obj.callMagic(hash) implBltinFunc iter(obj: PyObject): obj.callMagic(iter) -proc builtinNext*(args: seq[PyObject]): PyObject {. cdecl .} = - template callNext(obj): PyObject = - obj.callMagic(iternext) - checkArgNumAtLeast 1 - let obj = args[0] - if args.len == 1: - return callNext obj - checkArgNum 2 - let defVal = args[1] - result = obj.callNext - if result.isStopIter: + +template callWithKeyAndMayDefault(call; tk; N) = + checkArgNumAtLeast N + let obj = args[N-1] + if args.len == N: + return call obj + checkArgNum N+1 + let defVal = args[N] + result = obj.call + if result.isExceptionOf tk: return defVal -registerBltinFunction("next", builtinNext) +template genBltWithKeyAndMayDef(blt; tk; N, call){.dirty.} = + proc `builtin blt`*(args: seq[PyObject]): PyObject {. cdecl .} = + template callNext(obj): PyObject = call + callWithKeyAndMayDefault callNext, tk, N + registerBltinFunction(astToStr(blt), `builtin blt`) + +genBltWithKeyAndMayDef next, StopIter, 1: obj.callMagic(iternext) +genBltWithKeyAndMayDef getattr, Attribute, 2: args[0].callMagic(getattr, obj) + +template genBltOfNArg(blt; N, call){.dirty.} = + proc `builtin blt`*(args: seq[PyObject]): PyObject {. cdecl .} = + checkArgNum N + call + registerBltinFunction(astToStr(blt), `builtin blt`) + +genBltOfNArg setattr, 3: args[0].callMagic(setattr, args[1], args[2]) +genBltOfNArg delattr, 2: args[0].callMagic(delattr, args[1]) + implBltinFunc repr(obj: PyObject): obj.callMagic(repr) From c1ea5b254943bc7898d718553d75b38ba353f213 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 18:49:34 +0800 Subject: [PATCH 120/163] feat(inner): items, `[]` for list,tuple object --- Objects/listobject.nim | 4 +--- Objects/tupleobject.nim | 17 +++++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index f3295f6..a085a1e 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -29,8 +29,6 @@ genSequenceMagics "list", newPyListSimple, [mutable: read], [reprLockWithMsg"[...]", mutable: read], lsSeqToStr -proc len*(self: PyListObject): int{.inline.} = self.items.len - implListMagic setitem, [mutable: write]: if arg1.ofPyIntObject: let idx = getIndex(PyIntObject(arg1), self.len) @@ -149,7 +147,7 @@ implListMethod pop(), [mutable: write]: implListMethod remove(target: PyObject), [mutable: write]: var retObj: PyObject allowSelfReadWhenBeforeRealWrite: - retObj = indexPyListObjectMethod(selfNoCast, @[target]) + retObj = tpMethod(List, index)(selfNoCast, @[target]) if retObj.isThrownException: return retObj assert retObj.ofPyIntObject diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index aaa1b1e..5312095 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -25,9 +25,13 @@ template genCollectMagics*(items, ofPyNameObject, PyNameObject, mutRead, mutReadRepr, seqToStr){.dirty.} = + template len*(self: PyNameObject): int = self.items.len + template `[]`*(self: PyNameObject, i: int): PyObject = self.items[i] + iterator items*(self: PyNameObject): PyObject = + for i in self.items: yield i implNameMagic contains, mutRead: - for item in self.items: + for item in self: let retObj = item.callMagic(eq, other) if retObj.isThrownException: return retObj @@ -38,7 +42,7 @@ template genCollectMagics*(items, implNameMagic repr, mutReadRepr: var ss: seq[UnicodeVariant] - for item in self.items: + for item in self: var itemRepr: PyStrObject let retObj = item.callMagic(repr) errorIfNotString(retObj, "__repr__") @@ -48,7 +52,7 @@ template genCollectMagics*(items, implNameMagic len, mutRead: - newPyInt(self.items.len) + newPyInt(self.len) template genSequenceMagics*(nameStr, @@ -109,8 +113,8 @@ template genSequenceMagics*(nameStr, implNameMagic getitem: if other.ofPyIntObject: - let idx = getIndex(PyIntObject(other), self.items.len) - return self.items[idx] + let idx = getIndex(PyIntObject(other), self.len) + return self[idx] if other.ofPySliceObject: let slice = PySliceObject(other) let newObj = newPyNameSimple() @@ -179,6 +183,3 @@ proc hash*(self: PyTupleObject): Hash = self.hashCollection implTupleMagic hash: newPyInt hash(self) - -proc len*(t: PyTupleObject): int {. cdecl inline .} = - t.items.len From 8df80d3962f0226e0f9b3062b49be6c667c149b6 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 20:51:18 +0800 Subject: [PATCH 121/163] fix(py): `dict.__eq__` not content-based;;dict no tpToken --- Objects/dictobject.nim | 10 ++++++++-- Objects/pyobjectBase.nim | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index fa64c45..4cd94ee 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -15,7 +15,7 @@ export hash # currently not ordered # nim ordered table has O(n) delete time # todo: implement an ordered dict -declarePyType dict(reprLock, mutable): +declarePyType Dict(tpToken, reprLock, mutable): table: Table[PyObject, PyObject] @@ -80,7 +80,13 @@ implDictMagic hash: unhashable self implDictMagic New: newPyDict() - + +implDictMagic eq: + newPyBool( + other.ofPyDictObject() and + self.table == other.PyDictObject.table + ) + template keyError(other: PyObject): PyObject = var msg: PyStrObject let repr = other.pyType.magicMethods.repr(other) diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index 6b8ee41..12aa7aa 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -18,6 +18,7 @@ type Type, Tuple, List, + Dict, Str, Code, NimFunc, From 7457d84728a4d61b8541b9b8f58271a62da6d2b1 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 20:58:39 +0800 Subject: [PATCH 122/163] refact,feat(dict): ren dictobject.keys keysList;impl values,items,keys --- Objects/dictobject.nim | 34 ++++++++++++++++++++++++++-------- Python/bltinmodule.nim | 11 ++++++++++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index 4cd94ee..eeec325 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -5,10 +5,10 @@ import tables import macros import pyobject -import listobject +# import listobject (do not import, or it'll cause recursive import) import baseBundle import ../Utils/utils - +import ./[iterobject, tupleobject] import ./hash export hash @@ -25,6 +25,30 @@ proc newPyDict* : PyDictObject = proc hasKey*(dict: PyDictObject, key: PyObject): bool = return dict.table.hasKey(key) +proc contains*(dict: PyDictObject, key: PyObject): bool = dict.hasKey key + +template borIter(name, nname; R=PyObject){.dirty.} = + iterator name*(dict: PyDictObject): R = + for i in dict.table.nname: yield i +template borIter(name){.dirty.} = borIter(name, name) + +borIter items, keys +borIter keys +borIter values +borIter pairs, pairs, (PyObject, PyObject) + +implDictMagic iter, [mutable: read]: + genPyNimIteratorIter self.keys() +implDictMethod keys(), [mutable: read]: + genPyNimIteratorIter self.keys() +implDictMethod values(), [mutable: read]: + genPyNimIteratorIter self.values() + +iterator pyItems(dict: PyDictObject): PyTupleObject = + for (k, v) in dict.pairs: + yield newPyTuple([k, v]) +implDictMethod items(), [mutable: read]: + genPyNimIteratorIter self.pyItems proc `[]`*(dict: PyDictObject, key: PyObject): PyObject = return dict.table[key] @@ -175,12 +199,6 @@ implDictMethod copy(), [mutable: read]: # in real python this would return a iterator # this function is used internally -proc keys*(d: PyDictObject): PyListObject = - result = newPyList() - for key in d.table.keys: - let rebObj = tpMethod(List, append)(result, @[key]) - if rebObj.isThrownException: - unreachable("No chance for append to thrown exception") proc update*(d1, d2: PyDictObject) = diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 270464f..84c85d8 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -73,6 +73,15 @@ proc builtinPrint*(args: seq[PyObject]): PyObject {. cdecl .} = pyNone registerBltinFunction("print", builtinPrint) + +proc keysList(d: PyDictObject): PyListObject = + ## inner usage + result = newPyList() + for key in d.keys(): + let rebObj = tpMethod(List, append)(result, @[key]) + if rebObj.isThrownException: + unreachable("No chance for append to thrown exception") + implBltinFunc dir(obj: PyObject): # why in CPython 0 argument becomes `locals()`? no idea # get mapping proxy first then talk about how do deal with __dict__ of type @@ -80,7 +89,7 @@ implBltinFunc dir(obj: PyObject): mergedDict.update(obj.getTypeDict) if obj.hasDict: mergedDict.update(PyDictObject(obj.getDict)) - mergedDict.keys + mergedDict.keysList implBltinFunc id(obj: PyObject): From d5e93e2793612036162401be5fceb43ab72db81f Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 21:02:28 +0800 Subject: [PATCH 123/163] fix(py): list(dict) was on .keys() over .items() --- Objects/listobject.nim | 3 ++- Objects/tupleobject.nim | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index a085a1e..45b8fa2 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -8,6 +8,7 @@ import ./sliceobjectImpl import ./hash import iterobject import ./tupleobject +import ./dictobject import ../Utils/[utils, compat] declarePyType List(reprLock, mutable, tpToken): @@ -27,7 +28,7 @@ genSequenceMagics "list", implListMagic, implListMethod, ofPyListObject, PyListObject, newPyListSimple, [mutable: read], [reprLockWithMsg"[...]", mutable: read], - lsSeqToStr + lsSeqToStr, initWithDictUsingPairs=true implListMagic setitem, [mutable: write]: if arg1.ofPyIntObject: diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 5312095..499f264 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -19,6 +19,8 @@ proc newPyTuple*(items: seq[PyObject]): PyTupleObject = # shallow copy result.items = items +proc newPyTuple*(items: openArray[PyObject]): PyTupleObject{.inline.} = + newPyTuple @items template genCollectMagics*(items, implNameMagic, newPyNameSimple, @@ -59,7 +61,7 @@ template genSequenceMagics*(nameStr, implNameMagic, implNameMethod; ofPyNameObject, PyNameObject, newPyNameSimple; mutRead, mutReadRepr; - seqToStr): untyped{.dirty.} = + seqToStr; initWithDictUsingPairs=false): untyped{.dirty.} = bind genCollectMagics genCollectMagics items, @@ -102,8 +104,14 @@ template genSequenceMagics*(nameStr, if self.items.len != 0: self.items.setLen(0) if args.len == 1: - pyForIn i, args[0]: - self.items.add i + let arg = args[0] + template loop(a) = + pyForIn i, a: + self.items.add i + when initWithDictUsingPairs: + if arg.ofPyDictObject: loop tpMethod(Dict, items)(arg) + else: loop arg + else: loop arg pyNone From 25b128fbfdb2d9505edf22b02ba23cc1146e6b8c Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 21:04:18 +0800 Subject: [PATCH 124/163] feat(dict): all meth except fromkeys,popitem; fix(py): dict() accepted no arg --- Objects/dictobject.nim | 33 ++++++++++++++----- Objects/dictobjectImpl.nim | 65 ++++++++++++++++++++++++++++++++++++++ Objects/exceptions.nim | 18 +++++++---- Python/bltinmodule.nim | 2 +- 4 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 Objects/dictobjectImpl.nim diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index eeec325..b4b42e5 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -19,11 +19,11 @@ declarePyType Dict(tpToken, reprLock, mutable): table: Table[PyObject, PyObject] -proc newPyDict* : PyDictObject = +proc newPyDict*(table=initTable[PyObject, PyObject]()) : PyDictObject = result = newPyDictSimple() - result.table = initTable[PyObject, PyObject]() + result.table = table -proc hasKey*(dict: PyDictObject, key: PyObject): bool = +proc hasKey*(dict: PyDictObject, key: PyObject): bool = return dict.table.hasKey(key) proc contains*(dict: PyDictObject, key: PyObject): bool = dict.hasKey key @@ -101,15 +101,16 @@ implDictMagic len, [mutable: read]: newPyInt(self.table.len) implDictMagic hash: unhashable self - -implDictMagic New: - newPyDict() - implDictMagic eq: newPyBool( other.ofPyDictObject() and self.table == other.PyDictObject.table ) +implDictMagic Or(E: PyDictObject), [mutable: read]: + let res = newPyDict self.table + for (k, v) in E.table.pairs: + res.table[k] = v + res template keyError(other: PyObject): PyObject = var msg: PyStrObject @@ -175,7 +176,6 @@ implDictMethod get, [mutable: write]: let defval = args[1] result.handleBadHash: return self.table.getOrDefault(key, defVal) - # XXX: Python's dict.get(k, v) doesn't discard TypeError implDictMethod pop, [mutable: write]: checkargnumatleast 1 @@ -190,6 +190,21 @@ implDictMethod pop, [mutable: write]: # XXX: Python's dict.pop(k, v) discard TypeError, KeyError return defval +implDictMethod setdefault, [mutable: write]: + checkargnumatleast 1 + let key = args[0] + checkhashabletmpl(key) + let defVal = if args.len == 1: pyNone + else: + checkargnum 2 + args[1] + + result.handleBadHash: + if key in self: + return self[key] + self[key] = defVal + return defval + implDictMethod clear(), [mutable: write]: self.clear() implDictMethod copy(), [mutable: read]: @@ -204,3 +219,5 @@ implDictMethod copy(), [mutable: read]: proc update*(d1, d2: PyDictObject) = for k, v in d2.table.pairs: d1[k] = v + +# .__init__, .update, .keys, etc method is defined in ./dictobjectImpl diff --git a/Objects/dictobjectImpl.nim b/Objects/dictobjectImpl.nim new file mode 100644 index 0000000..51c1c7e --- /dev/null +++ b/Objects/dictobjectImpl.nim @@ -0,0 +1,65 @@ + + +import ./pyobject +import ../Python/call +import ./[stringobject, iterobject, numobjects] +import ./noneobject +import ./exceptions +import ./dictobject +export dictobject + + +# redeclare this for these are "private" macros + +methodMacroTmpl(Dict) + + + + +proc updateImpl*(self: PyDictObject, E: PyObject): PyObject = + if E.ofPyDictObject: + self.update PyDictObject E + return pyNone + let + keysFunc = E.callMagic(getattr, newPyAscii"keys") # getattr(E, "keys") + getitem = E.getMagic(getitem) + if not keysFunc.isThrownException and not getitem.isNil: + let ret = fastCall(keysFunc, @[]) + if ret.isThrownException: return ret + pyForIn i, ret: + self[i] = getitem(E, i) + else: + var idx = 0 + pyForIn ele, E: + let getter = ele.getMagic(getitem) + if getter.isNil: + return newTypeError newPyAscii( + "cannot convert dictionary update sequence element #" & + $idx & " to a sequence") + # only use getitem + let + k = getter(ele, pyIntZero) + v = getter(ele, pyIntOne) + self[k] = v + idx.inc + pyNone + +implDictMagic iOr(E: PyObject), [mutable: write]: self.updateImpl E + +# XXX: how to impl using std/table? +# implDictMethod popitem(), [mutable: write]: + +implDictMethod update(E: PyObject), [mutable: write]: + # XXX: `**kw` not supported in syntax + self.updateImpl E + +implDictMagic init: + let argsLen = args.len + case argsLen + of 0: pyNone + of 1: + let ret = self.updateImpl(args[argsLen-1]) + if ret.isThrownException: return ret + pyNone + else: + errArgNum argsLen, 1 diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 439ea51..24c9d2d 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -195,16 +195,20 @@ template getIterableWithCheck*(obj: PyObject): (PyObject, UnaryMethod) = retTuple = (iterobj, iternextFunc) retTuple +template errArgNum*(argsLen, expected: int; name="")= + bind fmt, newTypeError, newPyStr + var msg: string + let sargsLen{.inject.} = $argsLen + if name != "": + msg = name & " takes exactly " & $expected & fmt" argument ({sargsLen} given)" + else: + msg = "expected " & $expected & fmt" argument ({sargsLen} given)" + return newTypeError(newPyStr msg) template checkArgNum*(expected: int, name="") = - bind fmt, newTypeError, newPyStr + bind errArgNum if args.len != expected: - var msg: string - if name != "": - msg = name & " takes exactly " & $expected & fmt" argument ({args.len} given)" - else: - msg = "expected " & $expected & fmt" argument ({args.len} given)" - return newTypeError(newPyStr msg) + errArgNum args.len, expected, name template checkArgNumAtLeast*(expected: int, name="") = diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 84c85d8..951ddca 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -3,7 +3,7 @@ import strformat import neval import builtindict import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject, - notimplementedobject, sliceobjectImpl, exceptions] + notimplementedobject, sliceobjectImpl, dictobjectImpl, exceptions] import ../Utils/[utils, macroutils, compat] From 120063b6f9776b066e8dd5ff7e6912e3927a5a3d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 22 Jul 2025 21:37:24 +0800 Subject: [PATCH 125/163] refine(opt): followup 8bd66ba1772e038503fe03c4a3e82acdd812fd65 --- Python/compile.nim | 4 ++-- Python/cpython.nim | 2 +- Python/neval.nim | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/compile.nim b/Python/compile.nim index 7b036f2..33f397e 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -95,7 +95,7 @@ proc newCompilerUnit(st: SymTable, proc newCompiler(root: AsdlModl, fileName: PyStrObject): Compiler = result = new Compiler result.st = newSymTable(root) - result.units.add(newCompilerUnit(result.st, root, newPyStr(""))) + result.units.add(newCompilerUnit(result.st, root, newPyAscii"")) result.fileName = fileName @@ -713,7 +713,7 @@ compileMethod ListComp: let lineNo = astNode.lineNo.value assert astNode.generators.len == 1 let genNode = AstComprehension(astNode.generators[0]) - c.units.add(newCompilerUnit(c.st, astNode, newPyStr(""))) + c.units.add(newCompilerUnit(c.st, astNode, newPyAscii"")) # empty list let body = newBasicBlock() let ending = newBasicBlock() diff --git a/Python/cpython.nim b/Python/cpython.nim index 133ba1e..e02f26f 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -37,7 +37,7 @@ proc parseCompileEval*(input: string, lexer: Lexer, rootCst = parseWithState(input, lexer, Mode.Single, rootCst) except SyntaxError: let e = SyntaxError(getCurrentException()) - let excpObj = fromBltinSyntaxError(e, newPyStr(Fstdin)) + let excpObj = fromBltinSyntaxError(e, newPyAscii(Fstdin)) excpObj.printTb finished = true return true diff --git a/Python/neval.nim b/Python/neval.nim index c9f93a5..3c8ea10 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -305,13 +305,13 @@ proc evalFrame*(f: PyFrameObject): PyObject = handleException(reprObj) # todo: optimization - build a cache - let printFunction = PyNimFuncObject(bltinDict[newPyStr("print")]) + let printFunction = PyNimFuncObject(bltinDict[newPyAscii"print"]) let retObj = tpMagic(NimFunc, call)(printFunction, @[reprObj]) if retObj.isThrownException: handleException(retObj) of OpCode.LoadBuildClass: - sPush bltinDict[newPyStr("__build_class__")] + sPush bltinDict[newPyAscii"__build_class__"] of OpCode.ReturnValue: return sPop() From edddd9a5fcd76fc7d0baa99d4863809f6c6d1e22 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 23 Jul 2025 11:17:42 +0800 Subject: [PATCH 126/163] refine(cpython): rm unused bool result --- Parser/lexerTypes.nim | 1 + Parser/parser.nim | 2 ++ Python/cpython.nim | 10 +++++----- Python/karaxpython.nim | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Parser/lexerTypes.nim b/Parser/lexerTypes.nim index 6dce8b1..635e024 100644 --- a/Parser/lexerTypes.nim +++ b/Parser/lexerTypes.nim @@ -8,6 +8,7 @@ type Eval Lexer* = ref object + ## For CPython 3.13, this is roughly equal to `tok_state*` indentStack: seq[int] # Stack to track indentation levels lineNo: int tokenNodes*: seq[TokenNode] # might be consumed by parser diff --git a/Parser/parser.nim b/Parser/parser.nim index bf3ef72..2c67578 100644 --- a/Parser/parser.nim +++ b/Parser/parser.nim @@ -133,6 +133,8 @@ proc parseWithState*(input: string, parseNodeArg: ParseNode = nil, ): ParseNode = + ## like `_PyPegen_run_parser_from_string` or + ## `_PyPegen_run_parser(Parser *p)` in Python 3.13 lexer.lexString(input, mode) try: var tokenSeq = lexer.tokenNodes diff --git a/Python/cpython.nim b/Python/cpython.nim index e02f26f..0b9931c 100644 --- a/Python/cpython.nim +++ b/Python/cpython.nim @@ -31,7 +31,7 @@ const Fstdin = "" proc parseCompileEval*(input: string, lexer: Lexer, rootCst: var ParseNode, prevF: var PyFrameObject, finished: var bool - ): bool{.discardable.} = + ) = ## stuff to change, just a compatitable layer for ./jspython try: rootCst = parseWithState(input, lexer, Mode.Single, rootCst) @@ -40,18 +40,18 @@ proc parseCompileEval*(input: string, lexer: Lexer, let excpObj = fromBltinSyntaxError(e, newPyAscii(Fstdin)) excpObj.printTb finished = true - return true + return if rootCst.isNil: - return true + return finished = rootCst.finished if not finished: - return false + return let compileRes = compile(rootCst, Fstdin) if compileRes.isThrownException: PyExceptionObject(compileRes).printTb - return true + return let co = PyCodeObject(compileRes) when defined(debug): diff --git a/Python/karaxpython.nim b/Python/karaxpython.nim index 730c5ad..153d3d8 100644 --- a/Python/karaxpython.nim +++ b/Python/karaxpython.nim @@ -14,12 +14,12 @@ var finished = true var rootCst: ParseNode let lexerInst = newLexer("") var prevF: PyFrameObject -proc interactivePython(input: string): bool {. exportc, discardable .} = +proc interactivePython(input: string) {. exportc .} = echo input if finished: rootCst = nil lexerInst.clearIndent - return parseCompileEval(input, lexerInst, rootCst, prevF, finished) + parseCompileEval(input, lexerInst, rootCst, prevF, finished) let info = getVersionString(verbose=true) const gitRepoUrl{.strdefine.} = "" From 6ab9a6c416a81cbf23117ed639f447f5bfbe5fb2 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 23 Jul 2025 19:06:34 +0800 Subject: [PATCH 127/163] fix(inner): $codeobject not work for `OpCode.Delete*` --- Objects/codeobject.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/codeobject.nim b/Objects/codeobject.nim index 7a0cb7b..689b733 100644 --- a/Objects/codeobject.nim +++ b/Objects/codeobject.nim @@ -64,8 +64,8 @@ method `$`*(code: PyCodeObject): string = if opCode in hasArgSet: line &= fmt"{opArg:<4}" case opCode - of OpCode.LoadName, OpCode.StoreName, OpCode.LoadAttr, - OpCode.LoadGlobal, OpCode.StoreGlobal: + of OpCode.LoadName, OpCode.StoreName, OpCode.DeleteName, OpCode.LoadAttr, + OpCode.LoadGlobal, OpCode.StoreGlobal, OpCode.DeleteGlobal: line &= fmt" ({code.names[opArg]})" of OpCode.LoadConst: let constObj = code.constants[opArg] @@ -76,9 +76,9 @@ method `$`*(code: PyCodeObject): string = line &= fmt" ({reprStr})" else: line &= fmt" ({code.constants[opArg]})" - of OpCode.LoadFast, OpCode.StoreFast: + of OpCode.LoadFast, OpCode.StoreFast, OpCode.DeleteFast: line &= fmt" ({code.localVars[opArg]})" - of OpCode.LoadDeref, OpCode.StoreDeref: + of OpCode.LoadDeref, OpCode.StoreDeref, OpCode.DeleteDeref: if opArg < code.cellVars.len: line &= fmt" ({code.cellVars[opArg]})" else: From 7c3d9bea2c72b6239d372b8a37875ebc82883b5a Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 23 Jul 2025 12:49:07 +0800 Subject: [PATCH 128/163] feat(syntax/del_name): (not work for repl) --- Python/ast.nim | 2 ++ Python/compile.nim | 51 ++++++++++++--------------------------------- Python/neval.nim | 32 ++++++++++++++++++++++++++-- Python/symtable.nim | 9 ++++++++ 4 files changed, 54 insertions(+), 40 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index d847687..1448cbe 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -272,6 +272,8 @@ method setDelete(astNode: AstSubscript) = astnode.ctx = newAstDel() method setDelete(astNode: AstAttribute) = astnode.ctx = newAstDel() +method setDelete(astNode: AstName) = + astnode.ctx = newAstDel() # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE ast single_input, [AstInteractive]: diff --git a/Python/compile.nim b/Python/compile.nim index 33f397e..ecaeba9 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -178,7 +178,8 @@ proc addLoadConst(c: Compiler, pyObject: PyObject, lineNo: int) = {. pop .} -proc addLoadOp(c: Compiler, nameStr: PyStrObject, lineNo: int) = +template genScopeCase(X){.dirty.} = + proc `add X Op`(c: Compiler, nameStr: PyStrObject, lineNo: int) = let scope = c.tste.getScope(nameStr) var @@ -188,54 +189,28 @@ proc addLoadOp(c: Compiler, nameStr: PyStrObject, lineNo: int) = case scope of Scope.Local: opArg = c.tste.localId(nameStr) - opCode = OpCode.LoadFast + opCode = OpCode.`X Fast` of Scope.Global: opArg = c.tste.nameId(nameStr) - opCode = OpCode.LoadGlobal + opCode = OpCode.`X Global` of Scope.Cell: opArg = c.tste.cellId(nameStr) - opCode = OpCode.LoadDeref + opCode = OpCode.`X Deref` of Scope.Free: opArg = c.tste.freeId(nameStr) - opCode = OpCode.LoadDeref + opCode = OpCode.`X Deref` let instr = newArgInstr(opCode, opArg, lineNo) c.addOp(instr) - -proc addLoadOp(c: Compiler, name: AsdlIdentifier, lineNo: int) = + proc `add X Op`(c: Compiler, name: AsdlIdentifier, lineNo: int) = let nameStr = name.value - addLoadOp(c, nameStr, lineNo) + `add X Op`(c, nameStr, lineNo) -proc addStoreOp(c: Compiler, nameStr: PyStrObject, lineNo: int) = - let scope = c.tste.getScope(nameStr) - - var - opArg: int - opCode: OpCode - - case scope - of Scope.Local: - opArg = c.tste.localId(nameStr) - opCode = OpCode.StoreFast - of Scope.Global: - opArg = c.tste.nameId(nameStr) - opCode = OpCode.StoreGlobal - of Scope.Cell: - opArg = c.tste.cellId(nameStr) - opCode = OpCode.StoreDeref - of Scope.Free: - opArg = c.tste.freeId(nameStr) - opCode = OpCode.StoreDeref - - let instr = newArgInstr(opCode, opArg, lineNo) - c.addOp(instr) - - -proc addStoreOp(c: Compiler, name: AsdlIdentifier, lineNo: int) = - let nameStr = name.value - addStoreOp(c, nameStr, lineNo) +genScopeCase Load +genScopeCase Store +genScopeCase Delete proc assemble(cu: CompilerUnit, fileName: PyStrObject): PyCodeObject = @@ -813,8 +788,8 @@ compileMethod Name: c.addLoadOp(astNode.id, lineNo) elif astNode.ctx of AstStore: c.addStoreOp(astNode.id, lineNo) - #elif astNode.ctx of AstDel: - # c.addOp(newArgInstr(OpCode.DeleteName, astNode.id, lineNo)) + elif astNode.ctx of AstDel: + c.addDeleteOp(astNode.id, lineNo) else: unreachable # no other context implemented diff --git a/Python/neval.nim b/Python/neval.nim index 3c8ea10..1da4695 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -204,6 +204,27 @@ proc evalFrame*(f: PyFrameObject): PyObject = excpObj = excp break normalExecution + template notDefined(name: string) = + let msg = "name '" & name & "' is not defined" + handleException(newNameError newPyStr(msg)) + + template genPop(T, toDel){.dirty.} = + template pop(se: T, i: OpArg, unused: var PyObject): bool = + if i < 0 or i > toDel.high: false + else: + toDel.del i + true + genPop (ptr seq[PyObject]), se[] # fastLocals + genPop (seq[PyStrObject]), se # localVars + template deleteOrRaise(d, n; nMsg: string; elseDo) = + var unused: PyObject + if d.pop(n, unused): + continue + elseDo + notDefined(nMsg) + template deleteOrRaise(d, n; nMsg) = + deleteOrRaise(d, n, nMsg): discard + # the main interpreter loop try: # exception handler loop @@ -515,8 +536,7 @@ proc evalFrame*(f: PyFrameObject): PyObject = elif bltinDict.hasKey(name): obj = bltinDict[name] else: - let msg = fmt"name '{name.str}' is not defined" - handleException(newNameError newPyStr(msg)) + notDefined $name.str sPush obj of OpCode.SetupFinally: @@ -537,6 +557,14 @@ proc evalFrame*(f: PyFrameObject): PyObject = of OpCode.StoreFast: fastLocals[opArg] = sPop() + of OpCode.DeleteGlobal: + let name = names[opArg] + deleteOrRaise f.globals, name, $name.str + of OpCode.DeleteFast: + deleteOrRaise fastLocals, opArg, $opArg: + deleteOrRaise f.code.localVars, opArg, $opArg + #of OpCode.DeleteDeref: deleteOrRaise cellVars, opArg #.refObj = sPop + of OpCode.RaiseVarargs: case opArg of 0: diff --git a/Python/symtable.nim b/Python/symtable.nim index 175fd96..783c72f 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -86,6 +86,11 @@ proc addDeclaration(ste: SymTableEntry, name: AsdlIdentifier) = let nameStr = name.value ste.addDeclaration nameStr +proc rmDeclaration(ste: SymTableEntry, name: PyStrObject) = + ste.declaredVars.excl name +proc rmDeclaration(ste: SymTableEntry, name: AsdlIdentifier) = + ste.rmDeclaration(name.value) + proc addUsed(ste: SymTableEntry, name: PyStrObject) = ste.usedVars.incl name @@ -334,6 +339,10 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = ste.addDeclaration(nameNode.id) of AsdlExprContextTk.Load: ste.addUsed(nameNode.id) + of AsdlExprContextTk.Del: + #ste.rmDeclaration(nameNode.id) + # TODO: don't rm decl, but what to do? + discard else: unreachable From 288f85d177804bbd3056b1a868071a4ea146e530 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 28 Jul 2025 00:50:52 +0800 Subject: [PATCH 129/163] feat(syntax/except_with_name): (wrong scope in some cases) --- Python/ast.nim | 13 +++++++++---- Python/compile.nim | 22 ++++++++++++++++++---- Python/symtable.nim | 3 ++- tests/asserts/tryexcept.py | 7 ++++--- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index 1448cbe..e5017d6 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -776,11 +776,16 @@ ast except_clause, [AstExceptHandler]: return of 2: result.type = astTest(parseNode.children[1]) - else: - raiseSyntaxError("'except' with name not implemented", parseNode.children[2]) - + of 4: + # 'as' NAME + result.type = astTest(parseNode.children[1]) + assert parseNode.children[2].tokenNode.token == Token.`as` + let nameNode = parseNode.children[3] + if nameNode.tokenNode.token != Token.Name: + raiseSyntaxError("invalid syntax", nameNode) + result.name = newIdentifier nameNode.tokenNode.content + else: unreachable - # suite simple_stmt | NEWLINE INDENT stmt+ DEDENT ast suite, [seq[AsdlStmt]]: diff --git a/Python/compile.nim b/Python/compile.nim index ecaeba9..3794964 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -550,22 +550,36 @@ compileMethod Try: let isLast = idx == astNode.handlers.len-1 let handler = AstExcepthandler(handlerObj) - assert handler.name.isNil + let hLine = handler.lineno.value c.addBlock(excpBlocks[idx]) - if not handler.type.isNil: + let noAs = handler.name.isNil + if handler.type.isNil: + assert noAs + else: # In CPython duptop is required, here we don't need that, because in each # exception match comparison we don't pop the exception, # allowing further comparison # c.addop(OpCode.DupTop) c.compile(handler.type) - c.addop(newArgInstr(OpCode.CompareOp, int(CmpOp.ExcpMatch), handler.lineNo.value)) + c.addop(newArgInstr(OpCode.CompareOp, int(CmpOp.ExcpMatch), hLine)) if isLast: c.addop(newJumpInstr(OpCode.PopJumpIfFalse, ending, c.lastLineNo)) else: c.addop(newJumpInstr(OpCode.PopJumpIfFalse, excpBlocks[idx+1], c.lastLineNo)) + if not noAs: + # no need to `c.compile(handler.name)` + # as it's just a identifier + c.addop(OpCode.DupTop, hLine) + c.addStoreOp(handler.name, hLine) # now we are handling the exception, no need for future comparison - c.addop(OpCode.PopTop, handler.lineNo.value) + c.addop(OpCode.PopTop, hLine) c.compileSeq(handler.body) + if not noAs: + #[name=None; del name # Mark as artificial]# + c.addLoadConst(pyNone, hLine) + c.addStoreOp(handler.name, hLine) + + c.addDeleteOp(handler.name, hLine) # skip other handlers if not isLast: c.addop(newJumpInstr(OpCode.JumpAbsolute, ending, c.lastLineNo)) diff --git a/Python/symtable.nim b/Python/symtable.nim index 783c72f..5c26ffd 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -385,7 +385,8 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = elif astNode of AsdlExceptHandler: let excpNode = AstExcepthandler(astNode) - assert excpNode.name.isNil + if not excpNode.name.isNil: + ste.addDeclaration(excpNode.name) visitSeq(excpNode.body) visit(excpNode.type) else: diff --git a/tests/asserts/tryexcept.py b/tests/asserts/tryexcept.py index 2231916..b71383b 100644 --- a/tests/asserts/tryexcept.py +++ b/tests/asserts/tryexcept.py @@ -66,7 +66,8 @@ def nested(): except: c - -nested() - +try: + nested() +except NameError as e: + assert str(e) == "NameError: name 'c' is not defined" From 97bae3e8cfe977922e4f24d912a7d6d7ca7ff049 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 29 Jul 2025 16:25:50 +0800 Subject: [PATCH 130/163] feat(syntax/finally): try: ... finally: ... --- Python/ast.nim | 21 ++++++++++++++------- Python/bltinmodule.nim | 1 + Python/compile.nim | 41 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index e5017d6..6f465e0 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -751,13 +751,20 @@ ast try_stmt, [AstTry]: result.body = astSuite(parseNode.children[2]) for i in 1..((parseNode.children.len-1) div 3): let child1 = parseNode.children[i*3] - if not (child1.tokenNode.token == Token.except_clause): - raiseSyntaxError("else/finally in try not implemented", child1) - let handler = astExceptClause(child1) - let child3 = parseNode.children[i*3+2] - handler.body = astSuite(child3) - result.handlers.add(handler) - + # child2 is colon + let body = astSuite(parseNode.children[i*3+2]) + case child1.tokenNode.token + of Token.except_clause: + let handler = astExceptClause(child1) + handler.body = body + result.handlers.add(handler) + of Token.`finally`: + # child2 is colon + result.finalbody = body + of Token.`else`: + result.orelse = body + else: + unreachable ast with_stmt, [AsdlStmt]: raiseSyntaxError("with not implemented") diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 951ddca..7001f00 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -2,6 +2,7 @@ import strformat {.used.} # this module contains toplevel code, so never `importButNotUsed` import neval import builtindict +import ./compile import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject, notimplementedobject, sliceobjectImpl, dictobjectImpl, exceptions] import ../Utils/[utils, macroutils, compat] diff --git a/Python/compile.nim b/Python/compile.nim index 3794964..190143d 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -528,9 +528,9 @@ compileMethod Raise: c.compile(astNode.exc) c.addOp(newArgInstr(OpCode.RaiseVarargs, 1, astNode.lineNo.value)) -compileMethod Try: - assert astNode.orelse.len == 0 - assert astNode.finalbody.len == 0 + +proc codegen_try_except(c: Compiler, astNode: AstTry) = + assert astNode.orelse.len == 0, "not impl yet" assert 0 < astNode.handlers.len # the body here may not be necessary, I'm not sure. Add just in case. let body = newBasicBlock() @@ -588,6 +588,41 @@ compileMethod Try: c.addBlock(ending) c.addOp(OpCode.PopBlock, lastLineNo) +proc codegen_try_finally(c: Compiler, astNode: AstTry) = + let curLine = astNode.lineNo.value + + let lastLineNo = c.lastLineNo + let + ending = newBasicBlock() + exit = newBasicBlock() + + # `try` block + c.addOp(newJumpInstr(OpCode.SetupFinally, ending, curLine)) + + let body = newBasicBlock() + c.addBlock(body) + + + if astNode.handlers.len != 0: + codegen_try_except(c, astNode) + else: + c.compileSeq astNode.body + + c.addop(newJumpInstr(OpCode.JumpAbsolute, exit, lastLineNo)) + + # `finally` block + c.addBlock(ending) + c.compileSeq astNode.finalbody + + c.addBlock(exit) + c.addOp(OpCode.PopBlock, lastLineNo) + +compileMethod Try: + if astNode.finalbody.len != 0: + codegen_try_finally(c, astNode) + else: + codegen_try_except(c, astNode) + compileMethod Assert: let lineNo = astNode.lineNo.value From 8a8036951f862e2face6e119a59ecd10ec39825a Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 29 Jul 2025 16:27:56 +0800 Subject: [PATCH 131/163] fix(nimc): not compile when -d:debug --- Python/neval.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Python/neval.nim b/Python/neval.nim index 1da4695..42ad115 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -144,15 +144,21 @@ proc evalFrame*(f: PyFrameObject): PyObject = template sPeek(idx: int): PyObject = valStack[^idx] - template sSetTop(obj: PyObject) = + template sSetTop(obj: PyObject{atom}) = when defined(debug): assert(not obj.pyType.isNil) valStack[^1] = obj + template sSetTop(obj: PyObject) = + let o = obj + sSetTop o - template sPush(obj: PyObject) = + template sPush(obj: PyObject{atom}) = when defined(debug): assert(not obj.pyType.isNil) valStack.add obj + template sPush(obj: PyObject) = + let o = obj + sPush o template sEmpty: bool = valStack.len == 0 From 06960be2877ad5bb28f85cac7fc8f47825827376 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 29 Jul 2025 19:13:03 +0800 Subject: [PATCH 132/163] feat(syntax/strlit): sup u/r prefix --- Parser/lexer.nim | 57 +++++++++++++++++++++++++++++++--------- Parser/string_parser.nim | 14 +++++++--- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 3b06340..405d204 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -18,6 +18,11 @@ type Source = ref object lines: seq[string] +const + BPrefix = {'b', 'B'} + RPrefix = {'r', 'R'} + StrLitPrefix = BPrefix + RPrefix + {'u', 'U'} + StrLitQuote = {'"', '\''} template indentLevel(lexer: Lexer): int = lexer.indentStack[^1] @@ -130,24 +135,52 @@ proc getNextToken( template newTokenNodeWithNo(Tk): TokenNode = newTokenNode(Token.Tk, lexer.lineNo, idx) - case line[idx] - of 'a'..'z', 'A'..'Z', '_': + template addId = addToken(Name, "Invalid identifier") - of '0'..'9': - addToken(Number, "Invalid number") - of '"', '\'': - let pairingChar = line[idx] - - if idx == line.len - 1: - raiseSyntaxError("Invalid string syntax") + + template asIs(x): untyped = x + template addString(pairingChar: char, escaper: untyped = lexer.decode_unicode_with_escapes, tok=Token.String) = + ## PY-DIFF: We use different Token for bytes and str, for s as a String/Bytes Token, + ## `s` is translated content (e.g. `r'\n'` translated to Newline Char), + ## unlike CPython only has String Token and `s[0]` is prefix and `s[1]` is quotation mark, as it's no need to check again let l = line.skipUntil(pairingChar, idx+1) if idx + l + 1 == line.len: # pairing `"` not found raiseSyntaxError("Invalid string syntax") else: - let s = lexer.decode_unicode_with_escapes(line[idx+1..idx+l]) - result = newTokenNode(Token.String, lexer.lineNo, idx, s) + let s = escaper(line[idx+1..idx+l]) + result = newTokenNode(tok, lexer.lineNo, idx, s) idx += l + 2 + let curChar = line[idx] + case curChar + of {'a'..'z', 'A'..'Z', '_'} - StrLitPrefix: + addId + of StrLitPrefix: + let prefix = curChar + var quote: char + template nextIsQuote(): bool = + idx < line.high and (quote = line[idx+1]; quote) in StrLitQuote + if nextIsQuote(): + idx += 1 + case prefix + of BPrefix: addString quote, lexer.decode_bytes_with_escapes + of RPrefix: addString quote, asIs + else: addString quote + elif ( + prefix in BPrefix and quote in RPrefix or + quote in BPrefix and prefix in RPrefix + ) and nextIsQuote: # raw bytes: br, bR, rb, etc. + idx += 1 + addString quote, asIs + else: + addId + of '0'..'9': + addToken(Number, "Invalid number") + of StrLitQuote: + if idx == line.len - 1: + raiseSyntaxError("Invalid string syntax") + addString curChar + of '\n': result = newTokenNodeWithNo(Newline) idx += 1 @@ -259,7 +292,7 @@ proc getNextToken( of '@': addSingleOrDoubleCharToken(At, AtEqual, '=') else: - raiseSyntaxError(fmt"Unknown character {line[idx]}") + raiseSyntaxError(fmt"Unknown character {curChar}") assert result != nil diff --git a/Parser/string_parser.nim b/Parser/string_parser.nim index 1e002a3..0e3c6dd 100644 --- a/Parser/string_parser.nim +++ b/Parser/string_parser.nim @@ -25,11 +25,17 @@ proc lexMessage(info: LineInfo, kind: TranslateEscapeErr, _: string){.raises: [S else: raiseSyntaxError(arg, info.fileName, info.line, info.column) -proc decode_unicode_with_escapes*(L: lexerTypes.Lexer, s: string): string{. - raises: [SyntaxError].} = - var lex = newLexer[true](s, lexMessage) +template decode_string_with_escapes(lex; s: string): string = lex.lineInfo.fileName = L.fileName lex.lineInfo.line = L.lineNo lex.translateEscape - +{.push raises: [SyntaxError].} +proc decode_unicode_with_escapes*(L: lexerTypes.Lexer, s: string): string = + var lex = newLexer[true](s, lexMessage) + lex.decode_string_with_escapes s + +proc decode_bytes_with_escapes*(L: lexerTypes.Lexer, s: string): string = + var lex = newLexer[false](s, lexMessage) + lex.decode_string_with_escapes s +{.pop.} From 97daab1f577517634ece5087d01b6aa824c195fb Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 02:22:47 +0800 Subject: [PATCH 133/163] wip(gram): add BYTES token --- Parser/Grammar | 3 ++- Parser/lexer.nim | 4 ++-- Parser/token.nim | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Parser/Grammar b/Parser/Grammar index 7d3dd0b..f5ea4f3 100644 --- a/Parser/Grammar +++ b/Parser/Grammar @@ -107,7 +107,8 @@ atom_expr: ['await'] atom trailer* atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | - NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') + NAME | NUMBER | BYTES+ | STRING+ | '...' | 'None' | 'True' | 'False') +# XXX: PY-DIFF: we (npython), split `bytes` from STRING token type testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 405d204..5017528 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -163,7 +163,7 @@ proc getNextToken( if nextIsQuote(): idx += 1 case prefix - of BPrefix: addString quote, lexer.decode_bytes_with_escapes + of BPrefix: addString quote, lexer.decode_bytes_with_escapes, Token.Bytes of RPrefix: addString quote, asIs else: addString quote elif ( @@ -171,7 +171,7 @@ proc getNextToken( quote in BPrefix and prefix in RPrefix ) and nextIsQuote: # raw bytes: br, bR, rb, etc. idx += 1 - addString quote, asIs + addString quote, asIs, Token.Bytes else: addId of '0'..'9': diff --git a/Parser/token.nim b/Parser/token.nim index 3ca06bb..ac7c39c 100644 --- a/Parser/token.nim +++ b/Parser/token.nim @@ -14,6 +14,7 @@ const ("ENDMARKER" , "Endmarker"), ("NAME" , "Name"), ("NUMBER" , "Number"), + ("BYTES" , "Bytes"), ("STRING" , "String"), ("NEWLINE" , "Newline"), ("INDENT" , "Indent"), @@ -77,7 +78,7 @@ proc readGrammarToken: seq[string] {.compileTime.} = var tokenString: string discard line.parseUntil(tokenString, ':') # stored in tokenString result.add(tokenString) - + # everything inside pars proc readReserveName: HashSet[string] {.compileTime.} = @@ -159,7 +160,7 @@ proc genTerminatorSet: set[Token] {. compileTime .} = const terminatorSet = genTerminatorSet() # token nodes that should have a content field -const contentTokenSet* = {Token.Name, Token.Number, Token.String} +const contentTokenSet* = {Token.Name, Token.Number, Token.String, Token.Bytes} type TokenNode* = ref object @@ -167,7 +168,7 @@ type of terminatorSet: lineNo*: int colNo*: int - content*: string # only for name, number and string + content*: string # only for name, number, bytes, and string else: discard From 7aa5d62a15f71f012ef3db54c6d16582e4735750 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 02:44:39 +0800 Subject: [PATCH 134/163] feat(bytes): bltins, add `__bytes__` magic (almost no magic,method impl) --- Objects/byteobjects.nim | 85 +++++++++++++++++++++++++++++++++++++ Objects/byteobjectsImpl.nim | 25 +++++++++++ Objects/pyobjectBase.nim | 2 + Python/bltinmodule.nim | 6 ++- 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 Objects/byteobjects.nim create mode 100644 Objects/byteobjectsImpl.nim diff --git a/Objects/byteobjects.nim b/Objects/byteobjects.nim new file mode 100644 index 0000000..712d80b --- /dev/null +++ b/Objects/byteobjects.nim @@ -0,0 +1,85 @@ +## bytesobject and bytesarrayobject +import std/strformat +import ./pyobject + +import ./[listobject, tupleobject, stringobject, exceptions, iterobject] + +declarePyType Bytes(tpToken): + items: string + +declarePyType ByteArray(reprLock, mutable): + items: string + +type PyBytesWriter* = object + #overallocate*: bool + use_bytearray*: bool + s: seq[char] + +proc allocated*(self: PyBytesWriter): int{.error: "this writer is dynamically allocated".} +proc initPyBytesWriter*(): PyBytesWriter = discard + +proc len*(self: PyBytesWriter): int{.inline.} = self.s.len +proc add*(self: var PyBytesWriter, c: char){.inline.} = self.s.add c +proc reset*(self: var PyBytesWriter, cap: int=0) = + ## like `_PyBytesWriter_Alloc` + self.s = newSeqOfCap[char] cap +proc initPyBytesWriter*(cap: int): PyBytesWriter = + result = initPyBytesWriter() + result.reset cap +proc finish*(self: PyBytesWriter): PyObject + +template ofPyByteArrayObject*(obj: PyObject): bool = + bind pyByteArrayObjectType + obj.pyType == pyByteArrayObjectType + +type PyByteLike = PyBytesObject or PyByteArrayObject + +proc len*(s: PyByteLike): int {. inline, cdecl .} = s.items.len +proc `$`*(s: PyByteLike): string = s.items +iterator items*(s: PyByteLike): char = + for i in s.items: yield i +proc `[]`*(s: PyByteLike, i: int): char = s.items[i] + +template impl(B){.dirty.} = + + method `$`*(s: `Py B Object`): string = s.items + proc `newPy B`*(s: string = ""): `Py B Object` = + result = `newPy B Simple`() + result.items = s + proc `newPy B`*(size: int): `Py B Object` = + `newPy B` newString size + proc `&`*(s1, s2: `Py B Object`): `Py B Object` = + `newPy B`(s1.items & s2.items) + +impl Bytes +impl ByteArray + + +proc finish*(self: PyBytesWriter): PyObject = + var s: string + s.setLen self.len + when declared(copyMem): + copyMem s[0].addr, self.s[0].addr, self.len + else: + for i, c in self.s: s[i] = c + + if self.use_bytearray: newPyByteArray move s + else: newPyBytes move s + +proc repr*(b: PyBytesObject): string = + 'b' & '\'' & b.items & '\'' # TODO + +proc repr*(b: PyByteArrayObject): string = + "bytearray(" & + 'b' & '\'' & b.items & '\'' #[TODO]# & + ')' +proc `[]=`*(s: PyByteLike, i: int, c: char) = s.items[i] = c + +proc add*(self: PyByteArrayObject, b: PyByteLike) = self.items.add b.items + + + + + + + diff --git a/Objects/byteobjectsImpl.nim b/Objects/byteobjectsImpl.nim new file mode 100644 index 0000000..c05b292 --- /dev/null +++ b/Objects/byteobjectsImpl.nim @@ -0,0 +1,25 @@ + + + +import ./byteobjects +import ./pyobject +import ./[boolobject, numobjects, stringobject, exceptions] + + +export byteobjects + + +template impl(B, mutRead){.dirty.} = + methodMacroTmpl(B) + type `T B` = `Py B Object` + `impl B Magic` eq: + if not other.`ofPy B Object`: + return pyFalseObj + return newPyBool self == `T B`(other) + `impl B Magic` len, mutRead: newPyInt self.len + `impl B Magic` repr, mutRead: newPyAscii(repr self) + `impl B Magic` hash: newPyInt self.items + + +impl Bytes, [] +impl ByteArray, [mutable: read] diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index 12aa7aa..104de6a 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -19,6 +19,7 @@ type Tuple, List, Dict, + Bytes, Str, Code, NimFunc, @@ -96,6 +97,7 @@ type len: UnaryMethod str: UnaryMethod + bytes: UnaryMethod repr: UnaryMethod New: BltinFunc # __new__ is a `staticmethod` in Python diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 7001f00..80dd5cf 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -4,7 +4,9 @@ import neval import builtindict import ./compile import ../Objects/[bundle, typeobject, methodobject, descrobject, funcobject, - notimplementedobject, sliceobjectImpl, dictobjectImpl, exceptions] + notimplementedobject, sliceobjectImpl, dictobjectImpl, exceptions, + byteobjectsImpl, + ] import ../Utils/[utils, macroutils, compat] @@ -161,6 +163,8 @@ registerBltinObject("set", pySetObjectType) registerBltinObject("frozenset", pyFrozenSetObjectType) registerBltinObject("int", pyIntObjectType) registerBltinObject("str", pyStrObjectType) +registerBltinObject("bytes", pyBytesObjectType) +registerBltinObject("bytearray", pyByteArrayObjectType) registerBltinObject("property", pyPropertyObjectType) # not ready to use because no setup code is done when init new types # registerBltinObject("staticmethod", pyStaticMethodObjectType) From 3b4ad372a8a8677eb7367f3b7f98c598468c3561 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 03:31:59 +0800 Subject: [PATCH 135/163] feat(errors): ArithmeticError, OverflowError --- Objects/exceptions.nim | 44 +++++++++++++++++++++++++++++++----------- Python/bltinmodule.nim | 7 +++++-- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 24c9d2d..0464955 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -6,6 +6,8 @@ # we return exception object directly with a thrown flag inside # This might be a bit slower but we are not pursueing ultra performance anyway import std/enumutils +import std/macrocache +export macrocache.items import strformat import pyobject @@ -17,6 +19,7 @@ type ExceptionToken* {. pure .} = enum Name, NotImplemented, Type, + Arithmetic, Attribute, Value, Index, @@ -102,23 +105,22 @@ macro declareErrors: untyped = declareErrors - -template newProcTmpl(excpName) = +template newProcTmpl(excpName, tok){.dirty.} = # use template for lazy evaluation to use PyString # theses two templates are used internally to generate errors (default thrown) - proc `new excpName Error`*: PyBaseErrorObject{.inline.} = + proc `new excpName Error`*: `Py excpName ErrorObject`{.inline.} = let excp = `newPy excpName ErrorSimple`() - excp.tk = ExceptionToken.`excpName` + excp.tk = ExceptionToken.`tok` excp.thrown = true excp - proc `new excpName Error`*(msgStr: PyStrObject): PyBaseErrorObject{.inline.} = - let excp = `newPy excpName ErrorSimple`() - excp.tk = ExceptionToken.`excpName` - excp.thrown = true + proc `new excpName Error`*(msgStr: PyStrObject): `Py excpName ErrorObject`{.inline.} = + let excp = `new excpName Error`() excp.msg = msgStr excp +template newProcTmpl(excpName) = + newProcTmpl(excpName, excpName) macro genNewProcs: untyped = result = newStmtList() @@ -129,15 +131,32 @@ macro genNewProcs: untyped = genNewProcs -template newAttributeError*(tpName, attrName: PyStrObject): PyExceptionObject = +var subErrs*{.compileTime.}: seq[string] +macro declareSubError(E, baseE) = + let + eeS = E.strVal & "Error" + ee = ident eeS + bee = ident baseE.strVal & "Error" + typ = ident "py" & ee.strVal & "ObjectType" + btyp = ident "py" & bee.strVal & "ObjectType" + subErrs.add eeS + result = quote do: + declarePyType `ee`(base(`bee`)): discard + newProcTmpl(`E`, `baseE`) + `typ`.base = `btyp` + `typ`.name = `eeS` + +declareSubError Overflow, Arithmetic + +template newAttributeError*(tpName, attrName: PyStrObject): untyped = let msg = tpName & newPyAscii" has no attribute " & attrName newAttributeError(msg) -template newAttributeError*(tpName, attrName: string): PyExceptionObject = +template newAttributeError*(tpName, attrName: string): untyped = newAttributeError(tpName.newPyStr, attrName.newPyStr) -template newIndexTypeError*(typeName: PyStrObject, obj:PyObject): PyExceptionObject = +template newIndexTypeError*(typeName: PyStrObject, obj:PyObject): untyped = let name = obj.pyType.name let msg = typeName & newPyAscii(" indices must be integers or slices, not ") & newPyStr name newTypeError(msg) @@ -220,3 +239,6 @@ template checkArgNumAtLeast*(expected: int, name="") = else: msg = "expected at least " & $expected & fmt" argument ({args.len} given)" return newTypeError(newPyStr msg) + +proc PyErr_Format*(exc: PyBaseErrorObject, msg: PyStrObject) = + exc.msg = msg diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index 80dd5cf..b305b98 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -174,11 +174,14 @@ macro registerErrors: untyped = result = newStmtList() template registerTmpl(name:string, tp:PyTypeObject) = registerBltinObject(name, tp) + template reg(excpName, typeName: string){.dirty.} = + result.add getAst(registerTmpl(excpName, ident(typeName))) for i in 0..int(ExceptionToken.high): let tok = ExceptionToken(i) let tokenStr = tok.getTokenName let excpName = tok.getBltinName - let typeName = fmt"py{tokenStr}ErrorObjectType" - result.add getAst(registerTmpl(excpName, ident(typeName))) + reg excpName, "py" & tokenStr & "ErrorObjectType" + for s in subErrs: + reg s, "py" & s & "ObjectType" registerErrors From 4d91968022484ed4a944c1568a6e23ed4fb3b654 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 04:00:22 +0800 Subject: [PATCH 136/163] fix(py): replace toInt with toIntOrRetOF to handle overflow; impl PyNumber_[Index,AsSsize_t],PyLong_AsSsize_t --- Objects/listobject.nim | 4 +- Objects/numobjects.nim | 129 +++++++++++++++++++++++++++++++++-- Objects/sliceobject.nim | 19 ++++-- Objects/stringobjectImpl.nim | 4 +- 4 files changed, 142 insertions(+), 14 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 45b8fa2..5e94140 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -134,7 +134,7 @@ implListMethod insert(idx: PyIntObject, item: PyObject), [mutable: write]: elif self.items.len < idx: intIdx = self.items.len else: - intIdx = idx.toInt + intIdx = idx.toIntOrRetOF self.items.insert(item, intIdx) pyNone @@ -152,7 +152,7 @@ implListMethod remove(target: PyObject), [mutable: write]: if retObj.isThrownException: return retObj assert retObj.ofPyIntObject - let idx = PyIntObject(retObj).toInt + let idx = PyIntObject(retObj).toIntOrRetOF self.items.delete(idx) pyNone diff --git a/Objects/numobjects.nim b/Objects/numobjects.nim index 81aa8af..355935c 100644 --- a/Objects/numobjects.nim +++ b/Objects/numobjects.nim @@ -780,18 +780,140 @@ method `$`*(f: PyFloatObject): string = $f.v proc toInt*(pyInt: PyIntObject): int = - # XXX: the caller should take care of overflow - for i in countdown(pyInt.digits.len-1, 0): + ## XXX: the caller should take care of overflow + ## It raises `OverflowDefect` on non-danger build + for i in countdown(pyInt.digits.high, 0): result = result shl digitBits result += int(pyInt.digits[i]) if pyInt.sign == Negative: result *= -1 +const PY_ABS_LONG_MIN = cast[uint](int.low) ## \ +## we cannot use `0u - cast[uint](int.low)` unless with rangeChecks:off + +proc toInt*(pyInt: PyIntObject, overflow: var IntSign): int = + ## if overflow, `overflow` will be `IntSign.Negative` or `IntSign.Positive + ## (depending the sign of the argument) + ## and result be `-1` + ## + ## Otherwise, `overflow` will be `IntSign.Zero` + overflow = Zero + + var x = uint 0 + var prev{.noInit.}: uint + var sign = pyInt.sign + result = -1 + for i in countdown(pyInt.digits.high, 0): + prev = x + x = (x shl digitBits) or uint(pyInt.digits[i]) + if x shr digitBits != prev: + overflow = sign + return + #[ Haven't lost any bits, but casting to long requires extra + care (see comment above).]# + if x <= uint int.high: + result = cast[int](x) * ord(sign) + elif sign == Negative and x == PY_ABS_LONG_MIN: + result = int.low + else: + overflow = sign + +proc toInt*(pyInt: PyIntObject, res: var int): bool = + ## returns false on overflow (`x not_in int.low..int.high`) + var ovf: IntSign + res = pyInt.toInt(ovf) + result = ovf == IntSign.Zero + +proc PyNumber_Index*(item: PyObject): PyObject = + ## returns `PyIntObject` or exception + ## + ## CPython's defined at abstract.c + if item.ofPyIntObject: + return item + let fun = item.getMagic(index) + if fun.isNil: + return newTypeError newPyStr( + fmt"'{item.pyType.name:.200s}' object cannot be interpreted as an integer" + ) + + let i = fun(item) + if not i.ofPyIntObject: + return newTypeError newPyStr( + fmt"__index__ returned non-int (type {item.pyType.name:.200s})" + ) + +proc PyLong_AsSsize_t*(vv: PyIntObject, res: var int): PyOverflowErrorObject = + ## returns nil if not overflow + if not toInt(vv, res): + return newOverflowError( + newPyAscii"Python int too large to convert to C ssize_t") + +proc asLongAndOverflow*(vv: PyIntObject, ovlf: var bool): int{.inline.} = + ## PyLong_AsLongAndOverflow + ovlf = not toInt(vv, result) + +template toIntOrRetOF*(vv: PyIntObject): int = + ## a helper wrapper of `PyLong_AsSsize_t` + ## `return` OverflowError for outer function + var i: int + let ret = PyLong_AsSsize_t(vv, i) + if not ret.isNil: return ret + i + +proc PyLong_AsSsize_t*(v: PyObject, res: var int): PyBaseErrorObject = + if not v.ofPyIntObject: + res = -1 + return newTypeError newPyAscii"an integer is required" + PyLong_AsSsize_t(PyIntObject v, res) + +template PyNumber_AsSsize_tImpl(pyObj: PyObject, res: var int, handleTypeErr, handleValAndOverfMsg){.dirty.} = + let value = PyNumber_Index(pyObj) + if not value.ofPyIntObject: + res = -1 + handleTypeErr PyTypeErrorObject value + else: + let ivalue = PyIntObject value + if not toInt(ivalue, res): + handleValAndOverfMsg ivalue, ( + let tName{.inject.} = pyObj.pyType.name; + newPyStr fmt"cannot fit '{tName:.200s}' into an index-sized integer") + +proc PyNumber_AsSsize_t*(pyObj: PyObject, res: var int): PyExceptionObject = + ## returns nil if no error; otherwise returns TypeError or OverflowError + template handleTypeErr(e: PyTypeErrorObject) = return e + template handleOverfMsg(_; msg: PyStrObject) = return newOverflowError msg + PyNumber_AsSsize_tImpl pyObj, res, handleTypeErr, handleOverfMsg + +proc PyNumber_AsSsize_t*(pyObj: PyObject, res: var PyExceptionObject): int = + ## `res` [inout] + ## + ## CPython's defined at abstract.c + template handleTypeErr(e: PyTypeErrorObject) = res = e + template handleOverfMsg(_; msg: PyStrObject) = + PyErr_Format res, msg + PyNumber_AsSsize_tImpl pyObj, result, handleTypeErr, handleOverfMsg + +proc PyNumber_AsClampedSsize_t*(pyObj: PyObject, res: var int): PyTypeErrorObject = + ## C: `PyNumber_AsSsize_t(pyObj, NULL)` + ## clamp result if overflow + ## + ## returns nil unless Py's TypeError + template handleTypeErr(e: PyTypeErrorObject) = return e + template handleExc(i: PyIntObject; _) = + res = + if i.positive: high int + else: low int + return + PyNumber_AsSsize_tImpl(pyObj, res, handleTypeErr, handleExc) + proc toFloat*(pyInt: PyIntObject): float = parseFloat($pyInt) +proc newPyInt*[C: char](smallInt: C): PyIntObject = + newPyInt int smallInt # TODO + proc newPyInt*[C: Rune|char](str: openArray[C]): PyIntObject = fromStr(str) @@ -1045,8 +1167,7 @@ implFloatMagic hash: # used in list and tuple template getIndex*(obj: PyIntObject, size: int, sizeOpIdx: untyped = `<=`): int = - # todo: if overflow, then thrown indexerror - var idx = obj.toInt + var idx = obj.toIntOrRetOF if idx < 0: idx = size + idx if (idx < 0) or (sizeOpIdx(size, idx)): diff --git a/Objects/sliceobject.nim b/Objects/sliceobject.nim index 7a70684..d460c26 100644 --- a/Objects/sliceobject.nim +++ b/Objects/sliceobject.nim @@ -29,34 +29,41 @@ proc newPySlice*(start, stop, step: PyObject): PyObject = setAttrTmpl(stop) setAttrTmpl(step) - if slice.step.ofPyIntObject and (PyIntObject(slice.step).toInt == 0): + if slice.step.ofPyIntObject and (PyIntObject(slice.step).toIntOrRetOF == 0): return newValueError newPyAscii("slice step cannot be zero") slice # slice.indices defined in ./sliceobjectImpl -proc calLen*(self: PySliceObject): int = +proc calLen*(self: PySliceObject, res: var int): PyOverflowErrorObject = ## Get the length of the slice. ## .. note:: python's slice has no `__len__`. ## this is just a convenience method for internal use. template intOrNone(obj: PyObject, defaultValue: int): int = if obj.ofPyIntObject: - PyIntObject(obj).toInt + PyIntObject(obj).toIntOrRetOF else: defaultValue - rangeLen[int]( + res=rangeLen[int]( intOrNone(self.start, 0), intOrNone(self.stop, 0), intOrNone(self.step, 1) ) +template calLenOrRetOF*(self: PySliceObject): int = + bind calLen + var res: int + let ret = self.calLen res + if not ret.isNil: + return ret + res + proc getSliceItems*[T](slice: PySliceObject, src: openArray[T], dest: var (seq[T]|string)): PyObject = var start, stop, step: int let stepObj = slice.step if stepObj.ofPyIntObject: - # todo: overflow - step = PyIntObject(stepObj).toInt + step = PyIntObject(stepObj).toIntOrRetOF else: assert stepObj.ofPyNoneObject step = 1 diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index 994c444..96f0e39 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -117,10 +117,10 @@ when true: newObj: PyStrObject retObj: PyObject if self.str.ascii: - newObj = newPyString newAsciiUnicodeVariantOfCap slice.calLen + newObj = newPyString newAsciiUnicodeVariantOfCap slice.calLenOrRetOF retObj = tgetSliceItems(asciiStr) else: - newObj = newPyString newUnicodeUnicodeVariantOfCap slice.calLen + newObj = newPyString newUnicodeUnicodeVariantOfCap slice.calLenOrRetOF retObj = tgetSliceItems(unicodeStr) if retObj.isThrownException: return retObj From 9bfd07e615b01a4c0891cec8cc675354e70558da Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 04:01:15 +0800 Subject: [PATCH 137/163] feat(syntax): sup b/B prefix --- Python/ast.nim | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index 6f465e0..812124b 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -8,7 +8,7 @@ import strformat import asdl import ../Parser/[token, parser] import ../Objects/[pyobject, noneobject, - numobjects, boolobjectImpl, stringobjectImpl, + numobjects, boolobjectImpl, stringobjectImpl, byteobjects, sliceobject # pyEllipsis ] import ../Utils/[utils, compat] @@ -1016,6 +1016,14 @@ proc astAtomExpr(parseNode: ParseNode): AsdlExpr = # '[' [testlist_comp] ']' | # '{' [dictorsetmaker] '}' | # NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') + +template resStrOrBytesVia(newObj){.dirty.} = + var str: string + for child in parseNode.children: + str.add(child.tokenNode.content) + let pyString = newObj(str) + result = newAstConstant(pyString) + ast atom, [AsdlExpr]: let child1 = parseNode.children[0] case child1.tokenNode.token @@ -1082,11 +1090,10 @@ ast atom, [AsdlExpr]: result = newAstConstant(pyInt) of Token.STRING: - var str: string - for child in parseNode.children: - str.add(child.tokenNode.content) - let pyString = newPyString(str) - result = newAstConstant(pyString) + resStrOrBytesVia newPyString + + of Token.BYTES: + resStrOrBytesVia newPyBytes of Token.True: result = newAstConstant(pyTrueObj) From 30a1f16b59668c0581617617bc8609f9d24198e6 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 04:03:55 +0800 Subject: [PATCH 138/163] feat(bytes): `__new__` --- Objects/abstract.nim | 36 ++++++++++++++++++++++++++++++ Objects/byteobjects.nim | 44 +++++++++++++++++++++++++++++++++---- Objects/byteobjectsImpl.nim | 34 +++++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 Objects/abstract.nim diff --git a/Objects/abstract.nim b/Objects/abstract.nim new file mode 100644 index 0000000..0728111 --- /dev/null +++ b/Objects/abstract.nim @@ -0,0 +1,36 @@ + +import std/strformat +import ./pyobject +import ./numobjects +import ./[iterobject, exceptions, stringobject] +export PyNumber_Index, PyNumber_AsSsize_t, PyNumber_AsClampedSsize_t + + +template PySequence_Check*(o: PyObject): bool = + ## PY-DIFF: we check whether o has items: seq[PyObject] + when not compiles(o.items): false + else: o.items is seq[PyObject] +template ifPySequence_Check*(o: PyObject, body) = + when PySequence_Check(o): body +template ifPySequence_Check*(o: PyObject, body, elseDo): untyped = + when PySequence_Check(o): body + else: elseDo + +proc PyIter_Check*(obj: PyObject): bool = + let t = obj.getMagic(iternext) + not t.isNil # PY-DIFF: never be _PyObject_NextNotImplemented + +template PyObject_GetIter*(o: PyObject): PyObject = + bind newTypeError, newPyStr, getMagic, newPySeqIter + bind fmt, formatValue + bind ifPySequence_Check + let f = o.getMagic(iter) + if f.isNil: + ifPySequence_Check(o): + newPySeqIter(o.items) + do: + let n{.inject.} = o.pyType.name + newTypeError newPyStr( + fmt"'{n:.200s}' object is not iterable" + ) + else: f(o) diff --git a/Objects/byteobjects.nim b/Objects/byteobjects.nim index 712d80b..89f9f49 100644 --- a/Objects/byteobjects.nim +++ b/Objects/byteobjects.nim @@ -1,7 +1,7 @@ ## bytesobject and bytesarrayobject import std/strformat import ./pyobject - +import ./abstract import ./[listobject, tupleobject, stringobject, exceptions, iterobject] declarePyType Bytes(tpToken): @@ -77,9 +77,45 @@ proc `[]=`*(s: PyByteLike, i: int, c: char) = s.items[i] = c proc add*(self: PyByteArrayObject, b: PyByteLike) = self.items.add b.items - - - +template genFromIter(S; T; forInLoop; getLenHint: untyped=len){.dirty.} = + proc `PyBytes_From S`*(x: T): PyObject = + let size = x.getLenHint + var writer = initPyBytesWriter size + var value: int + forInLoop i, x: + let ret = PyNumber_AsClampedSsize_t(i, value) + if not ret.isNil: + return ret + if value < 0 or value > 256: + return newValueError newPyAscii"bytes must be in range(0, 256)" + writer.add cast[char](value) + writer.finish + +template sysForIn(x, it, body){.dirty.} = + for x in it: body +genFromIter List, PyListObject, sysForIn +genFromIter Tuple, PyTupleObject, sysForIn +template getLenHint(x): int = 64 # TODO +genFromIter Iterator, PyObject, pyForIn, getLenHint + + +proc PyBytes_FromObject*(x: PyObject): PyObject = + if x.pyType == pyBytesObjectType: return x + # TODO + #[ /* Use the modern buffer interface */ + if (PyObject_CheckBuffer(x)) + return _PyBytes_FromBuffer(x);]# + if x.pyType == pyListObjectType: return PyBytes_FromList PyListObject x + if x.pyType == pyTupleObjectType: return PyBytes_FromTuple PyTupleObject x + if not x.ofPyStrObject: + let it = PyObject_GetIter(x) + if not it.isNil: + return PyBytes_FromIterator(it) + if not it.isExceptionOf Type: + return it + return newTypeError newPyStr( + fmt"cannot convert '{x.pyType.name:.200s}' object to bytes" + ) diff --git a/Objects/byteobjectsImpl.nim b/Objects/byteobjectsImpl.nim index c05b292..6893589 100644 --- a/Objects/byteobjectsImpl.nim +++ b/Objects/byteobjectsImpl.nim @@ -1,5 +1,5 @@ - +import std/strformat import ./byteobjects import ./pyobject @@ -23,3 +23,35 @@ template impl(B, mutRead){.dirty.} = impl Bytes, [] impl ByteArray, [mutable: read] + +# TODO: encoding, errors params +implBytesMagic New(tp: PyObject, x: PyObject): + var bytes: PyObject + var fun: UnaryMethod + fun = x.getMagic(bytes) + if not fun.isNil: + result = fun(x) + if not result.ofPyBytesObject: + return newTypeError newPyString( + &"__bytes__ returned non-bytes (type {result.pyType.name:.200s})") + return + + if x.ofPyStrObject: + return newTypeError newPyAscii"string argument without an encoding" + # Is it an integer? + fun = x.getMagic(index) + if not fun.isNil: + var size: int + result = PyNumber_AsSsize_t(x, size) + if size == -1 and result.isThrownException: + if not result.isExceptionOf Type: + return # OverflowError + bytes = PyBytes_FromObject x + else: + if size < 0: + return newValueError newPyAscii"negative count" + bytes = newPyBytes size + else: + bytes = PyBytes_FromObject x + return bytes + From a63b9cc15d5fba070013a80f07b8a3030add6bf5 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 19:55:27 +0800 Subject: [PATCH 139/163] break(bytearray): internally use seq[char] over string --- Objects/byteobjects.nim | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Objects/byteobjects.nim b/Objects/byteobjects.nim index 89f9f49..6a48995 100644 --- a/Objects/byteobjects.nim +++ b/Objects/byteobjects.nim @@ -8,7 +8,7 @@ declarePyType Bytes(tpToken): items: string declarePyType ByteArray(reprLock, mutable): - items: string + items: seq[char] type PyBytesWriter* = object #overallocate*: bool @@ -32,46 +32,47 @@ template ofPyByteArrayObject*(obj: PyObject): bool = bind pyByteArrayObjectType obj.pyType == pyByteArrayObjectType +proc `$`(self: seq[char]): string = + result.setLen self.len + when declared(copyMem): + if self.len > 0: + copyMem result[0].addr, self[0].addr, self.len + else: + for i, c in self: result[i] = c + type PyByteLike = PyBytesObject or PyByteArrayObject proc len*(s: PyByteLike): int {. inline, cdecl .} = s.items.len -proc `$`*(s: PyByteLike): string = s.items +proc `$`*(s: PyByteLike): string = $s.items iterator items*(s: PyByteLike): char = for i in s.items: yield i proc `[]`*(s: PyByteLike, i: int): char = s.items[i] -template impl(B){.dirty.} = +template impl(B, InitT, newTOfCap){.dirty.} = - method `$`*(s: `Py B Object`): string = s.items - proc `newPy B`*(s: string = ""): `Py B Object` = + method `$`*(s: `Py B Object`): string = $s.items + proc `newPy B`*(s: InitT = default InitT): `Py B Object` = result = `newPy B Simple`() result.items = s proc `newPy B`*(size: int): `Py B Object` = - `newPy B` newString size + `newPy B` newTOfCap size proc `&`*(s1, s2: `Py B Object`): `Py B Object` = `newPy B`(s1.items & s2.items) -impl Bytes -impl ByteArray +impl Bytes, string, newString +impl ByteArray, seq[char], newSeq[char] proc finish*(self: PyBytesWriter): PyObject = - var s: string - s.setLen self.len - when declared(copyMem): - copyMem s[0].addr, self.s[0].addr, self.len - else: - for i, c in self.s: s[i] = c - - if self.use_bytearray: newPyByteArray move s - else: newPyBytes move s + if self.use_bytearray: newPyByteArray self.s + else: newPyBytes $self.s proc repr*(b: PyBytesObject): string = 'b' & '\'' & b.items & '\'' # TODO proc repr*(b: PyByteArrayObject): string = "bytearray(" & - 'b' & '\'' & b.items & '\'' #[TODO]# & + 'b' & '\'' & $b.items & '\'' #[TODO]# & ')' proc `[]=`*(s: PyByteLike, i: int, c: char) = s.items[i] = c @@ -118,4 +119,3 @@ proc PyBytes_FromObject*(x: PyObject): PyObject = ) - From c7301aaafc932a4d11c3197715352fc80bca896b Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 20:28:05 +0800 Subject: [PATCH 140/163] impr(iterobject): pyForIn `return` PyBaseErrorObject over PyObject --- Objects/iterobject.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/iterobject.nim b/Objects/iterobject.nim index 76f176a..72f3b4c 100644 --- a/Objects/iterobject.nim +++ b/Objects/iterobject.nim @@ -21,15 +21,16 @@ proc newPySeqIter*(items: seq[PyObject]): PySeqIterObject = template pyForIn*(it; iterableToLoop: PyObject; doWithIt) = ## pesudo code: `for it in iterableToLoop: doWithIt` + ## but `return` PyBaseErrorObject if python's exception is raised let (iterable, nextMethod) = getIterableWithCheck(iterableToLoop) if iterable.isThrownException: - return iterable + return PyBaseErrorObject iterable while true: let it = nextMethod(iterable) if it.isStopIter: break if it.isThrownException: - return it + return PyBaseErrorObject it doWithIt From 252979eec16b9219945af6b3739896a9b0e89bc7 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 21:54:03 +0800 Subject: [PATCH 141/163] feat(builtins): `bytearray.__init__` --- Objects/byteobjects.nim | 88 +++++++++++++++++++++++++++---------- Objects/byteobjectsImpl.nim | 62 ++++++++++++++++++-------- 2 files changed, 107 insertions(+), 43 deletions(-) diff --git a/Objects/byteobjects.nim b/Objects/byteobjects.nim index 6a48995..9f2ebcb 100644 --- a/Objects/byteobjects.nim +++ b/Objects/byteobjects.nim @@ -1,9 +1,10 @@ ## bytesobject and bytesarrayobject import std/strformat +import std/hashes import ./pyobject import ./abstract import ./[listobject, tupleobject, stringobject, exceptions, iterobject] - +import ./numobjects declarePyType Bytes(tpToken): items: string @@ -26,7 +27,7 @@ proc reset*(self: var PyBytesWriter, cap: int=0) = proc initPyBytesWriter*(cap: int): PyBytesWriter = result = initPyBytesWriter() result.reset cap -proc finish*(self: PyBytesWriter): PyObject +proc finish*(self: sink PyBytesWriter): PyObject template ofPyByteArrayObject*(obj: PyObject): bool = bind pyByteArrayObjectType @@ -43,9 +44,12 @@ proc `$`(self: seq[char]): string = type PyByteLike = PyBytesObject or PyByteArrayObject proc len*(s: PyByteLike): int {. inline, cdecl .} = s.items.len +proc hash*(s: PyByteLike): Hash {. inline, cdecl .} = hash s.items proc `$`*(s: PyByteLike): string = $s.items iterator items*(s: PyByteLike): char = for i in s.items: yield i +iterator ints*(s: PyByteLike): PyIntObject = + for i in s: yield newPyInt i proc `[]`*(s: PyByteLike, i: int): char = s.items[i] template impl(B, InitT, newTOfCap){.dirty.} = @@ -63,9 +67,13 @@ impl Bytes, string, newString impl ByteArray, seq[char], newSeq[char] -proc finish*(self: PyBytesWriter): PyObject = - if self.use_bytearray: newPyByteArray self.s - else: newPyBytes $self.s +proc finish*(self: sink PyBytesWriter): PyObject = + if self.use_bytearray: newPyByteArray move self.s + else: newPyBytes $(move self.s) + +proc finish*(self: sink PyBytesWriter, res: PyObject) = + if self.use_bytearray: PyByteArrayObject(res).items = move self.s + else: PyBytesObject(res).items = $(move self.s) proc repr*(b: PyBytesObject): string = 'b' & '\'' & b.items & '\'' # TODO @@ -77,20 +85,28 @@ proc repr*(b: PyByteArrayObject): string = proc `[]=`*(s: PyByteLike, i: int, c: char) = s.items[i] = c proc add*(self: PyByteArrayObject, b: PyByteLike) = self.items.add b.items +proc setLen*(self: PyByteArrayObject, n: int) = self.items.setLen n + +template fillFromIterable(writer: PyBytesWriter; x; forInLoop; errSubject: string) = + var value: int + forInLoop i, x: + let ret = PyNumber_AsClampedSsize_t(i, value) + if not ret.isNil: + return ret + if value < 0 or value > 256: + return newValueError newPyAscii(errSubject & " must be in range(0, 256)" & " not " & $value) + writer.add cast[char](value) template genFromIter(S; T; forInLoop; getLenHint: untyped=len){.dirty.} = - proc `PyBytes_From S`*(x: T): PyObject = - let size = x.getLenHint - var writer = initPyBytesWriter size - var value: int - forInLoop i, x: - let ret = PyNumber_AsClampedSsize_t(i, value) - if not ret.isNil: - return ret - if value < 0 or value > 256: - return newValueError newPyAscii"bytes must be in range(0, 256)" - writer.add cast[char](value) + proc `PyBytes_From S`(x: T): PyObject = + var writer = initPyBytesWriter x.getLenHint + writer.fillFromIterable(x, forInLoop, "bytes") writer.finish + proc `initFrom S`(self: PyByteArrayObject, x: T): PyBaseErrorObject = + var writer = initPyBytesWriter x.getLenHint + writer.use_bytearray = true + writer.fillFromIterable(x, forInLoop, "byte") + writer.finish self template sysForIn(x, it, body){.dirty.} = for x in it: body @@ -99,23 +115,47 @@ genFromIter Tuple, PyTupleObject, sysForIn template getLenHint(x): int = 64 # TODO genFromIter Iterator, PyObject, pyForIn, getLenHint - -proc PyBytes_FromObject*(x: PyObject): PyObject = - if x.pyType == pyBytesObjectType: return x +template fillFromObject(x: PyObject){.dirty.} = + mixin fromList, fromTuple, fromIterator # TODO #[ /* Use the modern buffer interface */ if (PyObject_CheckBuffer(x)) return _PyBytes_FromBuffer(x);]# - if x.pyType == pyListObjectType: return PyBytes_FromList PyListObject x - if x.pyType == pyTupleObjectType: return PyBytes_FromTuple PyTupleObject x + if x.pyType == pyListObjectType: fromList x + if x.pyType == pyTupleObjectType: fromTuple x if not x.ofPyStrObject: let it = PyObject_GetIter(x) - if not it.isNil: - return PyBytes_FromIterator(it) + if not it.isThrownException: + fromIterator it if not it.isExceptionOf Type: - return it + return PyBaseErrorObject it return newTypeError newPyStr( fmt"cannot convert '{x.pyType.name:.200s}' object to bytes" ) +template genFrom(ls, tup, itor){.dirty.} = + template fromList(x) = ls + template fromTuple(x) = tup + template fromIterator(it) = itor +proc PyBytes_FromObject*(x: PyObject): PyObject = + if x.pyType == pyBytesObjectType: return x + genFrom: return PyBytes_FromList PyListObject x + do: return PyBytes_FromTuple PyTupleObject x + do: return PyBytes_FromIterator(it) + fillFromObject x + +proc initFromObject*(self: PyByteArrayObject, x: PyObject): PyBaseErrorObject = + template retOnE(exp: PyBaseErrorObject) = + let e = exp + if not e.isNil: return e + else: return + genFrom: retOnE self.initFromList PyListObject x + do: retOnE self.initFromTuple PyTupleObject x + do: retOnE self.initFromIterator(it) + fillFromObject x + +proc PyByteArray_FromObject*(x: PyObject): PyObject = + let self = newPyByteArray() + result = self.initFromObject x + if result.isNil: return self diff --git a/Objects/byteobjectsImpl.nim b/Objects/byteobjectsImpl.nim index 6893589..52ac2d0 100644 --- a/Objects/byteobjectsImpl.nim +++ b/Objects/byteobjectsImpl.nim @@ -3,7 +3,9 @@ import std/strformat import ./byteobjects import ./pyobject -import ./[boolobject, numobjects, stringobject, exceptions] +import ./[boolobject, numobjects, stringobject, exceptions, noneobject, + iterobject, +] export byteobjects @@ -18,40 +20,62 @@ template impl(B, mutRead){.dirty.} = return newPyBool self == `T B`(other) `impl B Magic` len, mutRead: newPyInt self.len `impl B Magic` repr, mutRead: newPyAscii(repr self) - `impl B Magic` hash: newPyInt self.items + `impl B Magic` hash: newPyInt self.hash + `impl B Magic` iter, mutRead: + genPyNimIteratorIter self.ints impl Bytes, [] impl ByteArray, [mutable: read] -# TODO: encoding, errors params -implBytesMagic New(tp: PyObject, x: PyObject): - var bytes: PyObject - var fun: UnaryMethod - fun = x.getMagic(bytes) - if not fun.isNil: - result = fun(x) - if not result.ofPyBytesObject: - return newTypeError newPyString( - &"__bytes__ returned non-bytes (type {result.pyType.name:.200s})") - return - +template impl(x, fromSize, fromObject) = if x.ofPyStrObject: return newTypeError newPyAscii"string argument without an encoding" # Is it an integer? - fun = x.getMagic(index) + let fun = x.getMagic(index) if not fun.isNil: var size: int result = PyNumber_AsSsize_t(x, size) if size == -1 and result.isThrownException: - if not result.isExceptionOf Type: + if not result.isExceptionOf ExceptionToken.Type: return # OverflowError - bytes = PyBytes_FromObject x + fromObject x else: if size < 0: return newValueError newPyAscii"negative count" - bytes = newPyBytes size + fromSize size else: - bytes = PyBytes_FromObject x + fromObject x + +# TODO: encoding, errors params +implBytesMagic New(tp: PyObject, x: PyObject): + var bytes: PyObject + let fun = x.getMagic(bytes) + if not fun.isNil: + result = fun(x) + if not result.ofPyBytesObject: + return newTypeError newPyString( + &"__bytes__ returned non-bytes (type {result.pyType.name:.200s})") + return + + template fromSize(size) = bytes = newPyBytes size + template fromObject(o) = bytes = PyBytes_FromObject o + impl x, fromSize, fromObject return bytes + +# TODO: encoding, errors params +implByteArrayMagic init: + if args.len == 0: + return pyNone + checkArgNum 1 # TODO + let x = args[0] + if self.items.len != 0: + self.items.setLen(0) + + template fromSize(size) = self.setLen size + template fromObject(o) = + let e = self.initFromObject o + if not e.isNil: return e + impl x, fromSize, fromObject + pyNone From 7213b8a8dd16c5c53bd20f36150fdcdcf2c0b600 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 30 Jul 2025 23:37:19 +0800 Subject: [PATCH 142/163] feat(inner): followup 00393460da2f3048579: sup stop arg for Utils/`sequtils.find*`... ...sup x: T in addition to x: openArray[T] --- Objects/stringobjectImpl.nim | 5 ++-- Utils/sequtils.nim | 53 ++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index 96f0e39..7a90b41 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -87,8 +87,9 @@ proc newPyStrIter*(s: PyStrObject): PyStrIterObject = else: proc(i: int): PyStrObject = newPyString s.str.unicodeStr[i] -template findExpanded(it1, it2): int = it1.findWithoutMem(it2, key=uint32) -template findAllExpanded(it1, it2): untyped = it1.findAllWithoutMem(it2, key=uint32) +template findExpanded(it1, it2): int = uint32.findWithoutMem(it1, it2) +iterator findAllExpanded[A, B](it1: A, it2: B): int = + for i in uint32.findAllWithoutMem(it1, it2): yield i when true: # copied and modified from ./tupleobject.nim diff --git a/Utils/sequtils.nim b/Utils/sequtils.nim index 0c5d8de..20f2223 100644 --- a/Utils/sequtils.nim +++ b/Utils/sequtils.nim @@ -1,9 +1,12 @@ - +## Nim lacks `find` with start, stop param +## +## .. note:: functions in this module assume `start..= 0: yield i + else: break + + +wrapOA findAll proc contains*[T](s, sub: openArray[T]): bool{.inline.} = s.find(sub) > 0 From 70bd0a770ff3b093b01ab9619d3290c8c54fcf36 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 02:59:49 +0800 Subject: [PATCH 143/163] feat(inner): abstract: add helpers: `numAs*OrE`, `*OptArgAt` --- Objects/abstract.nim | 33 ++++++++++++++++++++++++++++++++- Objects/numobjects.nim | 8 ++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Objects/abstract.nim b/Objects/abstract.nim index 0728111..d87f67e 100644 --- a/Objects/abstract.nim +++ b/Objects/abstract.nim @@ -5,7 +5,38 @@ import ./numobjects import ./[iterobject, exceptions, stringobject] export PyNumber_Index, PyNumber_AsSsize_t, PyNumber_AsClampedSsize_t - + +template optionalTLikeArg[T](args; i: int, def: T; mapper): T = + if args.len > i: mapper args[i] + else: def + +template numAsIntOrRetE*(x: PyObject): int = + ## interpret int or int-able object `x` to `system.int` + var res: int + let e = x.PyNumber_AsSsize_t res + if not e.isNil: + return e + res + +template numAsClampedIndexOrRetE*(x: PyObject; size: int): int = + ## interpret int or int-able object `x` to `system.int`, clamping result in `0..] = None` + bind optionalTLikeArg, numAsIntOrRetE + optionalTLikeArg(args, i, def, numAsIntOrRetE) + +template clampedIndexOptArgAt*(args: seq[PyObject]; i: int, def: int, size: int): int = + ## parse arg `x: Optional[] = None`, clamped result in `0.. Date: Thu, 31 Jul 2025 03:02:03 +0800 Subject: [PATCH 144/163] feat(inner): pyobject: checkTypeOrRetTE, castTypeOrRetTE --- Objects/pyobject.nim | 51 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index 6eb3239..d203d42 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -202,18 +202,51 @@ proc objName2tpObjName(objName: string): string {. compileTime .} = result = objName & "Type" result[0] = result[0].toLowerAscii -# example here: For a definition like `i: PyIntObject` -# obj: i -# tp: PyIntObject like -# tpObj: pyIntObjectType like -template checkTypeTmpl(obj, tp, tpObj, methodName) = +template typeName*(o: PyObject): string = + o.pyType.name + +template checkTypeTmplImpl(obj: PyObject{atom}, tp, tpObjName; msgInner="") {.dirty.} = # should use a more sophisticated way to judge type if not (obj of tp): - let expected {. inject .} = tpObj.name - let got {. inject .}= obj.pyType.name - let mName {. inject .}= methodName - let msg = fmt"{expected} is requred for {mName} (got {got})" + let expected = tpObjName + let got = obj.typeName + let tmsgInner = msgInner + let msg = fmt"{expected} is requred{tmsgInner} (got {got})" return newTypeError newPyStr(msg) +template checkTypeTmplImpl(obj: PyObject, tp, tpObjName; msgInner="") = + bind checkTypeTmplImpl + let tobj = obj + checkTypeTmplImpl(tobj, tp, tpObjName, msgInner) + +template checkTypeTmpl(obj, tp, tpObj, methodName) = + checkTypeTmplImpl(obj, tp, tpObj.name, " for" & methodName) + +template checkTypeOrRetTE*(obj, tp; tpObj: PyTypeObject; methodName: string) = + ## example here: For a definition like `i: PyIntObject` + ## obj: i + ## tp: PyIntObject like + ## tpObj: pyIntObjectType like + bind checkTypeTmpl + checkTypeTmpl obj, tp, tpObj, methodName + +template checkTypeOrRetTE*(obj, tp; tpObj: PyTypeObject) = + bind checkTypeTmpl + checkTypeTmplImpl obj, tp, tpObj.name + +macro toTypeObject*[O: PyObject](tp: typedesc[O]): PyTypeObject = + ident ($tp).toLowerAscii & "Type" + +template checkTypeOrRetTE*(obj, tp) = + bind toTypeObject + checkTypeOrRetTE obj, tp, toTypeObject(tp) + +macro call2AndMoreArg(callee, arg1, arg2, more): untyped = + result = newCall(callee, arg1, arg2) + for i in more: result.add i + +template castTypeOrRetTE*[O: PyObject](obj: PyObject, tp: typedesc[O]; extraArgs: varargs[untyped]): O = + call2AndMoreArg(checkTypeOrRetTE, obj, tp, extraArgs) + cast[tp](obj) proc isSeqObject(n: NimNode): bool = n.kind == nnkBracketExpr and n[0].eqIdent"seq" and n[1].eqIdent"PyObject" From ba0f77ca6aba55163d28f939e1fc8c34fe48d483 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 03:03:28 +0800 Subject: [PATCH 145/163] feat(builtins): str: sup start, end param for index,count,find --- Objects/stringobjectImpl.nim | 62 +++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index 7a90b41..82fbe97 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -5,6 +5,7 @@ import baseBundle import stringobject import ./sliceobject import ../Utils/sequtils +import ./abstract export stringobject @@ -87,9 +88,31 @@ proc newPyStrIter*(s: PyStrObject): PyStrIterObject = else: proc(i: int): PyStrObject = newPyString s.str.unicodeStr[i] -template findExpanded(it1, it2): int = uint32.findWithoutMem(it1, it2) -iterator findAllExpanded[A, B](it1: A, it2: B): int = - for i in uint32.findAllWithoutMem(it1, it2): yield i +proc findExpanded[A, B](it1: A, it2: B; start=0, stop = it1.len): int{.inline.} = + uint32.findWithoutMem(it1, it2, start, stop) +iterator findAllExpanded[A, B](it1: A, it2: B, start=0, stop = it1.len): int = + for i in uint32.findAllWithoutMem(it1, it2, start, stop): yield i + +template implMethodGenTargetAndStartStop*(castTarget) {.dirty.} = + checkArgNumAtLeast 1 + let le = self.len + let + target = castTarget args[0] + start = args.clampedIndexOptArgAt(1, 0, le) + stop = args.clampedIndexOptArgAt(2, le,le) + +template asIs(x): untyped = x +template implMethodGenTargetAndStartStop* {.dirty.} = + bind asIs + implMethodGenTargetAndStartStop(asIs) + +template implMethodGenStrTargetAndStartStop = + template castToStr(x): untyped = x.castTypeOrRetTE PyStrObject + implMethodGenTargetAndStartStop castToStr + +template doFind(cb): untyped = + ## helper to avoid too much `...it2, start, stop)` code snippet + cb(it1, it2, start, stop) when true: # copied and modified from ./tupleobject.nim @@ -129,32 +152,35 @@ when true: return newObj - implStrMethod index(target: PyStrObject): + implStrMethod index: + implMethodGenStrTargetAndStartStop let res = doKindsWith2It(self.str, target.str): - sequtils.find(it1, it2) - it1.findExpanded(it2) - it1.findExpanded(it2) - sequtils.find(it1, it2) + doFind find + doFind findExpanded + doFind findExpanded + doFind find if res >= 0: return newPyInt(res) let msg = "substring not found" newValueError(newPyAscii msg) - implStrMethod count(target: PyStrObject): + implStrMethod count: + implMethodGenStrTargetAndStartStop var count: int template cntAll(it) = for _ in it: count.inc doKindsWith2It(self.str, target.str): - cntAll it1.findAll(it2) - cntAll it1.findAllExpanded(it2) - cntAll it1.findAllExpanded(it2) - cntAll it1.findAll(it2) + cntAll doFind findAll + cntAll doFind findAllExpanded + cntAll doFind findAllExpanded + cntAll doFind findAll newPyInt(count) -implStrMethod find(target: PyStrObject): +implStrMethod find: + implMethodGenStrTargetAndStartStop let res = doKindsWith2It(self.str, target.str): - sequtils.find(it1, it2) - it1.findExpanded(it2) - it1.findExpanded(it2) - sequtils.find(it1, it2) + doFind find + doFind findExpanded + doFind findExpanded + doFind find newPyInt(res) From cd958d7941b038b2c7ecbe544ca6e1c220ea5105 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 03:11:59 +0800 Subject: [PATCH 146/163] refact(tuple): exp genGetItem, rm unused param `newPyNameSimple` of `genCollectMagics`;... ...ren hash field to privateHash --- Objects/setobject.nim | 4 ++-- Objects/tupleobject.nim | 46 +++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Objects/setobject.nim b/Objects/setobject.nim index 60739b6..ac31e54 100644 --- a/Objects/setobject.nim +++ b/Objects/setobject.nim @@ -19,7 +19,7 @@ declarePyType Set(reprLock, mutable, tpToken): declarePyType FrozenSet(reprLock, tpToken): items: HashSet[PyObject] setHash: bool - hash: Hash + privateHash: Hash template setSeqToStr(ss): string = if ss.len == 0: @@ -78,7 +78,7 @@ template genSet(S, setSeqToStr, mutRead, mutReadRepr){.dirty.} = newL genCollectMagics items, - `impl S Magic`, `newPy S Simple`, + `impl S Magic`, `ofPy S Object`, `Py S Object`, mutRead, mutReadRepr, setSeqToStr diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 499f264..9415c43 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -11,7 +11,7 @@ import sliceobject declarePyType Tuple(reprLock, tpToken): items: seq[PyObject] setHash: bool - hash: Hash + privateHash: Hash proc newPyTuple*(items: seq[PyObject]): PyTupleObject = @@ -23,7 +23,7 @@ proc newPyTuple*(items: openArray[PyObject]): PyTupleObject{.inline.} = newPyTuple @items template genCollectMagics*(items, - implNameMagic, newPyNameSimple, + implNameMagic, ofPyNameObject, PyNameObject, mutRead, mutReadRepr, seqToStr){.dirty.} = @@ -56,21 +56,37 @@ template genCollectMagics*(items, implNameMagic len, mutRead: newPyInt(self.len) +template genGetitem*(nameStr, implNameMagic, newPyName, mutRead; getter: untyped = `[]`){.dirty.} = + bind ofPySliceObject, getIndex, PySliceObject, getSliceItems + implNameMagic getitem, mutRead: + if other.ofPyIntObject: + let idx = getIndex(PyIntObject(other), self.len) + return getter(self, idx) + if ofPySliceObject(other): + let slice = PySliceObject(other) + let newObj = newPyName() + let retObj = getSliceItems(slice, self.items, newObj.items) + if retObj.isThrownException: + return retObj + else: + return newObj + + return newIndexTypeError(newPyStr nameStr, other) template genSequenceMagics*(nameStr, implNameMagic, implNameMethod; ofPyNameObject, PyNameObject, - newPyNameSimple; mutRead, mutReadRepr; + newPyName; mutRead, mutReadRepr; seqToStr; initWithDictUsingPairs=false): untyped{.dirty.} = bind genCollectMagics genCollectMagics items, - implNameMagic, newPyNameSimple, + implNameMagic, ofPyNameObject, PyNameObject, mutRead, mutReadRepr, seqToStr implNameMagic add, mutRead: - var res = newPyNameSimple() + var res = newPyName() if other.ofPyNameObject: res.items = self.items & PyNameObject(other).items return res @@ -118,21 +134,7 @@ template genSequenceMagics*(nameStr, implNameMagic iter, mutRead: newPySeqIter(self.items) - - implNameMagic getitem: - if other.ofPyIntObject: - let idx = getIndex(PyIntObject(other), self.len) - return self[idx] - if other.ofPySliceObject: - let slice = PySliceObject(other) - let newObj = newPyNameSimple() - let retObj = slice.getSliceItems(self.items, newObj.items) - if retObj.isThrownException: - return retObj - else: - return newObj - - return newIndexTypeError(newPyStr nameStr, other) + genGetitem nameStr, implNameMagic, newPyName, mutRead implNameMethod index(target: PyObject), mutRead: for idx, item in self.items: @@ -182,9 +184,9 @@ template hashCollectionImpl*(items; hashForEmpty): Hash = !$result proc hashCollection*[T: PyObject](self: T): Hash = - if self.setHash: return self.hash + if self.setHash: return self.privateHash result = self.items.hashCollectionImpl Hash self.pyType.id - self.hash = result + self.privateHash = result self.setHash = true proc hash*(self: PyTupleObject): Hash = self.hashCollection From a4bf86d7fbcf30bfa9b865c40b318370d6ca1f3a Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 04:38:06 +0800 Subject: [PATCH 147/163] feat(builtins/sequence): sup magic iadd,mul,imul --- Objects/listobject.nim | 241 ++++++++++++++++++++++------------------ Objects/tupleobject.nim | 20 +++- 2 files changed, 150 insertions(+), 111 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 5e94140..cdea236 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -27,132 +27,155 @@ template lsSeqToStr(ss): string = '[' & ss.join", " & ']' genSequenceMagics "list", implListMagic, implListMethod, ofPyListObject, PyListObject, - newPyListSimple, [mutable: read], [reprLockWithMsg"[...]", mutable: read], + newPyList, [mutable: read], [reprLockWithMsg"[...]", mutable: read], lsSeqToStr, initWithDictUsingPairs=true -implListMagic setitem, [mutable: write]: - if arg1.ofPyIntObject: - let idx = getIndex(PyIntObject(arg1), self.len) - self.items[idx] = arg2 - return pyNone - if arg1.ofPySliceObject: - let slice = arg1.PySliceObject - let iterableToLoop = arg2 - case slice.stepAsInt - of 1, -1: - var ls: seq[PyObject] - pyForIn it, iterableToLoop: - ls.add it - self.items[slice.toNimSlice(self.len)] = ls - else: - let (iterable, nextMethod) = getIterableWithCheck(iterableToLoop) - if iterable.isThrownException: - return iterable - for i in iterInt(slice, self.len): - let it = nextMethod(iterable) - if it.isStopIter: - break - if it.isThrownException: - return it - self.items[i] = it - return pyNone - return newIndexTypeError(newPyAscii"list", arg1) - -implListMagic delitem, [mutable: write]: - if other.ofPyIntObject: - let idx = getIndex(PyIntObject(other), self.len) - self.items.delete idx - return pyNone - if other.ofPySliceObject: - let slice = PySliceObject(other) - case slice.stepAsInt: - of 1, -1: - self.items.delete slice.toNimSlice(self.len) + +template genMutableSequenceMethods*(mapper, unmapper, S, Ele, beforeAppend){.dirty.} = + ## `beforeAppend` body will be inserted before `append` method's implementation + bind times + bind ofPySliceObject, PySliceObject, getIterableWithCheck, stepAsInt, toNimSlice, + iterInt, unhashable, delete + bind echoCompat + proc extend*(self: `Py S Object`, other: PyObject): PyObject = + if other.`ofPy S Object`: + self.items &= `Py S Object`(other).items else: - for i in iterInt(slice, self.len): - self.items.delete i - return pyNone - return newIndexTypeError(newPyAscii"list", other) + pyForIn nextObj, other: + self.items.add nextObj.mapper + pyNone -implListMagic hash: unhashable self + `impl S Magic` imul, [mutable: write]: + var n: int + let e = PyNumber_AsSsize_t(other, n) + if not e.isNil: + return e + self.items = times(self.items, n) + + `impl S Magic` iadd, [mutable: write]: self.extend other + + `impl S Magic` setitem, [mutable: write]: + if arg1.ofPyIntObject: + let idx = getIndex(PyIntObject(arg1), self.len) + self.items[idx] = arg2.mapper + return pyNone + if ofPySliceObject(arg1): + let slice = PySliceObject(arg1) + let iterableToLoop = arg2 + case stepAsInt(slice) + of 1, -1: + var ls: seq[Ele] + pyForIn it, iterableToLoop: + ls.add it.mapper + self.items[toNimSlice(slice, self.len)] = ls + else: + let (iterable, nextMethod) = getIterableWithCheck(iterableToLoop) + if iterable.isThrownException: + return iterable + for i in iterInt(slice, self.len): + let it = nextMethod(iterable) + if it.isStopIter: + break + if it.isThrownException: + return it + self.items[i] = it.mapper + return pyNone + return newIndexTypeError(newPyAscii"list", arg1) + + `impl S Magic` delitem, [mutable: write]: + if other.ofPyIntObject: + let idx = getIndex(PyIntObject(other), self.len) + self.items.delete idx + return pyNone + if ofPySliceObject(other): + let slice = PySliceObject(other) + case stepAsInt(slice): + of 1, -1: + delete(self.items, toNimSlice(slice, self.len)) + else: + for i in iterInt(slice, self.len): + self.items.delete i + return pyNone + return newIndexTypeError(newPyAscii"list", other) + + `impl S Magic` hash: unhashable self + + `impl S Method` append(item: PyObject), [mutable: write]: + beforeAppend + self.items.add(item.mapper) + pyNone -implListMethod append(item: PyObject), [mutable: write]: - self.items.add(item) - pyNone + `impl S Method` clear(), [mutable: write]: + self.items.setLen 0 + pyNone -implListMethod clear(), [mutable: write]: - self.items.setLen 0 - pyNone + `impl S Method` copy(), [mutable: read]: + let newL = `newPy S`() + newL.items = self.items # shallow copy + newL -implListMethod copy(), [mutable: read]: - let newL = newPyList() - newL.items = self.items # shallow copy - newL + # some test methods just for debugging + when not defined(release): + # for lock testing + `impl S Method` doClear(), [mutable: read]: + # should fail because trying to write while reading + self.`clearPy S ObjectMethod`() -# some test methods just for debugging -when not defined(release): - # for lock testing - implListMethod doClear(), [mutable: read]: - # should fail because trying to write while reading - self.clearPyListObjectMethod() + `impl S Method` doRead(), [mutable: write]: + # trying to read while writing + return self.`doClearPy S ObjectMethod`() - implListMethod doRead(), [mutable: write]: - # trying to read whiel writing - return self.doClearPyListObjectMethod() + when Ele is PyObject: + # for checkArgTypes testing + `impl S Method` aInt(i: PyIntObject), [mutable: read]: + self.items.add(i) + pyNone - # for checkArgTypes testing - implListMethod aInt(i: PyIntObject), [mutable: read]: - self.items.add(i) - pyNone + # for macro pragma testing + macro hello(code: untyped): untyped = + code.body.insert(0, nnkCommand.newTree(bindSym("echoCompat"), newStrLitNode("hello"))) + code - # for macro pragma testing - macro hello(code: untyped): untyped = - code.body.insert(0, nnkCommand.newTree(ident("echoCompat"), newStrLitNode("hello"))) - code + `impl S Method` hello(), [hello]: + pyNone - implListMethod hello(), [hello]: + + `impl S Method` extend(other: PyObject), [mutable: write]: self.extend other + + + `impl S Method` insert(idx: PyIntObject, item: PyObject), [mutable: write]: + var intIdx: int + if idx.negative: + intIdx = 0 + elif self.items.len < idx: + intIdx = self.items.len + else: + intIdx = idx.toIntOrRetOF + self.items.insert(item.mapper, intIdx) pyNone -implListMethod extend(other: PyObject), [mutable: write]: - if other.ofPyListObject: - self.items &= PyListObject(other).items - else: - pyForIn nextObj, other: - self.items.add nextObj - pyNone - - -implListMethod insert(idx: PyIntObject, item: PyObject), [mutable: write]: - var intIdx: int - if idx.negative: - intIdx = 0 - elif self.items.len < idx: - intIdx = self.items.len - else: - intIdx = idx.toIntOrRetOF - self.items.insert(item, intIdx) - pyNone - - -implListMethod pop(), [mutable: write]: - if self.items.len == 0: - let msg = "pop from empty list" - return newIndexError newPyAscii(msg) - self.items.pop - -implListMethod remove(target: PyObject), [mutable: write]: - var retObj: PyObject - allowSelfReadWhenBeforeRealWrite: - retObj = tpMethod(List, index)(selfNoCast, @[target]) - if retObj.isThrownException: - return retObj - assert retObj.ofPyIntObject - let idx = PyIntObject(retObj).toIntOrRetOF - self.items.delete(idx) - pyNone + `impl S Method` pop(), [mutable: write]: + if self.items.len == 0: + let msg = "pop from empty list" + return newIndexError newPyAscii(msg) + unmapper self.items.pop + + `impl S Method` remove(target: PyObject), [mutable: write]: + var retObj: PyObject + allowSelfReadWhenBeforeRealWrite: + retObj = tpMethod(S, index)(selfNoCast, @[target]) + if retObj.isThrownException: + return retObj + assert retObj.ofPyIntObject + let idx = PyIntObject(retObj).toIntOrRetOF + self.items.delete(idx) + pyNone + +template asIs(x): untyped = x +genMutableSequenceMethods(asIs, asIs, List, PyObject): discard diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 9415c43..0292603 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -14,8 +14,12 @@ declarePyType Tuple(reprLock, tpToken): privateHash: Hash -proc newPyTuple*(items: seq[PyObject]): PyTupleObject = +proc newPyTuple(): PyTupleObject{.inline.} = + ## unpublic, used by `__mul__` method result = newPyTupleSimple() + +proc newPyTuple*(items: seq[PyObject]): PyTupleObject = + result = newPyTuple() # shallow copy result.items = items @@ -73,6 +77,11 @@ template genGetitem*(nameStr, implNameMagic, newPyName, mutRead; getter: untyped return newIndexTypeError(newPyStr nameStr, other) +proc times*[T](s: openArray[T], n: int): seq[T] = + result = newSeqOfCap[T](s.len * n) + for _ in 1..n: + result.add s + template genSequenceMagics*(nameStr, implNameMagic, implNameMethod; ofPyNameObject, PyNameObject, @@ -85,6 +94,13 @@ template genSequenceMagics*(nameStr, ofPyNameObject, PyNameObject, mutRead, mutReadRepr, seqToStr + implNameMagic mul, mutRead: + var n: int + let e = PyNumber_AsSsize_t(other, n) + if not e.isNil: + return e + newPyName self.items.times n + implNameMagic add, mutRead: var res = newPyName() if other.ofPyNameObject: @@ -171,7 +187,7 @@ proc tupleSeqToString(ss: openArray[UnicodeVariant]): UnicodeVariant = genSequenceMagics "tuple", implTupleMagic, implTupleMethod, ofPyTupleObject, PyTupleObject, - newPyTupleSimple, [], [reprLock], + newPyTuple, [], [reprLock], tupleSeqToString template hashCollectionImpl*(items; hashForEmpty): Hash = From 07044a28b151fa0a1a692abc0b837b7503262aab Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 04:59:54 +0800 Subject: [PATCH 148/163] feat(builtins/sequence): sup method reverse --- Objects/listobject.nim | 7 +++++-- Objects/pyobject.nim | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index cdea236..6d965fd 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -1,4 +1,4 @@ -import std/sequtils +import std/[sequtils, algorithm] import strformat import strutils @@ -33,7 +33,7 @@ genSequenceMagics "list", template genMutableSequenceMethods*(mapper, unmapper, S, Ele, beforeAppend){.dirty.} = ## `beforeAppend` body will be inserted before `append` method's implementation - bind times + bind times, reverse bind ofPySliceObject, PySliceObject, getIterableWithCheck, stepAsInt, toNimSlice, iterInt, unhashable, delete bind echoCompat @@ -110,6 +110,9 @@ template genMutableSequenceMethods*(mapper, unmapper, S, Ele, beforeAppend){.dir self.items.setLen 0 pyNone + `impl S Method` reverse(), [mutable: write]: + reverse(self.items) + pyNone `impl S Method` copy(), [mutable: read]: let newL = `newPy S`() diff --git a/Objects/pyobject.nim b/Objects/pyobject.nim index d203d42..f9b4753 100644 --- a/Objects/pyobject.nim +++ b/Objects/pyobject.nim @@ -268,12 +268,12 @@ macro checkArgTypes*(nameAndArg, code: untyped): untyped = # return `checkArgNum(1, "append")` like body.add newCall(ident("checkArgNum"), newIntLitNode(argNum), - newStrLitNode(methodName.strVal) + newStrLitNode($methodName) ) else: body.add newCall(ident("checkArgNumAtLeast"), newIntLitNode(argNum - 1), - newStrLitNode(methodName.strVal) + newStrLitNode($methodName) ) let remainingArgNode = ident(varargname) body.add(quote do: @@ -348,7 +348,7 @@ proc implMethod*(prototype, ObjectType, pragmas, body: NimNode, kind: MethodKind for i in ls: name.add i.strVal methodName = ident name - methodName.expectKind({nnkIdent, nnkSym}) + methodName.expectKind({nnkIdent, nnkSym, nnkClosedSymChoice}) ObjectType.expectKind(nnkIdent) body.expectKind(nnkStmtList) pragmas.expectKind(nnkBracket) @@ -428,7 +428,7 @@ proc implMethod*(prototype, ObjectType, pragmas, body: NimNode, kind: MethodKind typeObjNode, newIdentNode("registerBltinMethod") ), - newLit(methodName.strVal), + newLit($methodName), name ) of MethodKind.Magic: From 546558a791a4217166784c855d6ab956944fe484 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 05:06:54 +0800 Subject: [PATCH 149/163] feat(builtins): bytes/bytearray: impl at least the same methods as tuple/list --- Objects/byteobjects.nim | 42 +++++++++++---- Objects/byteobjectsImpl.nim | 102 +++++++++++++++++++++++++++++++++--- 2 files changed, 127 insertions(+), 17 deletions(-) diff --git a/Objects/byteobjects.nim b/Objects/byteobjects.nim index 9f2ebcb..6fa7aa3 100644 --- a/Objects/byteobjects.nim +++ b/Objects/byteobjects.nim @@ -7,10 +7,14 @@ import ./[listobject, tupleobject, stringobject, exceptions, iterobject] import ./numobjects declarePyType Bytes(tpToken): items: string + setHash: bool + privateHash: Hash declarePyType ByteArray(reprLock, mutable): items: seq[char] +proc hash*(self: PyBytesObject): Hash = self.hashCollection + type PyBytesWriter* = object #overallocate*: bool use_bytearray*: bool @@ -44,13 +48,14 @@ proc `$`(self: seq[char]): string = type PyByteLike = PyBytesObject or PyByteArrayObject proc len*(s: PyByteLike): int {. inline, cdecl .} = s.items.len -proc hash*(s: PyByteLike): Hash {. inline, cdecl .} = hash s.items proc `$`*(s: PyByteLike): string = $s.items iterator items*(s: PyByteLike): char = for i in s.items: yield i iterator ints*(s: PyByteLike): PyIntObject = for i in s: yield newPyInt i +proc contains*(s: PyByteLike, c: char): bool = c in s.items proc `[]`*(s: PyByteLike, i: int): char = s.items[i] +proc getInt*(s: PyByteLike, i: int): PyIntObject = newPyInt s[i] template impl(B, InitT, newTOfCap){.dirty.} = @@ -75,6 +80,8 @@ proc finish*(self: sink PyBytesWriter, res: PyObject) = if self.use_bytearray: PyByteArrayObject(res).items = move self.s else: PyBytesObject(res).items = $(move self.s) +proc newPyBytes*(s: seq[char]): PyBytesObject = newPyBytes $s + proc repr*(b: PyBytesObject): string = 'b' & '\'' & b.items & '\'' # TODO @@ -82,20 +89,37 @@ proc repr*(b: PyByteArrayObject): string = "bytearray(" & 'b' & '\'' & $b.items & '\'' #[TODO]# & ')' -proc `[]=`*(s: PyByteLike, i: int, c: char) = s.items[i] = c +proc `[]=`*(s: PyByteArrayObject, i: int, c: char) = s.items[i] = c +proc add*(s: PyByteArrayObject, c: char) = s.items.add c proc add*(self: PyByteArrayObject, b: PyByteLike) = self.items.add b.items proc setLen*(self: PyByteArrayObject, n: int) = self.items.setLen n -template fillFromIterable(writer: PyBytesWriter; x; forInLoop; errSubject: string) = +template checkCharRangeOrRetVE*(value: int; errSubject="byte") = + if value < 0 or value > 256: + return newValueError newPyAscii(errSubject & " must be in range(0, 256)") + +proc bufferNotImpl*(): PyNotImplementedErrorObject = + ## TODO:buffer: delete this once buffer api is implemented + newNotImplementedError newPyAscii"not impl for buffer api" + +template PyNumber_AsCharOr*(vv: PyObject, errSubject="byte"; orDoIt): char = + bind PyNumberAsClampedSsize_t, checkCharRangeOrRetVE var value: int + block: + let it{.inject.} = PyNumber_AsClampedSsize_t(vv, value) + if not it.isNil: + orDoIt + checkCharRangeOrRetVE(value, errSubject) + cast[char](value) + +template PyNumber_AsCharOrRet*(vv: PyObject, errSubject="byte"): char = + PyNumber_AsCharOr(vv, errSubject): + return it + +template fillFromIterable(writer: PyBytesWriter; x; forInLoop; errSubject: string) = forInLoop i, x: - let ret = PyNumber_AsClampedSsize_t(i, value) - if not ret.isNil: - return ret - if value < 0 or value > 256: - return newValueError newPyAscii(errSubject & " must be in range(0, 256)" & " not " & $value) - writer.add cast[char](value) + writer.add i.PyNumber_AsCharOrRet(errSubject) template genFromIter(S; T; forInLoop; getLenHint: untyped=len){.dirty.} = proc `PyBytes_From S`(x: T): PyObject = diff --git a/Objects/byteobjectsImpl.nim b/Objects/byteobjectsImpl.nim index 52ac2d0..246738c 100644 --- a/Objects/byteobjectsImpl.nim +++ b/Objects/byteobjectsImpl.nim @@ -1,17 +1,57 @@ import std/strformat - +import ../Utils/sequtils import ./byteobjects import ./pyobject -import ./[boolobject, numobjects, stringobject, exceptions, noneobject, - iterobject, +import ./[boolobject, numobjects, stringobjectImpl, exceptions, noneobject, + iterobject, hash, abstract, ] - +import ./tupleobject +from ./listobject import genMutableSequenceMethods export byteobjects +proc `&`(s: string, se: seq[char]): string = + result.setLen s.len + se.len + result.add s + when defined(copyMem): + copyMem result[s.len].addr, se[0].addr, se.len + else: + for i in se: result.add i +template `&`(se: seq[char], s: string): seq[char] = se & @s + +template binDoCorS(doSth, o): untyped{.dirty.} = + block binDoSth: + type Res = typeof(self.items.doSth('\0')) + const hasRes = Res is_not void + when hasRes: + var res: Res + template doRes(x) = res = x + else: + template doRes(x) = x + doRes doSth(self.items, + o.PyNumber_AsCharOr("bytes") do: + if o.ofPyBytesObject: + let ob = o.PyBytesObject + doRes self.items.doSth(ob.items) + break binDoSth + elif o.ofPyByteArrayObject: + let ob = o.PyByteArrayObject + doRes self.items.doSth(ob.items) + break binDoSth + else: + # TODO:buffer + return bufferNotImpl() + # return self.doSth s + ) + when hasRes: + res + +template doFind(self, target): untyped = + ## helper to avoid too much `...it2, start, stop)` code snippet + find(self, target, start, stop) -template impl(B, mutRead){.dirty.} = +template implCommons(B, mutRead){.dirty.} = methodMacroTmpl(B) type `T B` = `Py B Object` `impl B Magic` eq: @@ -20,13 +60,59 @@ template impl(B, mutRead){.dirty.} = return newPyBool self == `T B`(other) `impl B Magic` len, mutRead: newPyInt self.len `impl B Magic` repr, mutRead: newPyAscii(repr self) - `impl B Magic` hash: newPyInt self.hash + genGetitem astToStr(B), `impl B Magic`, `newPy B`, mutRead, getInt `impl B Magic` iter, mutRead: genPyNimIteratorIter self.ints + `impl B Magic` contains, mutRead: + newPyBool binDoCorS(contains, other) + #fmt"argument should be integer or bytes-like object, not '{other.pyType.name:.200s}'") + + `impl B Magic` add, mutRead: + template retRes(o): untyped = `newPy B`(self.items & o.items) + if other.ofPyBytesObject: + retRes PyBytesObject(other) + elif other.ofPyByteArrayObject: + retRes PyByteArrayObject(other) + else: + # TODO:buffer + newTypeError newPyStr( + fmt"can't concat {self.pyType.name:.100s} to {other.pyType.name:.100s}" + ) + + `impl B Method` find, mutRead: + implMethodGenTargetAndStartStop() + newPyInt binDoCorS(doFind, target) + + `impl B Method` index, mutRead: + implMethodGenTargetAndStartStop() + let res = binDoCorS(doFind, target) + if res >= 0: + return newPyInt(res) + newValueError(newPyAscii"subsection not found") + + `impl B Method` count: + implMethodGenTargetAndStartStop() + var count: int + template cntAll(it, o) = + for _ in findAll(it, o, start, stop): count.inc + binDoCorS(cntAll, target) + newPyInt(count) + + +implCommons bytes, [] +implCommons bytearray, [mutable: read] + + +implBytesMagic hash: newPyInt self.hash +implBytesMagic bytes: self +implByteArrayMagic bytes, [mutable: read]: newPyBytes self.items -impl Bytes, [] -impl ByteArray, [mutable: read] +genMutableSequenceMethods PyNumber_AsCharOrRet, newPyInt, ByteArray, char: + # before append + when compileOption"boundChecks": + if self.len == high int: + return newOverflowError newPyAscii"cannot add more objects to bytearray" template impl(x, fromSize, fromObject) = if x.ofPyStrObject: From 454eefe8991110a61dbdcaa77c185c2ad5b22786 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 06:35:43 +0800 Subject: [PATCH 150/163] fix(builtins/print): default sep used '\n' over ' ' --- Python/bltinmodule.nim | 25 +++++++++++++++++++++++-- Utils/compat.nim | 16 ++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index b305b98..71cef30 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -68,11 +68,32 @@ macro implBltinFunc(prototype, body:untyped): untyped = # haven't thought of how to deal with infinite num of args yet # kwargs seems to be neccessary. So stay this way for now # luckily it does not require much boilerplate + +const NewLine = "\n" proc builtinPrint*(args: seq[PyObject]): PyObject {. cdecl .} = - for obj in args: + const + sep = " " + endl = NewLine + + const noWrite = not declared(writeStdoutCompat) + when noWrite: + var res: string + template writeStdoutCompat(s) = res.add s + template toStr(obj): string = let objStr = obj.callMagic(str) errorIfNotString(objStr, "__str__") - echoCompat $PyStrObject(objStr).str + $PyStrObject(objStr).str + if args.len != 0: + writeStdoutCompat args[0].toStr + if args.len > 1: + for i in 1.. Date: Thu, 31 Jul 2025 08:50:33 +0800 Subject: [PATCH 151/163] fix(augassign): `**=`,`//=` SyntaxError; `*=`,`%=` NotImplementedError --- Parser/lexer.nim | 4 ++-- Python/ast.nim | 15 +++++++++------ Python/neval.nim | 26 ++++++++++++++++++-------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/Parser/lexer.nim b/Parser/lexer.nim index 5017528..9fa0a9e 100644 --- a/Parser/lexer.nim +++ b/Parser/lexer.nim @@ -219,7 +219,7 @@ proc getNextToken( result = newTokenNodeWithNo(DoubleStar) inc idx else: - addSingleCharToken(Star) + addSingleOrDoubleCharToken(Star, StarEqual, '=') of '/': if tailing('/'): inc idx @@ -230,7 +230,7 @@ proc getNextToken( result = newTokenNodeWithNo(DoubleSlash) inc idx else: - addSingleCharToken(Slash) + addSingleOrDoubleCharToken(Slash, SlashEqual, '=') of '|': addSingleOrDoubleCharToken(Vbar, VbarEqual, '=') of '&': diff --git a/Python/ast.nim b/Python/ast.nim index 812124b..1ca7882 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -535,21 +535,24 @@ ast augassign, [AsdlOperator]: of Token.Plusequal: newAstAdd() of Token.Minequal:newAstSub() of Token.Starequal: newAstMult() + #of Token.Atequal: newAstAt() of Token.Slashequal:newAstDiv() of Token.Percentequal: newAstMod() + #[ + of Token.Amperequal: newAstAmper() + of Token.Vbarequal: newAstVbar() + of Token.Circumflexequal: newAstCircumflex() + of Token.Leftshiftequal: newAstLeftshift() + of Token.Rightshiftequal: newAstRightshift() + ]# of Token.DoubleSlashequal: newAstFloorDiv() + of Token.DoubleStarequal: newAstPow() else: let msg = fmt"Complex augumented assign operation not implemented: " & $token raiseSyntaxError(msg) ) -#[ -Amperequal -Vbarequal - Circumflexequal - ]# - # del_stmt: 'del' exprlist ast del_stmt, [AsdlStmt]: var node = newAstDelete() diff --git a/Python/neval.nim b/Python/neval.nim index 42ad115..13201d8 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -264,14 +264,6 @@ proc evalFrame*(f: PyFrameObject): PyObject = of OpCode.UnaryNot: doUnary(Not) - of OpCode.BinaryPower: - doBinary(pow) - - of OpCode.BinaryMultiply: - doBinary(mul) - - of OpCode.BinaryModulo: - doBinary(Mod) of OpCode.StoreSubscr: let idx = sPop() @@ -298,6 +290,15 @@ proc evalFrame*(f: PyFrameObject): PyObject = of OpCode.BinarySubtract: doBinary(sub) + of OpCode.BinaryPower: + doBinary(pow) + + of OpCode.BinaryMultiply: + doBinary(mul) + + of OpCode.BinaryModulo: + doBinary(Mod) + of OpCode.BinaryFloorDivide: doBinary(floorDiv) @@ -310,6 +311,15 @@ proc evalFrame*(f: PyFrameObject): PyObject = of OpCode.InplaceSubtract: doInplace(sub) + of OpCode.InplacePower: + doInplace(pow) + + of OpCode.InplaceMultiply: + doInplace(mul) + + of OpCode.InplaceModulo: + doInplace(Mod) + of OpCode.InplaceFloorDivide: doInplace(floorDiv) From c2f4335058663b14faa584870bc7e0b8cddf2ef5 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 11:04:06 +0800 Subject: [PATCH 152/163] feat(syntax/import_from): sup `from x import a[, ...]` --- Python/ast.nim | 84 +++++++++++++++++++++++++++------------------ Python/compile.nim | 8 +++++ Python/neval.nim | 9 +++++ Python/symtable.nim | 4 +++ 4 files changed, 71 insertions(+), 34 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index 1ca7882..1c6c23c 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -115,7 +115,7 @@ proc astRaiseStmt(parseNode: ParseNode): AstRaise proc astImportStmt(parseNode: ParseNode): AsdlStmt proc astImportName(parseNode: ParseNode): AsdlStmt proc astDottedAsNames(parseNode: ParseNode): seq[AstAlias] -proc astDottedName(parseNode: ParseNode): AstAlias +proc astDottedName(parseNode: ParseNode): AsdlIdentifier proc astGlobalStmt(parseNode: ParseNode): AsdlStmt proc astNonlocalStmt(parseNode: ParseNode): AsdlStmt proc astAssertStmt(parseNode: ParseNode): AstAssert @@ -615,6 +615,54 @@ ast raise_stmt, [AstRaise]: else: raiseSyntaxError("Fancy raise not implemented", parseNode.children[2]) +template handleAlias = + if parseNode.children.len != 1: + raiseSyntaxError("import alias not implemented") + +# import_as_name NAME ['as' NAME] +ast import_as_name, [AstAlias]: + handleAlias + result = newAstAlias() + result.name = newIdentifier(parseNode.children[0].tokenNode.content) + +# import_as_names import_as_name (',' import_as_name)* [','] +ast import_as_names, [seq[Astalias]]: + for i in countup(0, parseNode.children.len, 2): + result.add parseNode.children[i].astImportAsName + + +# dotted_as_name dotted_name ['as' NAME] +ast dotted_as_name, [AstAlias]: + handleAlias + result = newAstAlias() + result.name = parseNode.children[0].astDottedName + +# dotted_as_names dotted_as_name (',' dotted_as_name)* +ast dotted_as_names, [seq[AstAlias]]: + if parseNode.children.len != 1: + raiseSyntaxError("import multiple modules in one line not implemented", + parseNode.children[1]) + result.add parseNode.children[0].astDottedAsName + +# dotted_name NAME ('.' NAME)* +ast dotted_name, [AsdlIdentifier]: + if parseNode.children.len != 1: + raiseSyntaxError("dotted import name not supported", parseNode.children[1]) + newIdentifier(parseNode.children[0].tokenNode.content) + + +# import_from ('from' (('.' | '...')* dotted_name | ('.' | '...')+) +# 'import' ('*' | '(' import_as_names ')' | import_as_names)) +ast import_from, [AsdlStmt]: + let node = newAstImportFrom() + setNo(node, parseNode.children[0]) + let m = parseNode.children[1] + # TODO:import_from '.'|'..." import '*' + node.module = m.astDottedName + let ls = parseNode.children[3] + for c in ls.astImportAsNames: + node.names.add c + node # import_stmt import_name | import_from ast import_stmt, [AsdlStmt]: @@ -623,7 +671,7 @@ ast import_stmt, [AsdlStmt]: of Token.import_name: result = astImportName(child) of Token.import_from: - raiseSyntaxError("Import from not implemented") + result = astImportFrom(child) else: unreachable("wrong import_stmt") @@ -635,38 +683,6 @@ ast import_name, [AsdlStmt]: node.names.add c node - #[ -ast import_from: - discard - -ast import_as_name: - discard -]# - -# dotted_as_name dotted_name ['as' NAME] -ast dotted_as_name, [AstAlias]: - if parseNode.children.len != 1: - raiseSyntaxError("import alias not implemented") - parseNode.children[0].astDottedName - - -#ast import_as_names: -# discard - - -# dotted_as_names dotted_as_name (',' dotted_as_name)* -ast dotted_as_names, [seq[AstAlias]]: - if parseNode.children.len != 1: - raiseSyntaxError("import multiple modules in one line not implemented", - parseNode.children[1]) - result.add parseNode.children[0].astDottedAsName - -# dotted_name NAME ('.' NAME)* -ast dotted_name, [AstAlias]: - if parseNode.children.len != 1: - raiseSyntaxError("dotted import name not supported", parseNode.children[1]) - result = newAstAlias() - result.name = newIdentifier(parseNode.children[0].tokenNode.content) proc astGlobalStmt(parseNode: ParseNode): AsdlStmt = raiseSyntaxError("global stmt not implemented") diff --git a/Python/compile.nim b/Python/compile.nim index 190143d..6341a32 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -646,6 +646,14 @@ compileMethod Import: c.addStoreOp(name, lineNo) +compileMethod ImportFrom: + let lineNo = astNode.lineNo.value + let modName = astNode.module + c.addOp(newArgInstr(OpCode.ImportName, c.tste.nameId(modName.value), lineNo)) + for n in astNode.names: + let name = AstAlias(n).name + c.addOp(newArgInstr(OpCode.ImportFrom, c.tste.nameId(name.value), lineNo)) + c.addStoreOp(name, lineNo) compileMethod Expr: let lineNo = astNode.value.lineNo.value diff --git a/Python/neval.nim b/Python/neval.nim index 13201d8..72fa72f 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -516,6 +516,15 @@ proc evalFrame*(f: PyFrameObject): PyObject = if retObj.isThrownException: handleException(retObj) sPush retObj + of OpCode.ImportFrom: + let module = sTop() + let name = names[opArg] + let retObj = module.callMagic(getattr, name, handleExcp=true) + if retObj.isThrownException: + handleException(retObj) + # TODO:_PyEval_ImportFrom + # after sys.module impl + sPush retObj of OpCode.JumpIfFalseOrPop: let top = sTop() diff --git a/Python/symtable.nim b/Python/symtable.nim index 5c26ffd..bc34028 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -275,6 +275,10 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = assert AstImport(astNode).names.len == 1 ste.addDeclaration(AstAlias(AstImport(astNode).names[0]).name) + of AsdlStmtTk.ImportFrom: + for n in AstImportFrom(astNode).names: + ste.addDeclaration(AstAlias(n).name) + of AsdlStmtTk.Expr: visit AstExpr(astNode).value From a5fd64cb078c79c46a09fb363ce9fbea66b9d7fe Mon Sep 17 00:00:00 2001 From: litlighilit Date: Thu, 31 Jul 2025 11:25:42 +0800 Subject: [PATCH 153/163] feat(syntax/import): sup `as` (alias), multi import in one stmt --- Python/ast.nim | 38 +++++++++++++++++++++++--------------- Python/compile.nim | 16 ++++++++-------- Python/symtable.nim | 6 +++--- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/Python/ast.nim b/Python/ast.nim index 1c6c23c..27cf624 100644 --- a/Python/ast.nim +++ b/Python/ast.nim @@ -615,15 +615,20 @@ ast raise_stmt, [AstRaise]: else: raiseSyntaxError("Fancy raise not implemented", parseNode.children[2]) -template handleAlias = - if parseNode.children.len != 1: - raiseSyntaxError("import alias not implemented") - +template identAt(i: int): untyped = + newIdentifier(parseNode.children[i].tokenNode.content) +template identAtOr(i: int; def: AsdlIdentifier): AsdlIdentifier = + if parseNode.children.len < i: def + else: identAt i + +template initAsName = + result.asname = identAtOr(2, result.name) + # import_as_name NAME ['as' NAME] ast import_as_name, [AstAlias]: - handleAlias result = newAstAlias() - result.name = newIdentifier(parseNode.children[0].tokenNode.content) + result.name = identAt 0 + initAsName # import_as_names import_as_name (',' import_as_name)* [','] ast import_as_names, [seq[Astalias]]: @@ -633,22 +638,21 @@ ast import_as_names, [seq[Astalias]]: # dotted_as_name dotted_name ['as' NAME] ast dotted_as_name, [AstAlias]: - handleAlias result = newAstAlias() result.name = parseNode.children[0].astDottedName - + initAsName + # dotted_as_names dotted_as_name (',' dotted_as_name)* ast dotted_as_names, [seq[AstAlias]]: - if parseNode.children.len != 1: - raiseSyntaxError("import multiple modules in one line not implemented", - parseNode.children[1]) - result.add parseNode.children[0].astDottedAsName + for i in countup(0, parseNode.children.len, 2): + let child = parseNode.children[i] + result.add child.astDottedAsName # dotted_name NAME ('.' NAME)* ast dotted_name, [AsdlIdentifier]: if parseNode.children.len != 1: raiseSyntaxError("dotted import name not supported", parseNode.children[1]) - newIdentifier(parseNode.children[0].tokenNode.content) + identAt(0) #TODO:dotted_name # import_from ('from' (('.' | '...')* dotted_name | ('.' | '...')+) @@ -660,8 +664,12 @@ ast import_from, [AsdlStmt]: # TODO:import_from '.'|'..." import '*' node.module = m.astDottedName let ls = parseNode.children[3] - for c in ls.astImportAsNames: - node.names.add c + if ls.tokenNode.token == Token.Star: + # TODO:import_star + raiseSyntaxError("`from ... import *` not implemented yet", node) + else: + for c in ls.astImportAsNames: + node.names.add c node # import_stmt import_name | import_from diff --git a/Python/compile.nim b/Python/compile.nim index 6341a32..23c48bd 100644 --- a/Python/compile.nim +++ b/Python/compile.nim @@ -639,21 +639,21 @@ compileMethod Assert: compileMethod Import: let lineNo = astNode.lineNo.value - if not astNode.names.len == 1: - unreachable - let name = AstAlias(astNode.names[0]).name - c.addOp(newArgInstr(OpCode.ImportName, c.tste.nameId(name.value), lineNo)) - c.addStoreOp(name, lineNo) - + for n in astNode.names: + let module = AstAlias(n) + let name = module.name + c.addOp(newArgInstr(OpCode.ImportName, c.tste.nameId(name.value), lineNo)) + c.addStoreOp(module.asname, lineNo) compileMethod ImportFrom: let lineNo = astNode.lineNo.value let modName = astNode.module c.addOp(newArgInstr(OpCode.ImportName, c.tste.nameId(modName.value), lineNo)) for n in astNode.names: - let name = AstAlias(n).name + let aliasNode = AstAlias(n) + let name = aliasNode.name c.addOp(newArgInstr(OpCode.ImportFrom, c.tste.nameId(name.value), lineNo)) - c.addStoreOp(name, lineNo) + c.addStoreOp(aliasNode.asname, lineNo) compileMethod Expr: let lineNo = astNode.value.lineNo.value diff --git a/Python/symtable.nim b/Python/symtable.nim index bc34028..7238774 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -272,12 +272,12 @@ proc collectDeclaration*(st: SymTable, astRoot: AsdlModl) = visitSeq(tryNode.finalbody) of AsdlStmtTk.Import: - assert AstImport(astNode).names.len == 1 - ste.addDeclaration(AstAlias(AstImport(astNode).names[0]).name) + for n in AstImport(astNode).names: + ste.addDeclaration(AstAlias(n).asname) of AsdlStmtTk.ImportFrom: for n in AstImportFrom(astNode).names: - ste.addDeclaration(AstAlias(n).name) + ste.addDeclaration(AstAlias(n).asname) of AsdlStmtTk.Expr: visit AstExpr(astNode).value From f9732f5e49d0df7708f009080c12b1bb31a02b42 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sun, 3 Aug 2025 18:32:32 +0800 Subject: [PATCH 154/163] feat(inner): exceptions: add errorIfNot,retIfExc --- Objects/exceptions.nim | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 0464955..0cd405d 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -183,19 +183,27 @@ template isThrownException*(pyObj: PyObject): bool = else: false - -template errorIfNotString*(pyObj: untyped, methodName: string) = - if not pyObj.ofPyStrObject: - let typeName {. inject .} = pyObj.pyType.name - let msg = methodName & fmt" returned non-string (type {typeName})" - return newTypeError newPyStr(msg) - -template errorIfNotBool*(pyObj: untyped, methodName: string) = - if not pyObj.ofPyBoolObject: - let typeName {. inject .} = pyObj.pyType.name - let msg = methodName & fmt" returned non-bool (type {typeName})" - return newTypeError(newPyStr msg) - +template retIt = return it +template errorIfNot*(S; expect: string, pyObj: PyObject, methodName: string, doIt: untyped=retIt) = + if not pyObj.`ofPy S Object`: + let typeName {. inject .} = pyObj.pyType.name + let texp {.inject.} = expect + let msg = methodName & fmt" returned non-{texp} (type {typeName})" + let it {.inject.} = newTypeError newPyStr(msg) + doIt + +template errorIfNotString*(pyObj: untyped, methodName: string, doIt: untyped=retIt) = + errorIfNot Str, "string", pyObj, methodName, doIt + +template errorIfNot*(S; pyObj: PyObject, methodName: string, doIt: untyped=retIt) = + errorIfNot S, astToStr(S), pyObj, methodName, doIt +template errorIfNotBool*(pyObj: PyObject, methodName: string, doIt: untyped=retIt) = + errorIfNot bool, pyObj, methodName, doIt + +template retIfExc*(e: PyBaseErrorObject) = + let exc = e + if not exc.isNil: + return exc template getIterableWithCheck*(obj: PyObject): (PyObject, UnaryMethod) = var retTuple: (PyObject, UnaryMethod) From 7d4be97f6de322ea6414ab0f8715aa649ddeb140 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sun, 3 Aug 2025 18:30:23 +0800 Subject: [PATCH 155/163] fix: PyObject_IsTrue carshed when shall raise TypeError --- Objects/boolobjectImpl.nim | 25 +++++++++++++++++-------- Python/neval.nim | 8 ++++++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/Objects/boolobjectImpl.nim b/Objects/boolobjectImpl.nim index b421800..3579888 100644 --- a/Objects/boolobjectImpl.nim +++ b/Objects/boolobjectImpl.nim @@ -54,20 +54,29 @@ implBoolMagic hash: newPyInt(Hash(self.b)) -proc PyObject_IsTrue*(v: PyObject): bool = - if v == pyTrueObj: return true - if v == pyFalseObj: return false - if v == pyNone: return false +proc PyObject_IsTrue*(v: PyObject, res: var bool): PyBaseErrorObject = + template ret(b: bool) = + res = b + return nil + if v == pyTrueObj: ret true + if v == pyFalseObj: ret false + if v == pyNone: ret false let boolMag = v.getMagic(bool) if not boolMag.isNil: - return boolMag(v).PyBoolObject.b + let obj = boolMag(v) + errorIfNotBool obj, "__bool__" + ret obj.PyBoolObject.b elif not v.getMagic(len).isNil: - return v.getMagic(len)(v).PyIntObject.positive + let obj = v.getMagic(len)(v) + errorIfNot int, obj, "__bool__" + ret obj.PyIntObject.positive # We currently don't define: # as_sequence # as_mapping - return true + ret true implBoolMagic New(tp: PyObject, obj: PyObject): - newPyBool PyObject_IsTrue obj + var b: bool + retIfExc PyObject_IsTrue(obj, b) + newPyBool b diff --git a/Python/neval.nim b/Python/neval.nim index 72fa72f..9550396 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -100,9 +100,13 @@ template doBinaryContain: PyObject = let res = op1.callMagic(contains, op2, handleExcp=true) res -# "fast" because check if it's a bool object first and save the callMagic(bool) template getBoolFast(obj: PyObject): bool = - PyObject_IsTrue(obj) + ## called "fast" only due to historical reason + var b: bool + let exc = PyObject_IsTrue(obj, b) + if not exc.isNil: + handleException(exc) + b # if declared as a local variable, js target will fail. See gh-10651 when defined(js): From 3708eb984a2c5c4b9cf83030a3c741ece0c13546 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 5 Aug 2025 11:36:23 +0800 Subject: [PATCH 156/163] fix(py): exceptions: no LookupError, no attrs --- Objects/boolobjectImpl.nim | 4 +- Objects/bundle.nim | 4 +- Objects/byteobjects.nim | 2 +- Objects/byteobjectsImpl.nim | 2 +- Objects/exceptions.nim | 89 ++++++++++------ Objects/exceptionsImpl.nim | 36 ++++--- Objects/listobject.nim | 2 +- Objects/noneobject.nim | 3 - Objects/noneobjectImpl.nim | 8 ++ Objects/setobject.nim | 2 +- Objects/stringobject.nim | 2 + Objects/stringobjectImpl.nim | 13 ++- Objects/tupleobject.nim | 200 +---------------------------------- Objects/tupleobjectImpl.nim | 197 ++++++++++++++++++++++++++++++++++ Objects/typeobject.nim | 4 +- Python/bltinmodule.nim | 6 +- Python/neval.nim | 26 +++-- 17 files changed, 331 insertions(+), 269 deletions(-) create mode 100644 Objects/noneobjectImpl.nim create mode 100644 Objects/tupleobjectImpl.nim diff --git a/Objects/boolobjectImpl.nim b/Objects/boolobjectImpl.nim index 3579888..59ca0d1 100644 --- a/Objects/boolobjectImpl.nim +++ b/Objects/boolobjectImpl.nim @@ -5,11 +5,11 @@ import macros import pyobject import exceptions import stringobject -import boolobject +import ./boolobject +export boolobject import numobjects import ./noneobject -export boolobject method `$`*(obj: PyBoolObject): string = $obj.b diff --git a/Objects/bundle.nim b/Objects/bundle.nim index 275cda7..f814f68 100644 --- a/Objects/bundle.nim +++ b/Objects/bundle.nim @@ -2,9 +2,9 @@ import baseBundle import codeobject, dictobject, frameobject, boolobjectImpl, listobject, moduleobject, methodobject, funcobject, pyobject, stringobjectImpl, rangeobject, exceptionsImpl, - sliceobject, tupleobject, cellobject, setobject + sliceobject, tupleobjectImpl, cellobject, setobject export baseBundle export codeobject, dictobject, frameobject, boolobjectImpl, listobject, moduleobject, methodobject, funcobject, pyobject, stringobjectImpl, rangeobject, exceptionsImpl, - sliceobject, tupleobject, cellobject, setobject + sliceobject, tupleobjectImpl, cellobject, setobject diff --git a/Objects/byteobjects.nim b/Objects/byteobjects.nim index 6fa7aa3..ed51e0d 100644 --- a/Objects/byteobjects.nim +++ b/Objects/byteobjects.nim @@ -3,7 +3,7 @@ import std/strformat import std/hashes import ./pyobject import ./abstract -import ./[listobject, tupleobject, stringobject, exceptions, iterobject] +import ./[listobject, tupleobjectImpl, stringobject, exceptions, iterobject] import ./numobjects declarePyType Bytes(tpToken): items: string diff --git a/Objects/byteobjectsImpl.nim b/Objects/byteobjectsImpl.nim index 246738c..12e2cb4 100644 --- a/Objects/byteobjectsImpl.nim +++ b/Objects/byteobjectsImpl.nim @@ -6,7 +6,7 @@ import ./pyobject import ./[boolobject, numobjects, stringobjectImpl, exceptions, noneobject, iterobject, hash, abstract, ] -import ./tupleobject +import ./tupleobjectImpl from ./listobject import genMutableSequenceMethods export byteobjects diff --git a/Objects/exceptions.nim b/Objects/exceptions.nim index 0cd405d..b6adeb9 100644 --- a/Objects/exceptions.nim +++ b/Objects/exceptions.nim @@ -7,33 +7,43 @@ # This might be a bit slower but we are not pursueing ultra performance anyway import std/enumutils import std/macrocache +import std/tables export macrocache.items -import strformat +from std/strutils import split +import std/strformat -import pyobject -import stringobject +import ./[pyobject, stringobject, noneobject, tupleobject] type ExceptionToken* {. pure .} = enum Base, Name, - NotImplemented, Type, Arithmetic, Attribute, Value, - Index, + Lookup, StopIter, Lock, Import, - UnboundLocal, - Key, Assertion, - ZeroDivision, Runtime, - Syntax, + Syntax, #TODO:SyntaxError: shall be with many attributes Memory, - KeyboardInterrupt, + KeyboardInterrupt, #TODO:BaseException shall be subclass of BaseException + +const ExcAttrs = toTable { + # values will be `split(',')` + Name: "name", + Attribute: "name,obj", + StopIter: "value", + Import: "msg,name,name_from,path", + Syntax: "end_lineno,end_offset,filename,lineno,msg,offset,print_file_and_line,text" +} +iterator extraAttrs(tok: ExceptionToken): NimNode = + ExcAttrs.withValue(tok, value): + for n in value.split(','): + yield ident n proc getTokenName*(excp: ExceptionToken): string = excp.symbolName proc getBltinName*(excp: ExceptionToken): string = @@ -52,7 +62,7 @@ type TraceBack* = tuple declarePyType BaseError(tpToken, typeName("Exception")): tk: ExceptionToken thrown: bool - msg: PyObject # could be nil + args: PyTupleObject # could not be nil context: PyBaseErrorObject # if the exception happens during handling another exception # used for tracebacks, set in neval.nim traceBacks: seq[TraceBack] @@ -65,13 +75,24 @@ type proc ofPyExceptionObject*(obj: PyObject): bool {. cdecl, inline .} = obj.ofPyBaseErrorObject +macro setAttrsNone(tok: static[ExceptionToken], self) = + result = newStmtList() + for n in extraAttrs(tok): + result.add newAssignment( + newDotExpr(self, n), + bindSym"pyNone" + ) macro declareErrors: untyped = result = newStmtList() for i in 1..int(ExceptionToken.high): let tok = ExceptionToken(i) let tokenStr = tok.getTokenName - + var attrs = newStmtList() + for n in extraAttrs(tok): + attrs.add newCall(n, newStmtList bindSym"PyObject") + if attrs.len == 0: # no extra attr + attrs.add nnkDiscardStmt.newTree(newEmptyNode()) let typeNode = nnkStmtList.newTree( nnkCommand.newTree( newIdentNode("declarePyType"), @@ -86,11 +107,7 @@ macro declareErrors: untyped = ident tok.getBltinName # or it'll be e.g. "stopitererror" ) ), - nnkStmtList.newTree( - nnkDiscardStmt.newTree( - newEmptyNode() - ) - ) + attrs ) ) @@ -112,11 +129,12 @@ template newProcTmpl(excpName, tok){.dirty.} = let excp = `newPy excpName ErrorSimple`() excp.tk = ExceptionToken.`tok` excp.thrown = true + setAttrsNone ExceptionToken.tok, excp excp proc `new excpName Error`*(msgStr: PyStrObject): `Py excpName ErrorObject`{.inline.} = let excp = `new excpName Error`() - excp.msg = msgStr + excp.args = newPyTuple [PyObject msgStr] excp template newProcTmpl(excpName) = @@ -147,14 +165,18 @@ macro declareSubError(E, baseE) = `typ`.name = `eeS` declareSubError Overflow, Arithmetic - -template newAttributeError*(tpName, attrName: PyStrObject): untyped = - let msg = tpName & newPyAscii" has no attribute " & attrName - newAttributeError(msg) - -template newAttributeError*(tpName, attrName: string): untyped = - newAttributeError(tpName.newPyStr, attrName.newPyStr) - +declareSubError ZeroDivision, Arithmetic +declareSubError Index, Lookup +declareSubError Key, Lookup +declareSubError UnboundLocal, Name +declareSubError NotImplemented, Runtime + +template newAttributeError*(tobj: PyObject, attrName: PyStrObject): untyped = + let msg = newPyStr(tobj.pyType.name) & newPyAscii" has no attribute " & attrName + let exc = newAttributeError(msg) + exc.name = attrName + exc.obj = tobj + exc template newIndexTypeError*(typeName: PyStrObject, obj:PyObject): untyped = let name = obj.pyType.name @@ -172,8 +194,8 @@ proc isStopIter*(obj: PyObject): bool = obj.isExceptionOf StopIter method `$`*(e: PyExceptionObject): string = result = "Error: " & $e.tk & " " - if not e.msg.isNil: - result &= $e.msg + # not e.args.isNil + result &= $e.args @@ -205,6 +227,11 @@ template retIfExc*(e: PyBaseErrorObject) = if not exc.isNil: return exc +template retIfExc*(e: PyObject) = + let exc = e + if exc.isThrownException: + return exc + template getIterableWithCheck*(obj: PyObject): (PyObject, UnaryMethod) = var retTuple: (PyObject, UnaryMethod) block body: @@ -248,5 +275,7 @@ template checkArgNumAtLeast*(expected: int, name="") = msg = "expected at least " & $expected & fmt" argument ({args.len} given)" return newTypeError(newPyStr msg) -proc PyErr_Format*(exc: PyBaseErrorObject, msg: PyStrObject) = - exc.msg = msg +proc PyErr_Format*[E: PyBaseErrorObject](exc: E, msg: PyStrObject) = + exc.args = newPyTuple [PyObject msg] + when compiles(exc.msg): + exc.msg = msg diff --git a/Objects/exceptionsImpl.nim b/Objects/exceptionsImpl.nim index 562c41d..5f787c8 100644 --- a/Objects/exceptionsImpl.nim +++ b/Objects/exceptionsImpl.nim @@ -1,6 +1,6 @@ import pyobject import baseBundle -import tupleobject +import ./[tupleobjectImpl, stringobjectImpl] import exceptions import ../Utils/utils @@ -16,30 +16,30 @@ macro genMethodMacros: untyped = genMethodMacros -template newMagicTmpl(excpName: untyped, excpNameStr: string) = +template newMagicTmpl(excpName: untyped, excpNameStr: string){.dirty.} = `impl excpName ErrorMagic` repr: # must return pyStringObject, used when formatting traceback var msg: string - if self.msg.isNil: - msg = "" # could be improved - elif self.msg.ofPyStrObject: - msg = $PyStrObject(self.msg) + assert not self.args.isNil + # ensure this is either an throwned exception or string for user-defined type + let msgObj = self.args.callMagic(repr) + if msgObj.isThrownException: + msg = "evaluating __repr__ failed" else: - # ensure this is either an throwned exception or string for user-defined type - let msgObj = self.msg.callMagic(repr) - if msgObj.isThrownException: - msg = "evaluating __repr__ failed" - else: - msg = $PyStrObject(msgObj) + msg = $PyStrObject(msgObj) let str = $self.tk & "Error: " & msg - newPyAscii(str) + newPyStr(str) + `impl excpName ErrorMagic` str: + if self.args.len == 0: newPyAscii() + else: PyObject_StrNonNil(self.args[0]) + # this is for initialization at Python level `impl excpName ErrorMagic` New: let excp = `newPy excpName ErrorSimple`() excp.tk = ExceptionToken.`excpName` - excp.msg = newPyTuple(args) + excp.args = newPyTuple(args) excp @@ -70,7 +70,13 @@ proc isExceptionType*(obj: PyObject): bool = proc fromBltinSyntaxError*(e: SyntaxError, fileName: PyStrObject): PyExceptionObject = - let excpObj = newSyntaxError newPyStr(e.msg) + let smsg = newPyStr(e.msg) + let excpObj = newSyntaxError smsg + excpObj.lineno = newPyInt e.lineNo + #TODO:end_lineno + excpObj.filename = fileName + excpObj.end_offset = newPyInt e.colNo + excpObj.msg = smsg # don't have code name excpObj.traceBacks.add (PyObject fileName, PyObject nil, e.lineNo, e.colNo) excpObj diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 6d965fd..1ea060d 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -7,7 +7,7 @@ import baseBundle import ./sliceobjectImpl import ./hash import iterobject -import ./tupleobject +import ./tupleobjectImpl import ./dictobject import ../Utils/[utils, compat] diff --git a/Objects/noneobject.nim b/Objects/noneobject.nim index 05623b5..e487a68 100644 --- a/Objects/noneobject.nim +++ b/Objects/noneobject.nim @@ -1,6 +1,5 @@ import pyobject import ./stringobject -import ./exceptions declarePyType None(tpToken): discard @@ -14,5 +13,3 @@ method `$`*(_: PyNoneObject): string = sNone implNoneMagic repr: newPyAscii sNone -implNoneMagic New(tp: PyObject): - return pyNone diff --git a/Objects/noneobjectImpl.nim b/Objects/noneobjectImpl.nim new file mode 100644 index 0000000..fa1765d --- /dev/null +++ b/Objects/noneobjectImpl.nim @@ -0,0 +1,8 @@ +{.used.} +import ./noneobject +import ./[pyobject, exceptions] + +methodMacroTmpl(None) + +implNoneMagic New(tp: PyObject): + return pyNone diff --git a/Objects/setobject.nim b/Objects/setobject.nim index ac31e54..7da581c 100644 --- a/Objects/setobject.nim +++ b/Objects/setobject.nim @@ -9,7 +9,7 @@ import pyobject import baseBundle import ./iterobject -import ./tupleobject +import ./tupleobjectImpl import ../Utils/[utils] import ./hash diff --git a/Objects/stringobject.nim b/Objects/stringobject.nim index caf6cb2..428fcdd 100644 --- a/Objects/stringobject.nim +++ b/Objects/stringobject.nim @@ -179,6 +179,8 @@ proc newPyString*(str: seq[Rune]): PyStrObject = newPyString newUnicodeUnicodeVariant(str) proc newPyAscii*(str: string): PyStrObject = newPyString newAsciiUnicodeVariant(str) +let empty = newPyAscii"" +proc newPyAscii*(): PyStrObject = empty ## empty string # TODO: make them faster proc newPyString*(r: Rune): PyStrObject{.inline.} = newPyString @[r] diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index 82fbe97..97c58cd 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -37,9 +37,7 @@ implStrMagic repr: implStrMagic hash: newPyInt(self.hash) -# TODO: encoding, errors params -implStrMagic New(tp: PyObject, obj: PyObject): - # ref: unicode_new -> unicode_new_impl -> PyObject_Str +proc PyObject_StrNonNil*(obj: PyObject): PyObject = let fun = obj.getMagic(str) if fun.isNil: return obj.callMagic(repr) @@ -48,6 +46,15 @@ implStrMagic New(tp: PyObject, obj: PyObject): return newTypeError newPyStr( &"__str__ returned non-string (type {result.pyType.name:.200s})") +proc PyObject_Str*(obj: PyObject): PyObject = + if obj.isNil: newPyAscii"" + else: PyObject_StrNonNil obj + +# TODO: encoding, errors params +implStrMagic New(tp: PyObject, obj: PyObject): + # ref: unicode_new -> unicode_new_impl -> PyObject_Str + PyObject_StrNonNil obj + implStrMagic add(i: PyStrObject): self & i diff --git a/Objects/tupleobject.nim b/Objects/tupleobject.nim index 0292603..c6b4b7b 100644 --- a/Objects/tupleobject.nim +++ b/Objects/tupleobject.nim @@ -1,12 +1,6 @@ -import std/hashes -import ./hash -import strformat - -import pyobject -import baseBundle -import iterobject -import sliceobject +import std/hashes +import ./pyobject declarePyType Tuple(reprLock, tpToken): items: seq[PyObject] @@ -14,8 +8,8 @@ declarePyType Tuple(reprLock, tpToken): privateHash: Hash -proc newPyTuple(): PyTupleObject{.inline.} = - ## unpublic, used by `__mul__` method +proc newPyTuple*(): PyTupleObject{.inline.} = + ## inner, used by `__mul__` method result = newPyTupleSimple() proc newPyTuple*(items: seq[PyObject]): PyTupleObject = @@ -24,188 +18,4 @@ proc newPyTuple*(items: seq[PyObject]): PyTupleObject = result.items = items proc newPyTuple*(items: openArray[PyObject]): PyTupleObject{.inline.} = - newPyTuple @items - -template genCollectMagics*(items, - implNameMagic, - ofPyNameObject, PyNameObject, - mutRead, mutReadRepr, seqToStr){.dirty.} = - - template len*(self: PyNameObject): int = self.items.len - template `[]`*(self: PyNameObject, i: int): PyObject = self.items[i] - iterator items*(self: PyNameObject): PyObject = - for i in self.items: yield i - - implNameMagic contains, mutRead: - for item in self: - let retObj = item.callMagic(eq, other) - if retObj.isThrownException: - return retObj - if retObj == pyTrueObj: - return pyTrueObj - return pyFalseObj - - - implNameMagic repr, mutReadRepr: - var ss: seq[UnicodeVariant] - for item in self: - var itemRepr: PyStrObject - let retObj = item.callMagic(repr) - errorIfNotString(retObj, "__repr__") - itemRepr = PyStrObject(retObj) - ss.add itemRepr.str - return newPyString(seqToStr(ss)) - - - implNameMagic len, mutRead: - newPyInt(self.len) - -template genGetitem*(nameStr, implNameMagic, newPyName, mutRead; getter: untyped = `[]`){.dirty.} = - bind ofPySliceObject, getIndex, PySliceObject, getSliceItems - implNameMagic getitem, mutRead: - if other.ofPyIntObject: - let idx = getIndex(PyIntObject(other), self.len) - return getter(self, idx) - if ofPySliceObject(other): - let slice = PySliceObject(other) - let newObj = newPyName() - let retObj = getSliceItems(slice, self.items, newObj.items) - if retObj.isThrownException: - return retObj - else: - return newObj - - return newIndexTypeError(newPyStr nameStr, other) - -proc times*[T](s: openArray[T], n: int): seq[T] = - result = newSeqOfCap[T](s.len * n) - for _ in 1..n: - result.add s - -template genSequenceMagics*(nameStr, - implNameMagic, implNameMethod; - ofPyNameObject, PyNameObject, - newPyName; mutRead, mutReadRepr; - seqToStr; initWithDictUsingPairs=false): untyped{.dirty.} = - - bind genCollectMagics - genCollectMagics items, - implNameMagic, - ofPyNameObject, PyNameObject, - mutRead, mutReadRepr, seqToStr - - implNameMagic mul, mutRead: - var n: int - let e = PyNumber_AsSsize_t(other, n) - if not e.isNil: - return e - newPyName self.items.times n - - implNameMagic add, mutRead: - var res = newPyName() - if other.ofPyNameObject: - res.items = self.items & PyNameObject(other).items - return res - else: - res.items = self.items - pyForIn i, other: - `&=` res.items, i - return res - implNameMagic eq, mutRead: - if not other.ofPyNameObject: - return pyFalseObj - let tOther = PyNameObject(other) - if self.items.len != tOther.items.len: - return pyFalseObj - for i in 0.. Date: Tue, 5 Aug 2025 12:23:19 +0800 Subject: [PATCH 157/163] fix(py): printTb was to stdout, all args was printed --- Python/traceback.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/traceback.nim b/Python/traceback.nim index fe4af15..91c4d66 100644 --- a/Python/traceback.nim +++ b/Python/traceback.nim @@ -32,8 +32,8 @@ proc printTb*(excp: PyExceptionObject) = singleExcpStrs.add "Traceback (most recent call last):" for tb in cur.traceBacks.reversed: singleExcpStrs.add tb.fmtTraceBack - singleExcpStrs.add $PyStrObject(tpMagic(BaseError, repr)(cur)).str + singleExcpStrs.add excp.pyType.name & ": " & $PyStrObject(tpMagic(BaseError, str)(cur)).str excpStrs.add singleExcpStrs.join("\n") cur = cur.context let joinMsg = "\n\nDuring handling of the above exception, another exception occured\n\n" - echoCompat excpStrs.reversed.join(joinMsg) + errEchoCompat excpStrs.reversed.join(joinMsg) From cd04936c17cfc68cb9875ed28ec4e5c71909da6a Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 5 Aug 2025 17:11:31 +0800 Subject: [PATCH 158/163] fix(py): exceptions: `__repr__` returns `xxError: ...` --- Objects/exceptionsImpl.nim | 37 +++++++++++++++++++----------------- Objects/stringobjectImpl.nim | 27 ++++++++++++++++++-------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/Objects/exceptionsImpl.nim b/Objects/exceptionsImpl.nim index 5f787c8..0ea26f5 100644 --- a/Objects/exceptionsImpl.nim +++ b/Objects/exceptionsImpl.nim @@ -15,31 +15,34 @@ macro genMethodMacros: untyped = genMethodMacros +proc BaseException_repr(self: PyBaseErrorObject): PyObject = + var msg: string = self.typeName + template `add%R`(o) = + let s = PyObject_ReprNonNil o + retIfExc s + msg.add $s + if self.args.len == 1: + msg.add '(' + `add%R` self.args[0] + msg.add ')' + else: + `add%R` self.args + newPyStr(msg) + +proc BaseException_str(self: PyBaseErrorObject): PyObject = + if self.args.len == 0: newPyAscii() + else: PyObject_StrNonNil(self.args[0]) template newMagicTmpl(excpName: untyped, excpNameStr: string){.dirty.} = - `impl excpName ErrorMagic` repr: - # must return pyStringObject, used when formatting traceback - var msg: string - assert not self.args.isNil - # ensure this is either an throwned exception or string for user-defined type - let msgObj = self.args.callMagic(repr) - if msgObj.isThrownException: - msg = "evaluating __repr__ failed" - else: - msg = $PyStrObject(msgObj) - let str = $self.tk & "Error: " & msg - newPyStr(str) - `impl excpName ErrorMagic` str: - if self.args.len == 0: newPyAscii() - else: PyObject_StrNonNil(self.args[0]) - + `impl excpName ErrorMagic` repr: BaseException_repr self + `impl excpName ErrorMagic` str: BaseException_str self # this is for initialization at Python level `impl excpName ErrorMagic` New: let excp = `newPy excpName ErrorSimple`() excp.tk = ExceptionToken.`excpName` - excp.args = newPyTuple(args) + excp.args = newPyTuple(args.toOpenArray(1, args.high)) excp diff --git a/Objects/stringobjectImpl.nim b/Objects/stringobjectImpl.nim index 97c58cd..56816bf 100644 --- a/Objects/stringobjectImpl.nim +++ b/Objects/stringobjectImpl.nim @@ -37,18 +37,29 @@ implStrMagic repr: implStrMagic hash: newPyInt(self.hash) -proc PyObject_StrNonNil*(obj: PyObject): PyObject = - let fun = obj.getMagic(str) +proc reprDefault*(self: PyObject): PyObject {. cdecl .} = + newPyString(fmt"<{self.typeName} object at {self.idStr}>") + +proc PyObject_ReprNonNil*(obj: PyObject): PyObject = + let fun = obj.getMagic(repr) if fun.isNil: - return obj.callMagic(repr) + return reprDefault obj result = fun(obj) - if not result.ofPyStrObject: - return newTypeError newPyStr( - &"__str__ returned non-string (type {result.pyType.name:.200s})") + result.errorIfNotString "__repr__" -proc PyObject_Str*(obj: PyObject): PyObject = +template nullOr(obj; elseCall): PyObject = if obj.isNil: newPyAscii"" - else: PyObject_StrNonNil obj + else: elseCall obj + +proc PyObject_Repr*(obj: PyObject): PyObject = obj.nullOr PyObject_ReprNonNil + +proc PyObject_StrNonNil*(obj: PyObject): PyObject = + let fun = obj.getMagic(str) + if fun.isNil: return PyObject_ReprNonNil(obj) + result = fun(obj) + result.errorIfNotString "__str__" + +proc PyObject_Str*(obj: PyObject): PyObject = obj.nullOr PyObject_StrNonNil # TODO: encoding, errors params implStrMagic New(tp: PyObject, obj: PyObject): From 8e2722e6a49c852658f14a831373c94ebc3d7680 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 5 Aug 2025 18:35:45 +0800 Subject: [PATCH 159/163] impr(opt): faster dir --- Objects/listobject.nim | 3 ++- Objects/pyobjectBase.nim | 6 +++++- Python/bltinmodule.nim | 19 +++++++------------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Objects/listobject.nim b/Objects/listobject.nim index 1ea060d..c48aea5 100644 --- a/Objects/listobject.nim +++ b/Objects/listobject.nim @@ -100,11 +100,12 @@ template genMutableSequenceMethods*(mapper, unmapper, S, Ele, beforeAppend){.dir `impl S Magic` hash: unhashable self - `impl S Method` append(item: PyObject), [mutable: write]: + proc add*(self: `Py S Object`, item: PyObject): PyObject = beforeAppend self.items.add(item.mapper) pyNone + `impl S Method` append(item: PyObject), [mutable: write]: self.add item `impl S Method` clear(), [mutable: write]: self.items.setLen 0 diff --git a/Objects/pyobjectBase.nim b/Objects/pyobjectBase.nim index 104de6a..c0f60ed 100644 --- a/Objects/pyobjectBase.nim +++ b/Objects/pyobjectBase.nim @@ -225,10 +225,14 @@ proc newPyType*(name: string): PyTypeObject = proc hasDict*(obj: PyObject): bool {. inline .} = obj of PyObjectWithDict +proc getDictUnsafe*(obj: PyObject): PyObject {. cdecl .} = + ## assuming obj.hasDict + PyObjectWithDict(obj).dict + proc getDict*(obj: PyObject): PyObject {. cdecl .} = if not obj.hasDict: unreachable("obj has no dict. Use hasDict before getDict") - PyObjectWithDict(obj).dict + obj.getDictUnsafe proc isClass*(obj: PyObject): bool {. cdecl .} = obj.pyType.kind == PyTypeToken.Type diff --git a/Python/bltinmodule.nim b/Python/bltinmodule.nim index b897755..8ea3cee 100644 --- a/Python/bltinmodule.nim +++ b/Python/bltinmodule.nim @@ -98,22 +98,17 @@ proc builtinPrint*(args: seq[PyObject]): PyObject {. cdecl .} = registerBltinFunction("print", builtinPrint) -proc keysList(d: PyDictObject): PyListObject = - ## inner usage - result = newPyList() - for key in d.keys(): - let rebObj = tpMethod(List, append)(result, @[key]) - if rebObj.isThrownException: - unreachable("No chance for append to thrown exception") - implBltinFunc dir(obj: PyObject): # why in CPython 0 argument becomes `locals()`? no idea # get mapping proxy first then talk about how do deal with __dict__ of type - var mergedDict = newPyDict() - mergedDict.update(obj.getTypeDict) + var res = newPyList() + template add(k) = + let ret = res.add k + assert ret.isPyNone + for k in obj.getTypeDict.keys(): add k if obj.hasDict: - mergedDict.update(PyDictObject(obj.getDict)) - mergedDict.keysList + for k in (PyDictObject(obj.getDictUnsafe)).keys: add k + res implBltinFunc id(obj: PyObject): From 816bb587c6a0687817cc316e3c17e75b0e67600d Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 6 Aug 2025 19:30:40 +0800 Subject: [PATCH 160/163] fix(py): missing exception type when lookup dict, elem's `__hash__`, `__eq__` called --- Objects/dictobject.nim | 110 ++++++++++++++++++++++++++++------------- Objects/hash.nim | 90 +++++++++++++++++++++++++++------ 2 files changed, 153 insertions(+), 47 deletions(-) diff --git a/Objects/dictobject.nim b/Objects/dictobject.nim index b4b42e5..b3bf9e2 100644 --- a/Objects/dictobject.nim +++ b/Objects/dictobject.nim @@ -9,6 +9,7 @@ import pyobject import baseBundle import ../Utils/utils import ./[iterobject, tupleobject] +from ./stringobject import PyStrObject import ./hash export hash @@ -24,8 +25,11 @@ proc newPyDict*(table=initTable[PyObject, PyObject]()) : PyDictObject = result.table = table proc hasKey*(dict: PyDictObject, key: PyObject): bool = - return dict.table.hasKey(key) -proc contains*(dict: PyDictObject, key: PyObject): bool = dict.hasKey key + ## may raises DictError where Python raises TypeError + dict.table.hasKey(key) +proc contains*(dict: PyDictObject, key: PyObject): bool = + ## may raises DictError where Python raises TypeError + dict.hasKey key template borIter(name, nname; R=PyObject){.dirty.} = iterator name*(dict: PyDictObject): R = @@ -62,13 +66,30 @@ proc clear*(dict: PyDictObject) = dict.table.clear proc `[]=`*(dict: PyDictObject, key, value: PyObject) = dict.table[key] = value +template retE(e) = return e +# TODO: overload all bltin types? +template withValue*(dict: PyDictObject, key: PyStrObject; value; body) = + ## we know `str.__eq__` and `str.__hash__` never raises + dict.table.withValue(key, value): body +template withValue*(dict: PyDictObject, key: PyStrObject; value; body, elseBody) = + ## we know `str.__eq__` and `str.__hash__` never raises + dict.table.withValue(key, value, body, elseBody) + +template withValue*(dict: PyDictObject, key: PyObject; value; body) = + ## `return` exception if error occurs on calling `__hash__` or `__eq__` + bind withValue, retE, handleHashExc + handleHashExc retE: + dict.table.withValue(key, value): body +template withValue*(dict: PyDictObject, key: PyObject; value; body, elseBody) = + ## `return` exception if error occurs on calling `__hash__` or `__eq__` + bind withValue, retE, handleHashExc + handleHashExc retE: + dict.table.withValue(key, value, body, elseBody) template checkHashableTmpl(res; obj) = let hashFunc = obj.pyType.magicMethods.hash if hashFunc.isNil: - let tpName = obj.pyType.name - let msg = "unhashable type: " & tpName - res = newTypeError newPyStr(msg) + res = unhashable obj return template checkHashableTmpl(obj) = @@ -77,12 +98,10 @@ template checkHashableTmpl(obj) = implDictMagic contains, [mutable: read]: checkHashableTmpl(other) - try: - result = self.table.getOrDefault(other, nil) - except DictError: - let msg = "__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" - return newTypeError newPyAscii(msg) - return newPyBool(not result.isNil) + var res: bool + handleHashExc retE: + res = self.table.hasKey(other) + newPyBool(res) implDictMagic repr, [mutable: read, reprLockWithMsg"{...}"]: var ss: seq[UnicodeVariant] @@ -112,33 +131,58 @@ implDictMagic Or(E: PyDictObject), [mutable: read]: res.table[k] = v res -template keyError(other: PyObject): PyObject = - var msg: PyStrObject +template keyError(other: PyObject): PyBaseErrorObject = let repr = other.pyType.magicMethods.repr(other) if repr.isThrownException: - msg = newPyAscii"exception occured when generating key error msg calling repr" + PyBaseErrorObject repr else: - msg = PyStrObject(repr) - newKeyError(msg) - -let badHashMsg = - newPyAscii"__hash__ method doesn't return an integer or __eq__ method doesn't return a bool" + PyBaseErrorObject newKeyError PyStrObject(repr) template handleBadHash(res; body){.dirty.} = - try: + template setRes(e) = res = e + handleHashExc setRes: body - except DictError: - res = newTypeError badHashMsg - return -proc getitemImpl(self: PyDictObject, other: PyObject): PyObject = - checkHashableTmpl(other) - result.handleBadHash: - result = self.table.getOrDefault(other, nil) - if not (result.isNil): - return result - return keyError other -implDictMagic getitem, [mutable: read]: self.getitemImpl other +proc getItem*(dict: PyDictObject, key: PyObject): PyObject = + ## unlike PyDict_GetItem (which suppresses all errors for historical reasons), + ## returns KeyError if missing `key`, TypeError if `key` unhashable + checkHashableTmpl(key) + dict.withValue(key, value): + return value[] + do: + return keyError key + +type GetItemRes*{.pure.} = enum + ## order value is the same as Python's + Error = -1 + Missing = 0 + Get = 1 + +proc getItemRef*(dict: PyDictObject, key: PyObject, res: var PyObject, exc: var PyBaseErrorObject): GetItemRes = + ## `PyDict_GetItemRef`: + ## if `key` missing, set res to nil and return KeyError + result = GetItemRes.Error + exc.checkHashableTmpl(key) + exc.handleBadHash: + dict.table.withValue(key, value): + res = value[] + result = GetItemRes.Get + do: + res = nil + exc = keyError key + result = GetItemRes.Missing + +proc getOpionalItem*(dict: PyDictObject; key: PyObject): PyObject = + ## like PyDict_GetItemWithError, can be used as `PyMapping_GetOptionalItem`: + ## returns nil if missing `key`, TypeError if `key` unhashable + var exc: PyBaseErrorObject + let res = dict.getItemRef(key, result, exc) + case res + of Get: discard + of Missing: result = nil + of Error: result = exc + +implDictMagic getitem, [mutable: read]: self.getitem other implDictMagic setitem, [mutable: write]: checkHashableTmpl(arg1) @@ -149,7 +193,7 @@ implDictMagic setitem, [mutable: write]: proc pop*(self: PyDictObject, other: PyObject, res: var PyObject): bool = ## - if `other` not in `self`, `res` is set to KeyError; ## - if in, set to value of that key; - ## - if `DictError`_ raised, `res` is set to TypeError + ## - if exception raised, `res` is set to that res.checkHashableTmpl(other) res.handleBadHash: if self.table.pop(other, res): @@ -171,7 +215,7 @@ implDictMethod get, [mutable: write]: let key = args[0] checkhashabletmpl(key) if args.len == 1: - return self.getitemimpl key + return self.getItem key checkargnum 2 let defval = args[1] result.handleBadHash: diff --git a/Objects/hash.nim b/Objects/hash.nim index 83c5dfa..5015059 100644 --- a/Objects/hash.nim +++ b/Objects/hash.nim @@ -1,9 +1,11 @@ import std/hashes import ./pyobject -import ./baseBundle +import ./[ + exceptions, stringobject, numobjects, boolobjectImpl, +] import ../Utils/utils -proc unhashable*(obj: PyObject): PyObject = newTypeError newPyAscii( +proc unhashable*(obj: PyObject): PyTypeErrorObject = newTypeError newPyAscii( "unhashable type '" & obj.pyType.name & '\'' ) @@ -11,30 +13,90 @@ proc rawHash*(obj: PyObject): Hash = ## for type.__hash__ hash(obj.id) -# hash functions for py objects -# raises an exception to indicate type error. Should fix this -# when implementing custom dict -proc hash*(obj: PyObject): Hash = - ## for builtins.hash +var curHashExc{.threadvar.}: PyBaseErrorObject +proc popCurHashExc(): PyBaseErrorObject = + ## never returns nil, assert exc happens + assert not curHashExc.isNil + result = curHashExc + curHashExc = nil + +proc PyObject_Hash*(obj: PyObject, exc: var PyBaseErrorObject): Hash = + exc = nil let fun = obj.pyType.magicMethods.hash if fun.isNil: + #PY-DIFF: we don't + #[ + ```c + if (!_PyType_IsReady(tp)) { + if (PyType_Ready(tp) < 0) + ... + ``` + ]# + # as we only allow declare python type via `declarePyType` return rawHash(obj) else: let retObj = fun(obj) - if not retObj.ofPyIntObject: - raise newException(DictError, retObj.pyType.name) - return hash(PyIntObject(retObj)) + if retObj.ofPyIntObject: + # ref CPython/Objects/typeobject.c:slot_tp_hash + let i = PyIntObject(retObj) + var ovf: bool + result = i.asLongAndOverflow(ovf) + if unlikely ovf: + result = hash(i) + elif retObj.isThrownException: + exc = PyBaseErrorObject retObj + else: + exc = unhashable obj + +proc PyObject_Hash*(obj: PyObject): PyObject = + var exc: PyBaseErrorObject + let h = PyObject_Hash(obj, exc) + if exc.isNil: newPyInt h + else: exc + +template signalDictError(msg) = + if not curHashExc.isNil: + raise newException(DictError, msg) + +# hash functions for py objects +# raises an exception to indicate type error. Should fix this +# when implementing custom dict +proc hash*(obj: PyObject): Hash = + ## inner usage for dictobject. + ## + ## .. warning:: remember wrap around `doDictOp` to handle exception + result = PyObject_Hash(obj, curHashExc) + signalDictError "hash" + +template handleHashExc*(handleExc; body) = + ## to handle exception from `hash`_ + bind popCurHashExc, DictError + try: body + except DictError: handleExc popCurHashExc() proc rawEq*(obj1, obj2: PyObject): bool = ## for type.__eq__ obj1.id == obj2.id -proc `==`*(obj1, obj2: PyObject): bool {. inline, cdecl .} = +proc PyObject_Eq*(obj1, obj2: PyObject, exc: var PyBaseErrorObject): bool = + ## XXX: CPython doesn't define such a function, it uses richcompare (e.g. `_Py_BaseObject_RichCompare`) + ## + ## .. note:: `__eq__` is not required to return a bool, + ## so this calls `PyObject_IsTrue`_ on result + exc = nil let fun = obj1.pyType.magicMethods.eq if fun.isNil: return rawEq(obj1, obj2) else: let retObj = fun(obj1, obj2) - if not retObj.ofPyBoolObject: - raise newException(DictError, retObj.pyType.name) - return PyBoolObject(retObj).b + if retObj.isThrownException: + exc = PyBaseErrorObject retObj + return + exc = PyObject_IsTrue(retObj, result) + +proc `==`*(obj1, obj2: PyObject): bool {. inline, cdecl .} = + ## inner usage for dictobject. + ## + ## .. warning:: remember wrap around `doDictOp` to handle exception + result = PyObject_Eq(obj1, obj2, curHashExc) + signalDictError "eq" From 15efcb87b46425e4b9e1edf0361ebef1af2105ee Mon Sep 17 00:00:00 2001 From: litlighilit Date: Wed, 6 Aug 2025 19:39:56 +0800 Subject: [PATCH 161/163] impr(opt): use withValue over `hasKey` and `[]` --- Objects/typeobject.nim | 12 ++++++------ Python/neval.nim | 13 +++++++------ Python/symtable.nim | 6 +++--- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Objects/typeobject.nim b/Objects/typeobject.nim index 8d40e8a..e37dc17 100644 --- a/Objects/typeobject.nim +++ b/Objects/typeobject.nim @@ -81,16 +81,16 @@ proc getAttr(self: PyObject, nameObj: PyObject): PyObject {. cdecl .} = if typeDict.isNil: unreachable("for type object dict must not be nil") var descr: PyObject - if typeDict.hasKey(name): - descr = typeDict[name] + typeDict.withValue(name, value): + descr = value[] let descrGet = descr.pyType.magicMethods.get if not descrGet.isNil: return descr.descrGet(self) if self.hasDict: let instDict = PyDictObject(self.getDict) - if instDict.hasKey(name): - return instDict[name] + instDict.withValue(name, val): + return val[] if not descr.isNil: return descr @@ -108,8 +108,8 @@ proc setAttr(self: PyObject, nameObj: PyObject, value: PyObject): PyObject {. cd if typeDict.isNil: unreachable("for type object dict must not be nil") var descr: PyObject - if typeDict.hasKey(name): - descr = typeDict[name] + typeDict.withValue(name, val): + descr = val[] let descrSet = descr.pyType.magicMethods.set if not descrSet.isNil: return descr.descrSet(self, value) diff --git a/Python/neval.nim b/Python/neval.nim index 8820c99..3bf5c42 100644 --- a/Python/neval.nim +++ b/Python/neval.nim @@ -562,12 +562,13 @@ proc evalFrame*(f: PyFrameObject): PyObject = of OpCode.LoadGlobal: let name = names[opArg] var obj: PyObject - if f.globals.hasKey(name): - obj = f.globals[name] - elif bltinDict.hasKey(name): - obj = bltinDict[name] - else: - notDefined name + f.globals.withValue(name, value): + obj = value[] + do: + bltinDict.withValue(name, value): + obj = value[] + do: + notDefined name sPush obj of OpCode.SetupFinally: diff --git a/Python/symtable.nim b/Python/symtable.nim index 7238774..65fa051 100644 --- a/Python/symtable.nim +++ b/Python/symtable.nim @@ -103,9 +103,9 @@ proc localId*(ste: SymTableEntry, localName: PyStrObject): int = proc nameId*(ste: SymTableEntry, nameStr: PyStrObject): int = # add entries for attribute lookup - if ste.names.hasKey(nameStr): - return ste.names[nameStr] - else: + ste.names.withValue(nameStr, value): + return value[] + do: result = ste.names.len ste.names[nameStr] = result From e3f23a34fbd33e5594dace629bc1ba52cbdb63fc Mon Sep 17 00:00:00 2001 From: litlighilit Date: Sat, 9 Aug 2025 20:10:44 +0800 Subject: [PATCH 162/163] fixup(4c7318ed65fc3f56): prepare for nim-lang/Nim#25092 --- Modules/getbuildinfo.nim | 11 ++++------- Python/getversion.nim | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Modules/getbuildinfo.nim b/Modules/getbuildinfo.nim index 165fb38..c938486 100644 --- a/Modules/getbuildinfo.nim +++ b/Modules/getbuildinfo.nim @@ -7,8 +7,7 @@ when NimMajor > 1: else: from std/os import `/../`, parentDir import ./os_findExe_patch -when defined(windows): - from std/strutils import stripLineEnd +from std/strutils import stripLineEnd ## see CPython/configure.ac @@ -22,11 +21,9 @@ else: bind srcdir_git let res = gorgeEx(git.exe & " --git-dir " & srcdir_git & " " & sub) assert res.exitCode == 0, res.output - when defined(windows): - var outp = res.output - outp.stripLineEnd - outp - else: res.output + var outp = res.output + outp.stripLineEnd + outp const version = git.exec"rev-parse --short HEAD" diff --git a/Python/getversion.nim b/Python/getversion.nim index 60f9b8a..648bbf9 100644 --- a/Python/getversion.nim +++ b/Python/getversion.nim @@ -9,5 +9,5 @@ proc Py_GetCompiler*: string = "[Nim " & NimVersion & ']' proc Py_GetVersion*: string = - &"{Version:.80} ({Py_GetBuildInfo():.80}) {Py_GetCompiler():.80}" # TODO with buildinfo, compilerinfo in form of "%.80s (%.80s) %.80s" + &"{Version:.80} ({Py_GetBuildInfo():.80}) {Py_GetCompiler():.80}" From 3e1f4a91288afd637b853c9f2fcebddb676209f8 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 11 Aug 2025 21:32:18 +0800 Subject: [PATCH 163/163] fix(py): repr(obj) missing "object" before "at" --- Objects/typeobject.nim | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/Objects/typeobject.nim b/Objects/typeobject.nim index e37dc17..3a6208c 100644 --- a/Objects/typeobject.nim +++ b/Objects/typeobject.nim @@ -67,16 +67,19 @@ proc defaultEq(o1, o2: PyObject): PyObject {. cdecl .} = if rawEq(o1, o2): pyTrueObj else: pyFalseObj -proc reprDefault(self: PyObject): PyObject {. cdecl .} = - newPyString(fmt"<{self.pyType.name} at {self.idStr}>") + +template asAttrNameOrRetE*(name: PyObject): PyStrObject = + bind ofPyStrObject, typeName, newTypeError, newPyStr, PyStrObject + bind formatValue, fmt + if not ofPyStrObject(name): + let n{.inject.} = typeName(name) + return newTypeError newPyStr( + fmt"attribute name must be string, not '{n:.200s}'",) + PyStrObject name # generic getattr -proc getAttr(self: PyObject, nameObj: PyObject): PyObject {. cdecl .} = - if not nameObj.ofPyStrObject: - let typeStr = nameObj.pyType.name - let msg = fmt"attribute name must be string, not {typeStr}" - return newTypeError(newPyStr msg) - let name = PyStrObject(nameObj) +proc getAttr(self: PyObject, nameObj: PyObject): PyObject {. cdecl .} = + let name = nameObj.asAttrNameOrRetE let typeDict = self.getTypeDict if typeDict.isNil: unreachable("for type object dict must not be nil") @@ -99,11 +102,7 @@ proc getAttr(self: PyObject, nameObj: PyObject): PyObject {. cdecl .} = # generic getattr proc setAttr(self: PyObject, nameObj: PyObject, value: PyObject): PyObject {. cdecl .} = - if not nameObj.ofPyStrObject: - let typeStr = nameObj.pyType.name - let msg = fmt"attribute name must be string, not {typeStr}" - return newTypeError(newPyStr msg) - let name = PyStrObject(nameObj) + let name = nameObj.asAttrNameOrRetE let typeDict = self.getTypeDict if typeDict.isNil: unreachable("for type object dict must not be nil")