diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c75c7af --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,133 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[flipjumpproject@gmail.com](mailto:flipjumpproject@gmail.com). +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..962f6b3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,85 @@ +# Introduction +First off, thank you for considering contributing to FlipJump. It's people like you that make the esoteric language community such a great, active and evolving community. + +Following these guidelines helps to communicate that you respect the time of the developers managing and developing this open source project. In return, they should reciprocate that respect in addressing your issue, assessing changes, and helping you finalize your pull requests. + +FlipJump is an open source project, and we love to receive contributions from our community — you!
+There are many ways to contribute, from writing tutorials or blog posts, writing new fj-programs, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into the FlipJump source / standard-library itself. + +Also, please take 2 minutes to show this project to the people you know that would **see the magic in this language.** + +Please, don't use the issue tracker for support-questions. Instead, use the [Questions thread](https://github.com/tomhea/flip-jump/discussions/176), or the [Discussions](https://github.com/tomhea/flip-jump/discussions) in general. + +## Responsibilities + * Ensure cross-platform compatibility for every change that's accepted. Windows & Ubuntu Linux. + * Ensure that code that goes into core passes the --regular [tests](tests/README.md). + * Create issues (+Discussions) for any major changes and enhancements that you wish to make. Discuss things transparently and get community feedback. + * Don't change the stl-api, only offer new options. feel free to discuss it first. + * Keep each PR as small as possible, preferably one new change/feature per PR. + * Be welcoming to newcomers and encourage diverse new contributors from all backgrounds. See the [Python Community Code of Conduct](https://www.python.org/psf/codeofconduct/). + +## Your First Contribution +Unsure where to begin contributing to FlipJump? You can start by creating and running your own FlipJump programs, on your own repos, and spread the rumor :)
+Also, please take a look at the [Contribution thread](https://github.com/tomhea/flip-jump/discussions/148). + +Working on your first Pull Request? You can learn how from this free series, [How to Contribute to an Open Source Project on GitHub](https://app.egghead.io/playlists/how-to-contribute-to-an-open-source-project-on-github). + +At this point, you're ready to make your changes! Feel free to ask for help; everyone is a beginner at first 😸 + +If a maintainer asks you to "rebase" your PR, they're saying that a lot of code has changed, and that you need to update your branch, so it's easier to merge. + +# Getting started +1. Create your own fork of the code +2. Do the changes in your fork (keep them minimal). +3. If you like the change and think the project could use it: + * Be sure you have followed the [code style](CONTRIBUTING.md#clean-code) for the project. + * be sure your project passes the --regular [tests](tests/README.md). + * Send a pull request. + +If you have **small or "obvious" fixes**, include SMALLFIX in the PR/issue name. +such fixes can be: +* Spelling / grammar fixes +* Typo correction, white space and formatting changes +* Comment clean up +* Functions/Classes rearrangements in the same file +It should still pass the --regular tests. + +# How to report a bug +When filing an issue, make sure to answer these five questions: + + 1. What version of FlipJump are you using (if no version, make sure you fetched the last changes, and specify the branch name)? + 2. What operating system are you using? + 3. What did you do? + 4. What did you expect to see? + 5. What did you see instead? +General questions should go to the [Questions thread](https://github.com/tomhea/flip-jump/discussions/176), or the [Discussions](https://github.com/tomhea/flip-jump/discussions) in general. + +# How to suggest a feature or enhancement +The FlipJump philosophy is to be the simplest langauge of all, that can do any modern computation. + +FlipJump should be below the OS, as it's a cpu-architecture after all. + +The FlipJump stl should be minimalistic, efficient in both space and time, and to offer macros similar to x86 ops.
+The generic stl macro should look like `macro_name n dst src` for an n-bit/hex variable, with dst being the destination-variable, and src being source-variable. (e.g. `hex.add n, dst, src`). + +If you find yourself wishing for a feature that doesn't exist, you are probably not alone. Some features that FlipJump has today have been added because our users saw the need. Open an issue on our issues list on GitHub which describes the feature you would like to see, why you need it, and how it should work. + +## Code review process +After feedback has been given to the Pull Request, we expect responses within two weeks. After two weeks we may close the pull request if it isn't showing any activity. + +# Community +You can chat with the core team and the community on [GitHub Discussions](https://github.com/tomhea/flip-jump/discussions). + +# Clean Code +Get familiar with [Clean Code](https://gist.github.com/wojteklu/73c6914cc446146b8b533c0988cf8d29) (mainly the functions/names sections). + +In short: +- use **clear names** (full words, **descriptive**, not-too-long), for variables, functions/macros (verb-name), and classes (nouns). +- **functions should do exactly one thing**. no side effects. They should be **very short** (and call other descriptive functions). IT IS POSSIBLE for a function to be 4-5 lines (and we should aim to that). + +Keep in mind that the developers of this community invested much of their time in making this project as clean, simple, and documented as they can. + +If you find a piece of code that isn't compliant with this standard, it probably has an open issue and is known, and if not, please open a new issue. + +Follow this rule but don't try to be perfect, and use the [80/20](https://en.wikipedia.org/wiki/Pareto_principle) principle. Yet, make an effort to make the code as simple, as much as you'd expect from others in this project's community. + diff --git a/README.md b/README.md index 500fd26..9e4adea 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # FlipJump FlipJump is the simplest programing language.
+Yet, it can do **any modern computation**. + It's an Esoteric language ([FlipJump esolangs page](https://esolangs.org/wiki/FlipJump)), with just 1 operation `a;b`: - `not *a; jump b` @@ -10,6 +12,9 @@ The operation takes 2 memory addresses - it flips (inverts) the bit the first ad This project is a **Macro Assembler**, an **Interpreter** and a **Tested Standard Library** to the language. +This calculator was built with only FlipJump ([source](programs/calc.fj)): +![Calculations using only FlipJump](res/calc.gif) + ## Hello, World! A simple fj [hello-world](programs/print_tests/hello_no-stl.fj) program, not using the standard library: @@ -18,13 +23,13 @@ A simple fj [hello-world](programs/print_tests/hello_no-stl.fj) program, not usi def startup @ code_start > IO { ;code_start IO: - ;0 + ;0 // the second op is reserved for Input/Output. code_start: } def output_bit bit < IO { - IO + bit; + IO + bit; // flipping IO+0 outputs 0; flipping IO+1 outputs 1. } def output_char ascii { rep(8, i) output_bit ((ascii>>i)&1) @@ -32,7 +37,7 @@ def output_char ascii { def end_loop @ loop_label { loop_label: - ;loop_label + ;loop_label // fj finishes on a self loop } startup @@ -44,6 +49,7 @@ def end_loop @ loop_label { output_char 'o' output_char ',' output_char ' ' + output_char 'W' output_char 'o' output_char 'r' @@ -78,20 +84,21 @@ Cloning into 'flip-jump'... ```bash >>> python src/fj.py programs/hello_world.fj Hello, World! ->>> python src/fj.py programs/hello_no-stl.fj --no-stl -Hello, World! ``` - - The --no-stl flag tells the assembler not to include the standard library. The flag is needed as we implemented the macros ourselves. +![Hello World in FlipJump](res/hello.gif) + + - The --no-stl flag tells the assembler not to include the standard library. for example: `python src/fj.py programs/hello_no-stl.fj --no-stl`. + - the -w [WIDTH] flag allows compiling the .fj files to a WIDTH-bits memory width. WIDTH is 64 by default. - You can use the -o flag to save the assembled file for later use too. - - you can find all the different flags with `python src/fj.py -h` + - you can find all the different flags with `python src/fj.py -h`. You can also **[Test the project](tests/README.md)** with the project's tests, and with your tests. You can also assemble and run separately: ```bash ->>> fja.py hello.fj -o hello_world.fjm ->>> fji.py hello.fjm +>>> fj.py --asm hello.fj -o hello_world.fjm +>>> fj.py --run hello_world.fjm Hello, World! ``` @@ -105,22 +112,20 @@ You can also use the faster (stable, but still in development) cpp-based interpr Hello, World! ``` - # Project Structure -**[src](src)** (assembler + interpreter source files): +**[src](src/README.md)** (assembler + interpreter source files): + - fj.py - the FlipJump Assembler & Interpreter script. + - fjm.py - read/write .fjm (flip-jump-memory) files. + - fjm_run.py - interpret / debug assembled fj files. - fj_parser.py - pythonic lex/yacc parser. - preprocessor.py - unwind all macros and reps. - - assembler.py - assembles the macroless fj file. - - fjm_run.py - interpreter assembled fj files. - - defs.py - classes/functions/constants used throughout the project. - - fjm.py - read/write .fjm (flip-jump-memory) files. - - fja.py - the FlipJump Assembler script. - - fji.py - the FlipJump Interpreter script. - - fj.py - the FlipJump Assembler & Interpreter script. + - assembler.py - assembles the macro-less fj file. + - [more...](src/README.md) + other branches: - [cpp_fji/](https://github.com/tomhea/flip-jump/tree/cpp-interpreter/src/cpp_fji) - the cpp interpreter (much faster, about 2Mfj/s). - - [riscv2fj/](https://github.com/tomhea/flip-jump/tree/riscv2fj/src/riscv2fj) - translates a riscv-executable to an equivalent fj code. + - [riscv2fj/](https://github.com/tomhea/flip-jump/tree/riscv2fj/src/riscv2fj) - translates a riscv-executable to an equivalent fj code. **[stl](stl)** (standard library files - macros. [list of all macros](https://esolangs.org/wiki/FlipJump#The_Standard_Library)): - runlib.fj - constants and initialization macros. @@ -137,12 +142,12 @@ other branches: - [calc.fj](programs/calc.fj) - command line calculator for 2 hex/dec numbers: ```a [+-*/%] b```. - [func_tests/](programs/func_tests) - performs function calls and operations on stack. - [hexlib_tests/](programs/hexlib_tests) - tests for the macros in stl/hexlib.fj. - - [quine16.fj](programs/quine16.fj) - a 16-bits quine by [lestrozi](https://github.com/lestrozi); prints itself. + - [quine16.fj](programs/quine16.fj) - a 16-bits quine by [lestrozi](https://github.com/lestrozi); when assembled with `-w16 -v0` - prints itself. - [pair_ns.fj](programs/concept_checks/pair_ns.fj) - simulating the concept of a Class using a namespace. - [print_dec.fj](programs/print_tests/print_dec.fj) - prints binary variables as decimals. - [multi_comp/](programs/multi_comp) - simulates a big project (compilation of multiple files). -**[tests](tests)** (FlipJump programs), for example: +**[tests](tests/README.md)** (FlipJump programs), for example: - compiled/ - the designated directory for the assembled tests files. - inout/ - .in and .out files for each test in the folder above. - conftest.py - pytest configuration file. @@ -158,8 +163,11 @@ A very extensive explanation can be found on the [GitHub wiki page](https://gith More detailed explanation and the **specifications of the FlipJump assembly** can be found on the [FlipJump esolangs page](https://esolangs.org/wiki/FlipJump). -Start by reading the [bitlib.fj](stl/bitlib.fj) standard library file. That's where the FlipJump magic begins. +Read more about the [flip-jump source files](src/README.md) and [how to run the tests](tests/README.md). + +If you want to contribute to this project, read the [CONTRIBUTING.md](CONTRIBUTING.md) file, and take a look at the [Discussions](https://github.com/tomhea/flip-jump/discussions/148). + +If you are new to FlipJump and you want to learn how modern computation can be executed using FlipJump, Start by reading the [bitlib.fj](stl/bitlib.fj) standard library file (start with `xor`, `if`). That's where the FlipJump magic begins. -If you want to contribute to this project, open a pull request, and start [Discussing](https://github.com/tomhea/flip-jump/discussions/148). +You can also write and run programs for yourself! It is just [that](README.md#how-to-run) easy :) -You can also write and run programs for yourself! It is just [that](#how-to-run) easy :) diff --git a/calc.fjm b/calc.fjm new file mode 100644 index 0000000..44bfb2e Binary files /dev/null and b/calc.fjm differ diff --git a/programs/calc.fj b/programs/calc.fj index a2f81f2..ebb7839 100644 --- a/programs/calc.fj +++ b/programs/calc.fj @@ -10,12 +10,9 @@ loop: should_quit: getch remove_spaces - line_ended finish, finish, before_start + line_ended finish, finish, err_loop before_start: - line_ended loop_new_line, finish, start - loop_new_line: - output '\n' - ;loop + line_ended loop, finish, start start: insert_number a @@ -41,7 +38,6 @@ loop: do_print: - output '\n' print_int a output '\n' @@ -52,16 +48,15 @@ loop: err_loop: line_ended print_err, print_err, err_getch print_err: - bit.print 8, err_string + bit.print 7, err_string line_ended loop, finish, finish finish: - output '\n' loop -def remove_spaces @ main_loop, try2, next_ascii, end < space1, ascii, space2 { +def remove_spaces @ main_loop, try2, next_ascii, end < space1, space2, ascii { main_loop: bit.cmp 8, ascii, space1, try2, next_ascii, try2 try2: @@ -191,15 +186,9 @@ def check_quit true, false @ try_quit1 < ascii, quit1, quit2 { -// does not print new-line -def getch @ check_n, do_print, end < end_line_r, end_line_n, ascii { +// does not echo input characters +def getch < ascii { bit.input ascii - bit.cmp 8, ascii, end_line_r, check_n, end, check_n - check_n: - bit.cmp 8, ascii, end_line_n, do_print, end, do_print - do_print: - bit.print ascii - end: } @@ -247,8 +236,5 @@ quit1: bit.vec 8, 'Q' quit2: bit.vec 8, 'q' -err_string: bit.str "\nError!\n" +err_string: bit.str "Error!\n" prompt_string: bit.str "> " - -// bit.ptr_init -// bit.stack 20 diff --git a/programs/quine16.fj b/programs/quine16.fj index 39bf637..8c9a7ad 100644 --- a/programs/quine16.fj +++ b/programs/quine16.fj @@ -1,4 +1,4 @@ -// $ python3 src/fja.py -w 16 quine16.fj -o quine16.fjm && python3 src/fji.py -s quine16.fjm >/tmp/output && diff quine16.fjm /tmp/output && echo "quine!" +// $ python3 src/fj.py --asm -w 16 -v 0 quine16.fj -o quine16.fjm && python3 src/fj.py --run -s quine16.fjm >/tmp/output && diff quine16.fjm /tmp/output && echo "quine!" // // Author: Luis Fernando Estrozi ( https://github.com/lestrozi ) // @@ -16,10 +16,10 @@ // I took the liberty to define a Quine for FlipJump as a binary (fjm) that prints itself (including the header struct). // // First, create quine16.fjm from the source: -// `$ python3 src/fja.py -w 16 quine16.fj -o quine16.fjm` +// `$ python3 src/fj.py --asm -w 16 quine16.fj -o quine16.fjm` // // Then the quine can be checked using: -// `$ python3 fji.py -s quine16.fjm >output` +// `$ python3 fj.py --run -s quine16.fjm >output` // `$ diff quine16.fjm output` // // diff --git a/programs/simple_math_checks/series_sum.fj b/programs/simple_math_checks/series_sum.fj new file mode 100644 index 0000000..7dbd785 --- /dev/null +++ b/programs/simple_math_checks/series_sum.fj @@ -0,0 +1,58 @@ +N = 8 // 32bits + +startup + +// starting printings: +output "params: a1 = " +print_hex_as_decimal a1 +output ", d = " +print_hex_as_decimal d +output ", n = " +print_hex_as_decimal n +output ".\n\n" + + +hex.mov N, n_1, n +hex.dec N, n_1 // n_1 = n-1 +hex.mul N, an, n_1, d // an = d(n-1) +hex.add N, an, a1 // an = a1 + d(n-1) + +// print an +output "an = " +print_hex_as_decimal an +output ".\n" + +hex.add N, an, a1 // a1 + an +hex.mul N, s, an, n // s = (a1+an) * n +hex.shr_bit N, s // s = (a1+an) * n/2 + +// print sum +output "Sum(a1, a2, ..., an) = " +print_hex_as_decimal s +output ".\n" + +loop +hex.init // inits the hex.mul + + +def print_hex_as_decimal hexxx < num, ret, print_num { + hex2bit N, num, hexxx + fcall print_num, ret +} + +print_num: + bit.print_dec_int N*4, num + ;ret +num: bit.vec N*4 +ret: bit + + +// inputs and variables: +a1: hex.vec 8, 1 +d: hex.vec 8, 3 +n: hex.vec 8, 12 + +an: hex.vec 8 +s: hex.vec 8 + +n_1: hex.vec N diff --git a/res/breakpoint.jpg b/res/breakpoint.jpg new file mode 100644 index 0000000..1344b69 Binary files /dev/null and b/res/breakpoint.jpg differ diff --git a/res/calc.gif b/res/calc.gif new file mode 100644 index 0000000..953f50d Binary files /dev/null and b/res/calc.gif differ diff --git a/res/calc__asm.jpg b/res/calc__asm.jpg new file mode 100644 index 0000000..29e7481 Binary files /dev/null and b/res/calc__asm.jpg differ diff --git a/res/calc__run.jpg b/res/calc__run.jpg new file mode 100644 index 0000000..a6635ca Binary files /dev/null and b/res/calc__run.jpg differ diff --git a/res/calc_stats.png b/res/calc_stats.png new file mode 100644 index 0000000..6904377 Binary files /dev/null and b/res/calc_stats.png differ diff --git a/res/hello.gif b/res/hello.gif new file mode 100644 index 0000000..5efe658 Binary files /dev/null and b/res/hello.gif differ diff --git a/res/pytest.gif b/res/pytest.gif new file mode 100644 index 0000000..d8ebc17 Binary files /dev/null and b/res/pytest.gif differ diff --git a/res/sum.gif b/res/sum.gif new file mode 100644 index 0000000..0297892 Binary files /dev/null and b/res/sum.gif differ diff --git a/res/test_parallel.gif b/res/test_parallel.gif new file mode 100644 index 0000000..a7652b4 Binary files /dev/null and b/res/test_parallel.gif differ diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..a5e2212 --- /dev/null +++ b/src/README.md @@ -0,0 +1,79 @@ +# FlipJump Source Code + +## The FlipJump Macro-Assembler + +The assembler has 4 steps: +- parsing the .fj text files into a dictionary of macros and their ops ([fj_parser.py](fj_parser.py)). +- resolving (unwinding) the macros (and reps) to get a straight stream of ops ([preprocessor.py](preprocessor.py)). +- resolving the label values and getting the ops binary data ([assembler.py](assembler.py)). +- writing the binary data into the executable ([fjm.py](fjm.py)). + +The whole process is executed within the [assemble()](assembler.py) function. +![Assembly of calc.fj](../res/calc__asm.jpg) + +- The [ops.py](ops.py) file contains the classes of the different ops. +- The [expr.py](expr.py) file contains the expression class (Expr), used to maintain a mathematical expression based on numbers and labels. + +## The FlipJump Interpreter + +The Interpreter ([fjm_run.py](fjm_run.py)) stores the entire memory in a dictionary {address: value}, and supports unaligned-word access. + +The whole interpretation is done within the [run()](fjm_run.py) function (also uses [fjm.py](fjm.py) to read the fjm file). +![Running the compiled calculator](../res/calc__run.jpg) + +The Interpreter has a built-in debugger, and it's activated by specifying breakpoints when called (via the [BreakpointHandler](breakpoints.py)). +The debugger can stop on the next breakpoint, or on a fixed number of executed ops after the current breakpoint. +In order to call the debugger with the right labels, get familiar with the [generating label names](README.md#Generated-Label-Names) (and see the debugger-image there). + +The [macro_usage_graph.py](macro_usage_graph.py) file allows presenting the macro-usage in a graph: +![The macro-usage statistics of calc.fj](../res/calc_stats.png) + +### FJM versions + +The .fjm file currently has 4 versions: + +0. The basic version +1. The normal version (more configurable than the basic version) +2. The relative-jumps version (good for further compression) +3. The compressed version + +You can specify the version you want with the `-v VERSION` flag.
+The assembler chooses **by default** version **3** if the `--outfile` is specified, and version **1** if it isn't. + +### Generated Label Names + +The generated label string is a concatenation of the macro call tree, each separated by '---', and finish with the label local-name. + +Each macro call string is as follows:\ +short_file_name **:** line **:** macro_called + +So if a->bit.b->hex.c->my_label: (a, bit.b called from file f2 lines 3,5; hex.c from file s1, line 72), the label's name will be:\ +f2:3:a---f2:5:bit.b---s1:72:hex.c---my_label + +On a rep-call (on index==i), the macro call string is:\ +short_file_name : line : rep{i} : macro_called\ +for example: f1:32:rep6:hex.print---f2:17:print_bit---print_label + +the short_file_name is (by default) s1,s2,s3,.. for the standard library files (in the order of [stl/conf.json - all](../stl/conf.json)), +and f1,f2,f3,.. for the compiled .fj files, in the order they are mentioned to the compiler (or appear in the test-line). + +You can place breakpoints to stop on specific labels using the `-d`, followed by a `-b` and a label name (or `-B` and a part of a label name). For example: +![Debugging Demo](../res/breakpoint.jpg) + +## More Files + +- The [fj.py](fj.py) file is the main FlipJump cli-script. run with --help to see its capabilities. +- The [fjm.py](fjm.py) file helps to read and write a .fjm file. +- The [defs.py](defs.py) file contains functionality used across the source files, and the project's definitions. +- The [exceptions.py](exceptions.py) file contains exceptions definitions. +- The [io_devices/](io_devices) folder contains modules for different Input/Output-handling classes. The standard one is [StandardIO.py](io_devices/StandardIO.py), and the tests uses the [FixedIO.py](io_devices/FixedIO.py). + + +# Read More + +The FlipJump source is built in a way that allows simple addition of new features. + +Every addition should be supported from the parsing level to the phase that is disappears, in the progression found in assemble() in [assembler](assembler.py). + +For example, if you want to add a new operation a@b that calculates a^2+b^2 or a! for factorial(a), it is simple as adding a parsing rule in [fj_parser.py](fj_parser.py), then adding the function to the op_string_to_function() in [expr.py](expr.py). + diff --git a/src/assembler.py b/src/assembler.py index c0dc773..5c43279 100644 --- a/src/assembler.py +++ b/src/assembler.py @@ -1,190 +1,222 @@ -import os -import pickle -from time import time -from tempfile import mkstemp +import dataclasses +from collections import defaultdict +from pathlib import Path +from typing import Deque, List, Dict, Tuple, Optional import fjm from fj_parser import parse_macro_tree from preprocessor import resolve_macros -from defs import eval_all, Verbose, SegmentEntry, FJAssemblerException, OpType +from defs import PrintTimer, save_debugging_labels +from ops import FlipJump, WordFlip, LastPhaseOp, NewSegment, ReserveBits, Padding +from exceptions import FJAssemblerException, FJException, FJWriteFjmException -def lsb_first_bin_array(int_value, bit_size): - return [int(c) for c in bin(int_value & ((1 << bit_size) - 1))[2:].zfill(bit_size)[-bit_size:]][::-1][:bit_size] +def assert_address_in_memory(w: int, address: int): + if address < 0 or address >= (1 << w): + raise FJAssemblerException(f"Not enough space with the {w}-width.") + + +def validate_addresses(w, first_address, last_address): + if first_address % w != 0 or last_address % w != 0: + raise FJAssemblerException(f'segment boundaries are unaligned: ' + f'[{hex(first_address)}, {hex(last_address - 1)}].') -def write_flip_jump(bits, f, j, w): - bits += lsb_first_bin_array(f, w) - bits += lsb_first_bin_array(j, w) + assert_address_in_memory(w, first_address) + assert_address_in_memory(w, last_address) -def close_segment(w, segment_index, boundary_addresses, writer, first_address, last_address, bits, wflips): +def add_segment_to_fjm(w: int, + fjm_writer: fjm.Writer, + first_address: int, last_address: int, + fj_words: List[int], wflip_words: List[int]) -> None: + validate_addresses(w, first_address, last_address) if first_address == last_address: return - assert_none_crossing_segments(segment_index, first_address, last_address, boundary_addresses) - data_start, data_length = writer.add_data(bits + wflips) + data_words = fj_words + wflip_words + data_start = fjm_writer.add_data(data_words) + + segment_start_address = first_address // w segment_length = (last_address - first_address) // w - if segment_length < data_length: - raise FJAssemblerException(f'segment-length is smaller than data-length: {segment_length} < {data_length}') - writer.add_segment(first_address // w, segment_length, data_start, data_length) - - bits.clear() - wflips.clear() - - -def clean_segment_index(index, boundary_addresses): - clean_index = 0 - for entry in boundary_addresses[:index]: - if entry[0] == SegmentEntry.WflipAddress: - clean_index += 1 - return clean_index - - -def assert_none_crossing_segments(curr_segment_index, old_address, new_address, boundary_addresses): - min_i = None - min_seg_start = None - - last_start = None - last_start_i = None - for i, entry in enumerate(boundary_addresses): - if entry[0] == SegmentEntry.StartAddress: - last_start = entry[1] - last_start_i = i - if entry[0] == SegmentEntry.WflipAddress: - if entry[1] != last_start: - if old_address < last_start < new_address: - if min_i is None or min_seg_start > last_start: - min_i = last_start_i - min_seg_start = last_start - - if min_i is not None: - raise FJAssemblerException(f"Overlapping segments (address {hex(new_address)}): " - f"seg[{clean_segment_index(curr_segment_index, boundary_addresses)}]" - f"=({hex(boundary_addresses[curr_segment_index][1])}..) and " - f"seg[{clean_segment_index(min_i, boundary_addresses)}]=({hex(min_seg_start)}..)") - - -def get_next_wflip_entry_index(boundary_addresses, index): - length = len(boundary_addresses) - while boundary_addresses[index][0] != SegmentEntry.WflipAddress: - index += 1 - if index >= length: - raise FJAssemblerException(f'No WflipAddress entry found in boundary_addresses.') - return index - - -def labels_resolve(ops, labels, boundary_addresses, w, writer, - *, verbose=False): # TODO handle verbose? - if max(e[1] for e in boundary_addresses) >= (1 << w): - raise FJAssemblerException(f"Not enough space with the {w}-width.") - bits = [] - wflips = [] - segment_index = 0 - last_start_seg_index = segment_index - first_address = boundary_addresses[last_start_seg_index][1] - wflip_address = boundary_addresses[get_next_wflip_entry_index(boundary_addresses, 0)][1] + try: + fjm_writer.add_segment(segment_start_address, segment_length, data_start, len(data_words)) + except FJWriteFjmException as e: + raise FJAssemblerException(f"failed to add the segment: " + f"{fjm_writer.get_segment_addresses_repr(segment_start_address, segment_length)}.") from e + + fj_words.clear() + wflip_words.clear() + + +@dataclasses.dataclass +class WFlipSpot: + list: List[int] + index: int + address: int + + +class BinaryData: + def __init__(self, w: int, first_segment: NewSegment): + self.w = w + + self.first_address = first_segment.start_address + self.wflip_address = first_segment.wflip_start_address + + self.current_address = self.first_address + + self.fj_words: List[int] = [] + self.wflip_words: List[int] = [] + + self.padding_ops_indices: List[int] = [] # indices in self.fj_words + + # return_address -> { (f3, f2, f1, f0) -> start_flip_address } + self.wflips_dict: Dict[int, Dict[Tuple[int, ...],]] = defaultdict(lambda: {}) + + def get_wflip_spot(self) -> WFlipSpot: + if self.padding_ops_indices: + index = self.padding_ops_indices.pop() + return WFlipSpot(self.fj_words, index, self.first_address + self.w * index) + + wflip_spot = WFlipSpot(self.wflip_words, len(self.wflip_words), self.wflip_address) + self.wflip_words += (0, 0) + self.wflip_address += 2*self.w + return wflip_spot + + def close_and_add_segment(self, fjm_writer: fjm.Writer) -> None: + add_segment_to_fjm(self.w, fjm_writer, self.first_address, self.wflip_address, self.fj_words, self.wflip_words) + + def insert_fj_op(self, flip: int, jump: int) -> None: + self.fj_words += (flip, jump) + self.current_address += 2*self.w + + def insert_wflip_ops(self, word_address: int, flip_value: int, return_address: int) -> None: + if 0 == flip_value: + self.insert_fj_op(0, return_address) + else: + return_dict = self.wflips_dict[return_address] + + # this is the order of flip_addresses (tested with many other orders) that produces the best + # found-statistic for searching flip_bit[:i] with different i's in return_dict. + flip_addresses = [word_address + i for i in range(self.w) if flip_value & (1 << i)][::-1] + + # insert the first op + self.insert_fj_op(flip_addresses.pop(), 0) + last_return_address_index = self.fj_words, len(self.fj_words) - 1 + + while flip_addresses: + flips_key = tuple(flip_addresses) + ops_list, last_address_index = last_return_address_index + + if flips_key in return_dict: + # connect the last op to the already created wflip-chain + ops_list[last_address_index] = return_dict[flips_key] + return + else: + # insert a new wflip op, and connect the last one to it + wflip_spot = self.get_wflip_spot() + + ops_list[last_address_index] = wflip_spot.address + return_dict[flips_key] = wflip_spot.address + + wflip_spot.list[wflip_spot.index] = flip_addresses.pop() + last_return_address_index = wflip_spot.list, wflip_spot.index + 1 + + ops_list, last_address_index = last_return_address_index + ops_list[last_address_index] = return_address + + def insert_padding(self, ops_count: int) -> None: + for i in range(len(self.fj_words), len(self.fj_words) + 2 * ops_count, 2): + self.padding_ops_indices.append(i) + self.fj_words += (0, 0) + self.current_address += ops_count * (2*self.w) + + def insert_new_segment(self, fjm_writer: fjm.Writer, first_address: int, wflip_first_address: int) -> None: + self.close_and_add_segment(fjm_writer) + + self.first_address = first_address + self.wflip_address = wflip_first_address + self.current_address = self.first_address + + self.padding_ops_indices.clear() + + def insert_reserve_bits(self, fjm_writer: fjm.Writer, new_first_address: int) -> None: + add_segment_to_fjm(self.w, fjm_writer, self.first_address, new_first_address, self.fj_words, []) + + self.first_address = new_first_address + self.current_address = self.first_address + + self.padding_ops_indices.clear() + + +def labels_resolve(ops: Deque[LastPhaseOp], labels: Dict[str, int], + w: int, fjm_writer: fjm.Writer) -> None: + """ + resolve the labels and expressions to get the list of fj ops, and add all the data and segments into the fjm_writer. + @param ops:[in]: the list ops returned from the preprocessor stage + @param labels:[in]: dictionary from label to its resolved value + @param w: the memory-width + @param fjm_writer: [out]: the .fjm file writer + """ + first_segment: NewSegment = ops.popleft() + if not isinstance(first_segment, NewSegment): + raise FJAssemblerException(f"The first op must be of type NewSegment (and not {first_segment}).") + + binary_data = BinaryData(w, first_segment) for op in ops: - ids = eval_all(op, labels) - if ids: - raise FJAssemblerException(f"Can't resolve the following names: {', '.join(ids)} (in op {op}).") - vals = [datum.val for datum in op.data] - - if op.type == OpType.FlipJump: - f, j = vals - bits += [f, j] - elif op.type == OpType.Segment: - segment_index += 2 - close_segment(w, last_start_seg_index, boundary_addresses, writer, first_address, wflip_address, bits, - wflips) - last_start_seg_index = segment_index - first_address = boundary_addresses[last_start_seg_index][1] - wflip_address = boundary_addresses[get_next_wflip_entry_index(boundary_addresses, segment_index)][1] - elif op.type == OpType.Reserve: - segment_index += 1 - last_address = boundary_addresses[segment_index][1] - close_segment(w, last_start_seg_index, boundary_addresses, writer, first_address, last_address, bits, []) - first_address = last_address - elif op.type == OpType.WordFlip: - to_address, by_address, return_address = vals - flip_bits = [i for i in range(w) if by_address & (1 << i)] - - if len(flip_bits) <= 1: - bits += [to_address + flip_bits[0] if flip_bits else 0, - return_address] - else: - bits += [to_address + flip_bits[0], - wflip_address] - next_op = wflip_address - for bit in flip_bits[1:-1]: - next_op += 2*w - wflips += [to_address+bit, - next_op] - wflips += [to_address + flip_bits[-1], - return_address] - wflip_address = next_op + 2 * w - - if wflip_address >= (1 << w): - raise FJAssemblerException(f"Not enough space with the {w}-width.") + if isinstance(op, FlipJump): + try: + binary_data.insert_fj_op(op.get_flip(labels), op.get_jump(labels)) + except FJException as e: + raise FJAssemblerException(f"Can't resolve labels in op {op}.") from e + + elif isinstance(op, WordFlip): + try: + binary_data.insert_wflip_ops(op.get_word_address(labels), op.get_flip_value(labels), + op.get_return_address(labels)) + except FJException as e: + raise FJAssemblerException(f"Can't resolve labels in op {op}.") from e + + elif isinstance(op, Padding): + binary_data.insert_padding(op.ops_count) + + elif isinstance(op, NewSegment): + binary_data.insert_new_segment(fjm_writer, op.start_address, op.wflip_start_address) + + elif isinstance(op, ReserveBits): + binary_data.insert_reserve_bits(fjm_writer, op.first_address_after_reserved) + else: raise FJAssemblerException(f"Can't resolve/assemble the next opcode - {str(op)}") - close_segment(w, last_start_seg_index, boundary_addresses, writer, first_address, wflip_address, bits, wflips) - - -def assemble(input_files, output_file, w, - *, version=0, flags=0, - warning_as_errors=True, - show_statistics=False, preprocessed_file=None, debugging_file=None, verbose=None): - if verbose is None: - verbose = set() - - writer = fjm.Writer(output_file, w, version=version, flags=flags) - - temp_preprocessed_file, temp_fd = False, 0 - if preprocessed_file is None: - temp_fd, preprocessed_file = mkstemp() - temp_preprocessed_file = True - - if Verbose.Time in verbose: - print(' parsing: ', end='', flush=True) - start_time = time() - macros = parse_macro_tree(input_files, w, warning_as_errors, verbose=Verbose.Parse in verbose) - if Verbose.Time in verbose: - print(f'{time() - start_time:.3f}s') - - if Verbose.Time in verbose: - print(' macro resolve: ', end='', flush=True) - start_time = time() - ops, labels, boundary_addresses = resolve_macros(w, macros, output_file=preprocessed_file, - show_statistics=show_statistics, - verbose=Verbose.MacroSolve in verbose) - if Verbose.Time in verbose: - print(f'{time() - start_time:.3f}s') - - if Verbose.Time in verbose: - print(' labels resolve: ', end='', flush=True) - start_time = time() - labels_resolve(ops, labels, boundary_addresses, w, writer, verbose=Verbose.LabelSolve in verbose) - if Verbose.Time in verbose: - print(f'{time() - start_time:.3f}s') - - if Verbose.Time in verbose: - print(' create binary: ', end='', flush=True) - start_time = time() - writer.write_to_file() - if Verbose.Time in verbose: - print(f'{time() - start_time:.3f}s') - - if temp_preprocessed_file: - os.close(temp_fd) - - labels = {label: labels[label].val for label in labels} - - if debugging_file: - with open(debugging_file, 'wb') as f: - pickle.dump(labels, f, pickle.HIGHEST_PROTOCOL) - - return labels + binary_data.close_and_add_segment(fjm_writer) + + +def assemble(input_files: List[Tuple[str, Path]], w: int, fjm_writer: fjm.Writer, *, + warning_as_errors: bool = True, debugging_file_path: Optional[Path] = None, + show_statistics: bool = False, print_time: bool = True)\ + -> None: + """ + runs the assembly pipeline. assembles the input files to a .fjm. + @param input_files:[in]: a list of (short_file_name, fj_file_path). The files will to be parsed in that given order. + @param w: the memory-width + @param fjm_writer:[out]: the .fjm file writer + @param warning_as_errors: treat warnings as errors (stop execution on warnings) + @param debugging_file_path:[out]: is specified, save debug information in this file + @param show_statistics: if true shows macro-usage statistics + @param print_time: if true prints the times of each assemble-stage + """ + with PrintTimer(' parsing: ', print_time=print_time): + macros = parse_macro_tree(input_files, w, warning_as_errors) + + with PrintTimer(' macro resolve: ', print_time=print_time): + ops, labels = resolve_macros(w, macros, show_statistics=show_statistics) + + with PrintTimer(' labels resolve: ', print_time=print_time): + labels_resolve(ops, labels, w, fjm_writer) + + with PrintTimer(' create binary: ', print_time=print_time): + fjm_writer.write_to_file() + save_debugging_labels(debugging_file_path, labels) diff --git a/src/breakpoints.py b/src/breakpoints.py new file mode 100644 index 0000000..4b38a6f --- /dev/null +++ b/src/breakpoints.py @@ -0,0 +1,205 @@ +import pickle +from os import path +from pathlib import Path +from typing import Optional, List, Dict, Set + +import easygui + +import fjm + +from defs import macro_separator_string, RunStatistics, load_debugging_labels + + +class BreakpointHandlerUnnecessary(Exception): + pass + + +def display_message_box_and_get_answer(msg: str, title: str, choices: List[str]) -> str: + # might generate an 'import from collections is deprecated' warning if using easygui-version <= 0.98.3. + return easygui.buttonbox(msg, title, choices) + + +def get_nice_label_repr(label: str, pad: int = 0) -> str: + parts = label.split(macro_separator_string) + return ' ->\n'.join(f"{' '*(pad+i)}{part}" for i, part in enumerate(parts)) + + +class BreakpointHandler: + """ + Handle breakpoints (know when breakpoints happen, query user for action). + """ + def __init__(self, breakpoints: Dict[int, str], address_to_label: Dict[int, str]): + self.breakpoints = breakpoints + self.address_to_label = address_to_label + + if 0 not in self.address_to_label: + self.address_to_label[0] = 'memory_start_0x0000' + + self.next_break = None + + def should_break(self, ip: int, op_counter: int) -> bool: + return self.next_break == op_counter or ip in self.breakpoints + + def get_address_str(self, address: int) -> str: + if address in self.breakpoints and self.breakpoints[address] is not None: + label_repr = get_nice_label_repr(self.breakpoints[address], pad=4) + return f'{hex(address)[2:]}:\n{label_repr}' + elif address in self.address_to_label: + label_repr = get_nice_label_repr(self.address_to_label[address], pad=4) + return f'{hex(address)[2:]}:\n{label_repr}' + else: + address_before = max([a for a in self.address_to_label if a <= address]) + label_repr = get_nice_label_repr(self.address_to_label[address_before], pad=4) + return f'{hex(address)[2:]} ({hex(address - address_before)} after:)\n{label_repr}' + + def get_message_box_body(self, ip: int, mem: fjm.Reader, op_counter: int) -> str: + address = self.get_address_str(ip) + flip = self.get_address_str(mem.get_word(ip)) + jump = self.get_address_str(mem.get_word(ip + mem.w)) + return f'Address {address}.\n\n{op_counter} ops executed.\n\nflip {flip}.\n\njump {jump}.' + + def query_user_for_debug_action(self, ip: int, mem: fjm.Reader, op_counter: int) -> str: + title = "Breakpoint" if ip in self.breakpoints else "Debug Step" + body = self.get_message_box_body(ip, mem, op_counter) + actions = ['Single Step', 'Skip 10', 'Skip 100', 'Skip 1000', 'Continue', 'Continue All'] + + action = display_message_box_and_get_answer(body, title, actions) + if action is None: + action = 'Continue All' + return action + + def apply_debug_action(self, action: str, op_counter: int) -> None: + """ + @raise BreakpointHandlerUnnecessary for the "Continue All" action + """ + if action == 'Single Step': + self.next_break = op_counter + 1 + elif action == 'Skip 10': + self.next_break = op_counter + 10 + elif action == 'Skip 100': + self.next_break = op_counter + 100 + elif action == 'Skip 1000': + self.next_break = op_counter + 1000 + elif action == 'Continue': + self.next_break = None + elif action == 'Continue All': + self.next_break = None + raise BreakpointHandlerUnnecessary() + + +def handle_breakpoint(breakpoint_handler: BreakpointHandler, ip: int, mem: fjm.Reader, statistics: RunStatistics) \ + -> BreakpointHandler: + """ + show debug message, query user for action, apply its action. + @param breakpoint_handler: the breakpoint handler + @param ip: current ip + @param mem: the memory + @param statistics: the statistics of the current run + @return: the breakpoint handler (or None if it is not necessary anymore) + """ + print(' program break', end="", flush=True) + with statistics.pause_timer: + action = breakpoint_handler.query_user_for_debug_action(ip, mem, statistics.op_counter) + print(f': {action}') + + try: + breakpoint_handler.apply_debug_action(action, statistics.op_counter) + except BreakpointHandlerUnnecessary: + breakpoint_handler = None + + return breakpoint_handler + + +def get_breakpoints(breakpoint_addresses: Optional[Set[int]], + breakpoint_labels: Optional[Set[str]], + breakpoint_contains_labels: Optional[Set[str]], + label_to_address: Dict[str, int])\ + -> Dict[int, str]: + """ + generate the breakpoints' dictionary + """ + breakpoints = {} + + update_breakpoints_from_addresses_set(breakpoint_addresses, breakpoints) + update_breakpoints_from_breakpoint_contains_set(breakpoint_contains_labels, breakpoints, label_to_address) + update_breakpoints_from_breakpoint_set(breakpoint_labels, breakpoints, label_to_address) + + return breakpoints + + +def update_breakpoints_from_breakpoint_set(breakpoint_labels: Optional[Set[str]], + breakpoints: Dict[int, Optional[str]], + label_to_address: Dict[str, int]) -> None: + """ + add breakpoints from breakpoint_labels. + param breakpoints[in,out] - adds breakpoints to it + """ + if breakpoint_labels: + for bl in breakpoint_labels: + if bl not in label_to_address: + print(f"Warning: Breakpoint label {bl} can't be found!") + else: + address = label_to_address[bl] + breakpoints[address] = bl + + +def update_breakpoints_from_breakpoint_contains_set(breakpoint_contains_labels: Optional[Set[str]], + breakpoints: Dict[int, Optional[str]], + label_to_address: Dict[str, int]) -> None: + """ + add breakpoints generated with breakpoint_contains_labels. + param breakpoints[in,out] - adds breakpoints to it + """ + # TODO improve the speed of this part with suffix trees + if breakpoint_contains_labels: + for bcl in breakpoint_contains_labels: + for label in label_to_address: + if bcl in label: + address = label_to_address[label] + breakpoints[address] = label + + +def update_breakpoints_from_addresses_set(breakpoint_addresses: Optional[Set[int]], + breakpoints: Dict[int, Optional[str]]) -> None: + """ + add breakpoints of addresses breakpoint_addresses. + param breakpoints[in,out] - adds breakpoints to it + """ + if breakpoint_addresses: + for address in breakpoint_addresses: + breakpoints[address] = None + + +def load_labels_dictionary(debugging_file: Optional[Path], labels_file_needed: bool) -> Dict[str, int]: + """ + load the labels_dictionary from debugging_file, if possible. + """ + if debugging_file is None: + if labels_file_needed: + print(f"Warning: debugging labels can't be found! no debugging file specified.") + return {} + + if not debugging_file.is_file(): + print(f"Warning: debugging file {debugging_file} can't be found!") + return {} + + return load_debugging_labels(debugging_file) + + +def get_breakpoint_handler(debugging_file: Path, breakpoint_addresses: Set[int], breakpoint_labels: Set[str], + breakpoint_contains_labels: Set[str]) -> Optional[BreakpointHandler]: + """ + generate the breakpoint handler from the debugging file and the breakpoint sets. + @param debugging_file: the debug file path (created at assemble time) + @param breakpoint_addresses: set of addresses to break at + @param breakpoint_labels: set of labels to break at + @param breakpoint_contains_labels: set of strings, to break at every label that contains one of them + @return: the breakpoint handler + """ + labels_file_needed = any((breakpoint_addresses, breakpoint_contains_labels)) + label_to_address = load_labels_dictionary(debugging_file, labels_file_needed) + + address_to_label = {label_to_address[label]: label for label in label_to_address} + breakpoints = get_breakpoints(breakpoint_addresses, breakpoint_labels, breakpoint_contains_labels, label_to_address) + + return BreakpointHandler(breakpoints, address_to_label) if breakpoints else None diff --git a/src/defs.py b/src/defs.py index 5d4f21a..b02fd69 100644 --- a/src/defs.py +++ b/src/defs.py @@ -1,262 +1,137 @@ +from __future__ import annotations + +import dataclasses import json +import lzma from enum import IntEnum # IntEnum equality works between files. from pathlib import Path -from operator import mul, add, sub, floordiv, lshift, rshift, mod, xor, or_, and_ - - -main_macro = ('', 0) - - -# TODO use the op-strings (instead of the function) up-to the last point possible (to make deepcopy simpler) -parsing_op2func = {'+': add, '-': sub, '*': mul, '/': floordiv, '%': mod, - '<<': lshift, '>>': rshift, '^': xor, '|': or_, '&': and_, - '#': lambda x: x.bit_length(), - '?:': lambda a, b, c: b if a else c, - '<': lambda a, b: 1 if a < b else 0, - '>': lambda a, b: 1 if a > b else 0, - '<=': lambda a, b: 1 if a <= b else 0, - '>=': lambda a, b: 1 if a >= b else 0, - '==': lambda a, b: 1 if a == b else 0, - '!=': lambda a, b: 1 if a != b else 0, - } - - -class FJException(Exception): - pass - - -class FJParsingException(FJException): - pass - - -class FJPreprocessorException(FJException): - pass - - -class FJExprException(FJException): - pass - - -class FJAssemblerException(FJException): - pass - - -class FJReadFjmException(FJException): - pass - - -class FJWriteFjmException(FJException): - pass - - -def smart_int16(num): - try: - return int(num, 16) - except ValueError as ve: - raise FJException(f'{num} is not a number!') from ve - - -STL_PATH = Path(__file__).parent.parent / 'stl' -with open(STL_PATH / 'conf.json', 'r') as stl_json: - STL_OPTIONS = json.load(stl_json) - - -def get_stl_paths(): - return [STL_PATH / f'{lib}.fj' for lib in STL_OPTIONS['all']] - - -id_re = r'[a-zA-Z_][a-zA-Z_0-9]*' -dot_id_re = fr'(({id_re})|\.*)?(\.({id_re}))+' - -bin_num = r'0[bB][01]+' -hex_num = r'0[xX][0-9a-fA-F]+' -dec_num = r'[0-9]+' - -char_escape_dict = {'0': 0x0, 'a': 0x7, 'b': 0x8, 'e': 0x1b, 'f': 0xc, 'n': 0xa, 'r': 0xd, 't': 0x9, 'v': 0xb, - '\\': 0x5c, "'": 0x27, '"': 0x22, '?': 0x3f} -escape_chars = ''.join(k for k in char_escape_dict) -char = fr'[ -~]|\\[{escape_chars}]|\\[xX][0-9a-fA-F]{{2}}' - -number_re = fr"({bin_num})|({hex_num})|('({char})')|({dec_num})" -string_re = fr'"({char})*"' - +from time import time +from typing import List, Dict -def get_char_value_and_length(s): - if s[0] != '\\': - return ord(s[0]), 1 - if s[1] in char_escape_dict: - return char_escape_dict[s[1]], 2 - return int(s[2:4], 16), 4 +from ops import CodePosition, Op -class Verbose(IntEnum): - Parse = 1 - MacroSolve = 2 - LabelDict = 3 - LabelSolve = 4 - Run = 5 - Time = 6 - PrintOutput = 7 +def get_stl_paths() -> List[Path]: + """ + @return: list of the ordered standard-library paths + """ + stl_path = Path(__file__).parent.parent / 'stl' + with open(stl_path / 'conf.json', 'r') as stl_json: + stl_options = json.load(stl_json) + return [stl_path / f'{lib}.fj' for lib in stl_options['all']] class TerminationCause(IntEnum): - Looping = 0 - Input = 1 - NullIP = 2 - - def __str__(self): - return ['looping', 'input', 'ip<2w'][self.value] - - -class SegmentEntry(IntEnum): - StartAddress = 0 - ReserveAddress = 1 - WflipAddress = 2 - - -class OpType(IntEnum): # op.data array content: - - FlipJump = 1 # expr, expr # Survives until (2) label resolve - WordFlip = 2 # expr, expr, expr # Survives until (2) label resolve - Segment = 3 # expr # Survives until (2) label resolve - Reserve = 4 # expr # Survives until (2) label resolve - Label = 5 # ID # Survives until (1) macro resolve - Macro = 6 # ID, expr [expr..] # Survives until (1) macro resolve - Rep = 7 # expr, ID, macro_call # Survives until (1) macro resolve - - -class Op: - def __init__(self, op_type, data, file, line): - self.type = op_type - self.data = data - self.file = file - self.line = line - - def __str__(self): - return f'{f"{self.type}:"[7:]:10} Data: {", ".join([str(d) for d in self.data])} ' \ - f'File: {self.file} (line {self.line})' - - def macro_trace_str(self): - assert self.type == OpType.Macro - macro_name, param_len = self.data[0] - return f'macro {macro_name}({param_len}) (File {self.file}, line {self.line})' - - def rep_trace_str(self, iter_value, iter_times): - assert self.type == OpType.Rep - _, iter_name, macro = self.data - macro_name, param_len = macro.data[0] - return f'rep({iter_name}={iter_value}, out of 0..{iter_times-1}) ' \ - f'macro {macro_name}({param_len}) (File {self.file}, line {self.line})' - - -class Expr: - def __init__(self, expr): - self.val = expr - - # replaces every string it can with its dictionary value, and evaluates anything it can. - # returns the list of unknown id's - def eval(self, id_dict, file, line): - if self.is_tuple(): - op, exps = self.val - res = [e.eval(id_dict, file, line) for e in exps] - if any(res): - return sum(res, start=[]) - else: - try: - self.val = parsing_op2func[op](*[e.val for e in exps]) - return [] - except BaseException as e: - raise FJExprException(f'{repr(e)}. bad math operation ({op}): {str(self)} in file {file} (line {line})') - elif self.is_str(): - if self.val in id_dict: - self.val = id_dict[self.val].val - return self.eval({}, file, line) - else: - return [self.val] - return [] - - def is_int(self): - return type(self.val) is int - - def is_str(self): - return type(self.val) is str - - def is_tuple(self): - return type(self.val) is tuple - - def __str__(self): - if self.is_tuple(): - op, exps = self.val - if len(exps) == 1: - e1 = exps[0] - return f'(#{str(e1)})' - elif len(exps) == 2: - e1, e2 = exps - return f'({str(e1)} {op} {str(e2)})' - else: - e1, e2, e3 = exps - return f'({str(e1)} ? {str(e2)} : {str(e3)})' - if self.is_str(): - return self.val - if self.is_int(): - return hex(self.val)[2:] - raise FJExprException(f'bad expression: {self.val} (of type {type(self.val)})') - - -def eval_all(op, id_dict=None): - if id_dict is None: - id_dict = {} - - ids = [] - for expr in op.data: - if type(expr) is Expr: - ids += expr.eval(id_dict, op.file, op.line) - if op.type == OpType.Rep: - macro_op = op.data[2] - ids += eval_all(macro_op, id_dict) - return ids - - -def get_all_used_labels(ops): - used_labels, declared_labels = set(), set() - for op in ops: - if op.type == OpType.Rep: - n, i, macro_call = op.data - used_labels.update(n.eval({}, op.file, op.line)) - new_labels = set() - new_labels.update(*[e.eval({}, op.file, op.line) for e in macro_call.data[1:]]) - used_labels.update(new_labels - {i}) - elif op.type == OpType.Label: - declared_labels.add(op.data[0]) - else: - for expr in op.data: - if type(expr) is Expr: - used_labels.update(expr.eval({}, op.file, op.line)) - return used_labels, declared_labels - - -def id_swap(op, id_dict): - new_data = [] - for datum in op.data: - if type(datum) is str and datum in id_dict: - swapped_label = id_dict[datum] - if not swapped_label.is_str(): - raise FJExprException(f'Bad label swap (from {datum} to {swapped_label}) in {op}.') - new_data.append(swapped_label.val) - else: - new_data.append(datum) - op.data = tuple(new_data) - - -def new_label(counter, name): - if name == '': - return Expr(f'_.label{next(counter)}') - else: - return Expr(f'_.label{next(counter)}_{name}') - - -wflip_start_label = '_.wflip_area_start_' - - -def next_address() -> Expr: - return Expr('$') + Looping = 0 # Finished by jumping to the last op, without flipping it (the "regular" finish/exit) + EOF = 1 # Finished by reading input when there is no more input + NullIP = 2 # Finished by jumping back to the initial op 0 (bad finish) + UnalignedWord = 3 # FOR FUTURE SUPPORT - tried to access an unaligned word (bad finish) + UnalignedOp = 4 # FOR FUTURE SUPPORT - tried to access a dword-unaligned op (bad finish) + + def __str__(self) -> str: + return ['looping', 'EOF', 'ip<2w', 'unaligned-word', 'unaligned-op'][self.value] + + +macro_separator_string = "---" + +io_bytes_encoding = 'raw_unicode_escape' + + +_debug_json_encoding = 'utf-8' +_debug_json_lzma_format = lzma.FORMAT_RAW +_debug_json_lzma_filters: List[Dict[str, int]] = [{"id": lzma.FILTER_LZMA2}] + + +def save_debugging_labels(debugging_file_path: Path, labels: Dict[str, int]) -> None: + """ + save the labels' dictionary to the debugging-file as lzma2-compressed json + @param debugging_file_path: the file's path + @param labels: the labels' dictionary + """ + if debugging_file_path: + with open(debugging_file_path, 'wb') as f: + data = json.dumps(labels).encode(_debug_json_encoding) + compressed_data = lzma.compress(data, format=_debug_json_lzma_format, filters=_debug_json_lzma_filters) + f.write(compressed_data) + + +def load_debugging_labels(debugging_file_path: Path) -> Dict[str, int]: + """ + loads and decompresses the labels' dictionary from the lzma2-compressed debugging-file + @param debugging_file_path: the file's path + @return: the labels' dictionary + """ + if debugging_file_path: + with open(debugging_file_path, 'rb') as f: + compressed_data = f.read() + data = lzma.decompress(compressed_data, format=_debug_json_lzma_format, filters=_debug_json_lzma_filters) + return json.loads(data.decode(_debug_json_encoding)) + + +class PrintTimer: + """ + prints the time a code segment took. + usage: + with PrintTimer('long_function time: '): + long_function() + """ + def __init__(self, init_message: str, *, print_time: bool = True): + self.init_message = init_message + self.print_time = print_time + + def __enter__(self) -> None: + if self.print_time: + self.start_time = time() + print(self.init_message, end='', flush=True) + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + if self.print_time: + print(f'{time() - self.start_time:.3f}s') + + +@dataclasses.dataclass +class Macro: + """ + The python representation of a .fj macro (macro declaration). + """ + params: List[str] + local_params: List[str] + ops: List[Op] + namespace: str + code_position: CodePosition + + +class RunStatistics: + """ + maintains times and counters of the current run. + """ + class PauseTimer: + def __init__(self): + self.paused_time = 0 + + def __enter__(self): + self.pause_start_time = time() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.paused_time += time() - self.pause_start_time + + def __init__(self, w: int): + self._op_size = 2 * w + self._after_null_flip = 2 * w + + self.op_counter = 0 + self.flip_counter = 0 + self.jump_counter = 0 + + self._start_time = time() + self.pause_timer = self.PauseTimer() + + def get_run_time(self) -> float: + return time() - self._start_time - self.pause_timer.paused_time + + def register_op(self, ip: int, flip_address: int, jump_address: int) -> None: + self.op_counter += 1 + if flip_address >= self._after_null_flip: + self.flip_counter += 1 + if jump_address != ip + self._op_size: + self.jump_counter += 1 diff --git a/src/exceptions.py b/src/exceptions.py new file mode 100644 index 0000000..8a098d5 --- /dev/null +++ b/src/exceptions.py @@ -0,0 +1,26 @@ +class FJException(Exception): + pass + + +class FJParsingException(FJException): + pass + + +class FJPreprocessorException(FJException): + pass + + +class FJExprException(FJException): + pass + + +class FJAssemblerException(FJException): + pass + + +class FJReadFjmException(FJException): + pass + + +class FJWriteFjmException(FJException): + pass diff --git a/src/expr.py b/src/expr.py new file mode 100644 index 0000000..56cff74 --- /dev/null +++ b/src/expr.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +from typing import Union, Tuple, Set, Dict +from operator import mul, add, sub, floordiv, lshift, rshift, mod, xor, or_, and_ + +from exceptions import FJExprException + + +# dictionary from a math-op string, to its pythonic function. +# @note: if changed, update Expr.__str__(). +op_string_to_function = { + '+': add, '-': sub, '*': mul, '/': floordiv, '%': mod, + '<<': lshift, '>>': rshift, '^': xor, '|': or_, '&': and_, + '#': lambda x: x.bit_length(), + '?:': lambda a, b, c: b if a else c, + '<': lambda a, b: 1 if a < b else 0, + '>': lambda a, b: 1 if a > b else 0, + '<=': lambda a, b: 1 if a <= b else 0, + '>=': lambda a, b: 1 if a >= b else 0, + '==': lambda a, b: 1 if a == b else 0, + '!=': lambda a, b: 1 if a != b else 0, +} + + +class Expr: + """ + The python representation of a .fj expression (from labels, consts and math-ops) + """ + def __init__(self, expr: Union[int, str, Tuple[str, Tuple[Expr, ...]]]): + self.value = expr + + def is_int(self) -> bool: + return isinstance(self.value, int) + + def __int__(self): + if self.is_int(): + return self.value + raise FJExprException(f"Can't resolve labels: {', '.join(self.all_unknown_labels())}") + + def all_unknown_labels(self) -> Set[str]: + """ + @return: all labels used (recursively) in this expression. + """ + if isinstance(self.value, int): + return set() + if isinstance(self.value, str): + return {self.value} + return set(label for expr in self.value[1] for label in expr.all_unknown_labels()) + + def eval_new(self, params_dict: Dict[str, Expr]) -> Expr: + """ + creates a new Expr, as minimal as possible. + replaces every string it can with its dictionary value, and evaluates any op it can. + @param params_dict: the label->ExprValue dictionary to be used + @raise FJExprException if math op failed + @return: the new Expr + """ + if isinstance(self.value, int): + return Expr(self.value) + + if isinstance(self.value, str): + if self.value in params_dict: + return params_dict[self.value].eval_new({}) + return Expr(self.value) + + op, args = self.value + evaluated_args: Tuple[Expr, ...] = tuple(e.eval_new(params_dict) for e in args) + if all(isinstance(e.value, int) for e in evaluated_args): + try: + return Expr(op_string_to_function[op](*(arg.value for arg in evaluated_args))) + except Exception as e: + raise FJExprException(f'{repr(e)}. bad math operation ({op}): {str(self)}.') + return Expr((op, evaluated_args)) + + def exact_eval(self, labels: Dict[str, int]) -> int: + """ + evaluates the expression's value with the labels + @param labels: the label->value dictionary to be used + @raise FJExprException if it can't evaluate + @return: the integer-value of the expression + """ + if isinstance(self.value, int): + return self.value + + if isinstance(self.value, str): + if self.value in labels: + return labels[self.value] + raise FJExprException(f"Can't evaluate label {self.value} in expression {self}") + + op, args = self.value + evaluated_args: Tuple[int, ...] = tuple(e.exact_eval(labels) for e in args) + try: + return op_string_to_function[op](*evaluated_args) + except Exception as e: + raise FJExprException(f'{repr(e)}. bad math operation ({op}): {str(self)}.') + + def __str__(self) -> str: + if isinstance(self.value, tuple): + op, expressions = self.value + if len(expressions) == 1: + e1 = expressions[0] + return f'(#{str(e1)})' + elif len(expressions) == 2: + e1, e2 = expressions + return f'({str(e1)} {op} {str(e2)})' + else: + e1, e2, e3 = expressions + return f'({str(e1)} ? {str(e2)} : {str(e3)})' + if isinstance(self.value, str): + return self.value + if isinstance(self.value, int): + return hex(self.value)[2:] + raise FJExprException(f'bad expression: {self.value} (of type {type(self.value)})') + + +def get_minimized_expr(op: str, params: Tuple[Expr, ...]) -> Expr: + """ + tries to calculate the op on the params, if possible. returns the resulting Expr. + @param op: the math-op string + @param params: the op parameters + @return: the expression + """ + if all(param.is_int() for param in params): + return Expr(op_string_to_function[op](*map(int, params))) + else: + return Expr((op, params)) diff --git a/src/fj.py b/src/fj.py index ceaa780..8d68228 100644 --- a/src/fj.py +++ b/src/fj.py @@ -1,110 +1,343 @@ import os import argparse -from os.path import isfile, abspath -from tempfile import mkstemp - -from assembler import assemble -from fjm_run import debug_and_run -from defs import Verbose, FJReadFjmException, FJException, get_stl_paths - - -def main(): - parser = argparse.ArgumentParser(description='Assemble and Run FlipJump programs.') - parser.add_argument('file', help="the FlipJump files.", nargs='+') - parser.add_argument('-s', '--silent', help="don't show assemble & run times", action='store_true') - parser.add_argument('-o', '--outfile', help="output assembled file.") - parser.add_argument('--no-macros', help="output no-macros file.") - parser.add_argument('-d', '--debug', help="debug file (used for breakpoints).", nargs='?', const=True) - parser.add_argument('-v', '--version', help="fjm version", type=int, default=0) - parser.add_argument('-f', '--flags', help="default running flags", type=int, default=0) - parser.add_argument('-w', '--width', help="specify memory-width. 64 by default.", - type=int, default=64, choices=[8, 16, 32, 64]) - parser.add_argument('--Werror', help="make all warnings into errors.", action='store_true') - parser.add_argument('--no-stl', help="don't assemble/link the standard library files.", action='store_true') - parser.add_argument('--stats', help="show macro usage statistics.", action='store_true') - parser.add_argument('-b', '--breakpoint', help="pause when reaching this label", - default=[], action='append') - parser.add_argument('-B', '--any_breakpoint', help="pause when reaching any label containing this", - default=[], action='append') +import lzma +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Tuple, List, Callable - args = parser.parse_args() +import assembler +import fjm_run +import fjm +from io_devices.StandardIO import StandardIO + +from defs import get_stl_paths +from exceptions import FJReadFjmException +from breakpoints import get_breakpoint_handler +ErrorFunc = Callable[[str], None] - ##### - ASSEMBLE +def get_temp_directory_suffix(args: argparse.Namespace) -> str: + """ + create a suffix for the temp directory name, using args. + @param args: the parsed arguments + @return: the suffix + """ + return f'__{"_".join(map(os.path.basename, args.files))}__temp_directory' - verbose_set = set() - if not args.silent: - verbose_set.add(Verbose.Time) + +def get_file_tuples(args: argparse.Namespace) -> List[Tuple[str, Path]]: + """ + get the list of .fj files to be assembled (stl + files). + @param args: the parsed arguments + @return: a list of file-tuples - (file_short_name, file_path) + """ + file_tuples = [] if not args.no_stl: - args.file = get_stl_paths() + args.file - for file in args.file: - file = abspath(file) - if not file.endswith('.fj'): - parser.error(f'file {file} is not a .fj file.') - if not isfile(abspath(file)): - parser.error(f'file {file} does not exist.') - - temp_assembled_file, temp_assembled_fd = False, 0 - if args.outfile is None: - temp_assembled_fd, args.outfile = mkstemp() - temp_assembled_file = True - else: - if not args.outfile.endswith('.fjm'): - parser.error(f'output file {args.outfile} is not a .fjm file.') - - temp_debug_file, temp_debug_fd = False, 0 - if args.debug is None and (len(args.breakpoint) > 0 or len(args.any_breakpoint) > 0): - print(f"Warning - breakpoints are used but the debugging flag (-d) is not specified. " - f"Debugging data will be saved.") - args.debug = True - if args.debug is True: - temp_debug_fd, args.debug = mkstemp() - temp_debug_file = True + for i, stl_path in enumerate(get_stl_paths(), start=1): + file_tuples.append((f"s{i}", stl_path)) - try: - assemble(args.file, args.outfile, args.width, - version=args.version, flags=args.flags, - warning_as_errors=args.Werror, - show_statistics=args.stats, verbose=verbose_set, - preprocessed_file=args.no_macros, debugging_file=args.debug) - except FJException as e: - print() - print(e) - exit(1) + for i, file in enumerate(args.files, start=1): + file_tuples.append((f"f{i}", Path(file))) + + return file_tuples - if temp_assembled_file: - os.close(temp_assembled_fd) +def verify_file_exists(error_func: ErrorFunc, path: Path) -> None: + """ + verify that the file exists. + @param error_func: the parser's error function + @param path: the file's path + """ + if not path.is_file(): + error_func(f'file {path} does not exist.') - ##### - RUN +def verify_fj_files(error_func: ErrorFunc, file_tuples: List[Tuple[str, Path]]) -> None: + """ + verify that all files exist and with the right suffix. + @param error_func: the parser's error function + @param file_tuples: a list of file-tuples - (file_short_name, file_path) + """ + for _, path in file_tuples: + verify_file_exists(error_func, path) + if '.fj' != path.suffix: + error_func(f'file {path} is not a .fj file.') - verbose_set = {Verbose.PrintOutput} - if not args.silent: - verbose_set.add(Verbose.Time) + +def verify_fjm_file(error_func: ErrorFunc, path: Path) -> None: + """ + verify that this file exists and with the right suffix. + @param error_func: the parser's error function + @param path: the file's path + """ + verify_file_exists(error_func, path) + if '.fjm' != path.suffix: + error_func(f'file {path} is not a .fjm file.') + + +def get_files_paths(args: argparse.Namespace, error_func: ErrorFunc, temp_dir_name: str) -> Tuple[Path, Path, Path]: + """ + generate the files paths from args, and create temp paths under temp_dir_name if necessary. + @param args: the parsed arguments + @param error_func: parser's error function + @param temp_dir_name: the temp directory's name + @return: the path of the debug-file, the (to-be-compiled) fjm, and the input fjm + """ + out_fjm_path = get_fjm_file_path(args, error_func, temp_dir_name) + debug_path = get_debug_file_path(args, error_func, temp_dir_name) + in_fjm_path = Path(args.files[0]) if args.run else out_fjm_path + + return debug_path, in_fjm_path, out_fjm_path + + +def run(in_fjm_path: Path, debug_file: Path, args: argparse.Namespace, error_func: ErrorFunc) -> None: + """ + prepare and verify arguments and io_device, and run the .fjm program. + @param in_fjm_path: the input .fjm-file path + @param debug_file: the debug-file path + @param args: the parsed arguments + @param error_func: the parser's error function + """ + verify_fjm_file(error_func, in_fjm_path) + if debug_file: + verify_file_exists(error_func, debug_file) breakpoint_set = set(args.breakpoint) - breakpoint_any_set = set(args.any_breakpoint) + breakpoint_contains_set = set(args.breakpoint_contains) + + io_device = StandardIO(not args.no_output) try: - run_time, ops_executed, flips_executed, output, termination_cause = \ - debug_and_run(args.outfile, debugging_file=args.debug, - defined_input=None, - verbose=verbose_set, - breakpoint_labels=breakpoint_set, - breakpoint_any_labels=breakpoint_any_set) + breakpoint_handler = get_breakpoint_handler(debug_file, set(), breakpoint_set, breakpoint_contains_set) + termination_statistics = fjm_run.run( + in_fjm_path, + io_device=io_device, + show_trace=args.trace, + time_verbose=not args.silent, + breakpoint_handler=breakpoint_handler + ) if not args.silent: - print(f'finished by {termination_cause} after {run_time:.3f}s ({ops_executed:,} ops executed, {flips_executed / ops_executed * 100:.2f}% flips)') - print() + print(termination_statistics) except FJReadFjmException as e: print() print(e) exit(1) - if temp_debug_file: - os.close(temp_debug_fd) + +def get_version(args: argparse.Namespace) -> int: + """ + @param args: the parsed arguments + @return: the chosen version, or default if not specified + """ + if args.version is not None: + return args.version + + if args.outfile is not None: + return fjm.CompressedVersion + return fjm.NormalVersion + + +def assemble(out_fjm_file: Path, debug_file: Path, args: argparse.Namespace, error_func: ErrorFunc) -> None: + """ + prepare and verify arguments, and assemble the .fj files. + @param out_fjm_file: the to-be-compiled .fjm-file path + @param debug_file: the debug-file path + @param args: the parsed arguments + @param error_func: the parser's error function + """ + file_tuples = get_file_tuples(args) + verify_fj_files(error_func, file_tuples) + + fjm_writer = fjm.Writer(out_fjm_file, args.width, get_version(args), flags=args.flags, lzma_preset=args.lzma_preset) + assembler.assemble(file_tuples, args.width, fjm_writer, + warning_as_errors=args.werror, debugging_file_path=debug_file, + show_statistics=args.stats, print_time=not args.silent) + + +def get_fjm_file_path(args: argparse.Namespace, error_func: ErrorFunc, temp_dir_name: str) -> Path: + """ + get the output-fjm path from args. If unspecified, create a temporary file under temp_dir_name. + @param args: the parsed arguments + @param error_func: the parser's error function + @param temp_dir_name: a temporary directory that files can safely be created in + @return: the output-fjm path + """ + out_fjm_file = args.outfile + + if out_fjm_file is None: + if args.asm: + error_func(f'assemble-only is used, but no outfile is specified.') + out_fjm_file = os.path.join(temp_dir_name, 'out.fjm') + elif not args.run and not out_fjm_file.endswith('.fjm'): + error_func(f'output file {out_fjm_file} is not a .fjm file.') + + return Path(out_fjm_file) + + +def get_debug_file_path(args: argparse.Namespace, error_func: ErrorFunc, temp_dir_name: str) -> Path: + """ + get the debug-file path from args. If unspecified, create a temporary file under temp_dir_name. + @param args: the parsed arguments + @param error_func: the parser's error function + @param temp_dir_name: a temporary directory that files can safely be created in + @return: the debug-file path + """ + debug_file = args.debug + debug_file_needed = not args.asm and any((args.breakpoint, args.breakpoint_contains)) + + if debug_file is None and debug_file_needed: + if not args.silent: + parser_warning = 'Parser Warning - breakpoints are used but the debugging flag (-d) is not specified.' + if args.werror: + error_func(parser_warning) + print(f"{parser_warning} Debugging data will be saved.") + debug_file = True + + if debug_file is True: + if args.asm: + error_func('assemble-only is used with the debug flag, but no debug file is specified.') + if args.run: + error_func('run-only is used with the debug flag, but no debug file is specified.') + debug_file = os.path.join(temp_dir_name, 'debug.fjd') + + if isinstance(debug_file, str): + debug_file = Path(debug_file) + + return debug_file + + +def add_run_only_arguments(parser: argparse.ArgumentParser) -> None: + """ + add the arguments that are usable in run time. + @param parser: the parser + """ + run_arguments = parser.add_argument_group('run arguments', 'Ignored when using the --assemble option') + + run_arguments.add_argument('-t', '--trace', help="output every running opcode", action='store_true') + run_arguments.add_argument('--no_output', help="don't print the program's output", action='store_true') + + run_arguments.add_argument('-b', '--breakpoint', metavar='NAME', default=[], nargs="+", + help="pause when reaching this label") + run_arguments.add_argument('-B', '--breakpoint_contains', metavar='NAME', default=[], nargs="+", + help="pause when reaching any label containing this") + + +def add_assemble_only_arguments(parser: argparse.ArgumentParser) -> None: + """ + add the arguments that are usable in assemble time. + @param parser: the parser + """ + asm_arguments = parser.add_argument_group('assemble arguments', 'Ignored when using the --run option') + + asm_arguments.add_argument('-o', '--outfile', metavar='PATH', help="output assembled file") + + asm_arguments.add_argument('-w', '--width', type=int, default=64, choices=[8, 16, 32, 64], metavar='WIDTH', + help="specify memory-width. 64 by default") + + supported_versions = ', '.join(f"{version}: {name}" for version, name in fjm.SUPPORTED_VERSIONS.items()) + asm_arguments.add_argument('-v', '--version', metavar='VERSION', type=int, default=None, + help=f"fjm version (default of {fjm.CompressedVersion}-compressed " + f"if --outfile specified; version {fjm.NormalVersion} otherwise). " + f"supported versions: {supported_versions}.") # as in get_version() + asm_arguments.add_argument('-f', '--flags', help="the default .fjm unpacking & running flags", type=int, default=0) + + asm_arguments.add_argument('--lzma_preset', type=int, default=lzma.PRESET_DEFAULT, choices=list(range(10)), + help=f"The preset used for the LZMA2 algorithm compression (" + f"{lzma.PRESET_DEFAULT} by default; " + f"used when version={fjm.CompressedVersion}).") + + asm_arguments.add_argument('--werror', help="treat all assemble warnings as errors", action='store_true') + asm_arguments.add_argument('--no_stl', help="don't assemble/link the standard library files", action='store_true') + asm_arguments.add_argument('--stats', help="show macro code-size statistics", action='store_true') + + +def add_universal_arguments(parser: argparse.ArgumentParser) -> None: + """ + add the arguments that are usable in both --asm and --run options. + @param parser: the parser + """ + parser.add_argument('files', help="the .fj files to assemble (if run-only, the .fjm file to run)", nargs='+') + parser.add_argument('-s', '--silent', action='store_true', + help="don't show assemble & run times, and run statistics") + parser.add_argument('-d', '--debug', nargs='?', const=True, metavar='PATH', + help="debug-file path (used for breakpoints). If you both assemble & run, " + "you may use this option without specifying a path, and a temporary file will be used") + + +def add_command_arguments(parser: argparse.ArgumentParser) -> None: + """ + add the mutually exclusive --asm and --run options. + @param parser: the parser + """ + action = parser.add_mutually_exclusive_group() + action.add_argument('-a', '--asm', action='store_true', help="assemble only. Ignores any run-arguments") + action.add_argument('-r', '--run', action='store_true', help="run only. Ignores any assemble-arguments") + + +def add_arguments(parser: argparse.ArgumentParser) -> None: + """ + add the parser's arguments. + @param parser: the parser + """ + add_command_arguments(parser) + add_universal_arguments(parser) + add_assemble_only_arguments(parser) + add_run_only_arguments(parser) + + +def get_argument_parser() -> argparse.ArgumentParser: + """ + create the argument parser (with specific description and usage). + @return: the argument parser + """ + return argparse.ArgumentParser( + description='Assemble and Run FlipJump programs.', + usage=f'fj.py [--asm | --run] [arguments] files [files ...]\n' + f'example usage:\n' + f' fj.py a.fj b.fj // assemble and run\n' + f' fj.py a.fj b.fj -o out.fjm // assemble save and run\n' + f' fj.py code.fj -d -B swap_start exit_label // assemble and debug\n\n' + f' fj.py --asm -o o.fjm a.fj -d dir/debug.fjd // assemble and save debug info\n' + f' fj.py --asm -o out.fjm a.fj b.fj --no_stl -w 32 ' + f'// assemble without the standard library, 32 bit memory\n\n' + f' fj.py --run prog.fjm // just run\n' + f' fj.py --run o.fjm -d dir/debug.fjd -B label // run and debug\n ' + ) + + +def parse_arguments() -> Tuple[argparse.Namespace, ErrorFunc]: + """ + parse the command line arguments. + @return: the parsed arguments, and the parser's error function + """ + parser = get_argument_parser() + add_arguments(parser) + args = parser.parse_args() + + return args, parser.error + + +def execute_assemble_run(args: argparse.Namespace, error_func: ErrorFunc) -> None: + """ + prepare temp files, and execute the run and assemble functions. + @param args: the parsed arguments + @param error_func: parser's error function + """ + with TemporaryDirectory(suffix=get_temp_directory_suffix(args)) as temp_dir_name: + debug_path, in_fjm_path, out_fjm_path = get_files_paths(args, error_func, temp_dir_name) + + if not args.run: + assemble(out_fjm_path, debug_path, args, error_func) + + if not args.asm: + run(in_fjm_path, debug_path, args, error_func) + + +def main() -> None: + args, error_func = parse_arguments() + execute_assemble_run(args, error_func) if __name__ == '__main__': diff --git a/src/fj_parser.py b/src/fj_parser.py index 9c5411d..8d48155 100644 --- a/src/fj_parser.py +++ b/src/fj_parser.py @@ -1,28 +1,38 @@ from os import path +from pathlib import Path +from typing import Set, List, Tuple, Dict, Union -from sly import Lexer, Parser +import sly +from sly.yacc import YaccProduction as ParsedRule +from sly.lex import Token -from defs import get_char_value_and_length, get_all_used_labels, \ - main_macro, next_address, \ - OpType, Op, Expr, FJParsingException, \ - number_re, dot_id_re, id_re, string_re +from defs import Macro +from exceptions import FJExprException, FJParsingException +from expr import Expr, get_minimized_expr +from ops import get_used_labels, get_declared_labels, \ + CodePosition, MacroName, Op, initial_macro_name, \ + MacroCall, RepCall, FlipJump, WordFlip, Label, Segment, Reserve, Pad +global curr_file, curr_file_short_name, curr_text, error_occurred, curr_namespace -global curr_file, curr_text, error_occurred, curr_namespace, reserved_names +def get_position(lineno: int) -> CodePosition: + return CodePosition(curr_file, curr_file_short_name, lineno) -def syntax_error(line, msg=''): + +def syntax_error(lineno: int, msg='') -> None: global error_occurred error_occurred = True + curr_position = get_position(lineno) print() if msg: - print(f"Syntax Error in file {curr_file} line {line}:") + print(f"Syntax Error in {curr_position}:") print(f" {msg}") else: - print(f"Syntax Error in file {curr_file} line {line}") + print(f"Syntax Error in {curr_position}") -def syntax_warning(line, is_error, msg=''): +def syntax_warning(line: int, is_error: bool, msg: str = '') -> None: if is_error: global error_occurred error_occurred = True @@ -37,9 +47,35 @@ def syntax_warning(line, is_error, msg=''): print() -class FJLexer(Lexer): +# Regex for the parser + +id_re = r'[a-zA-Z_][a-zA-Z_0-9]*' +dot_id_re = fr'(({id_re})|\.*)?(\.({id_re}))+' + +bin_num = r'0[bB][01]+' +hex_num = r'0[xX][0-9a-fA-F]+' +dec_num = r'[0-9]+' + +char_escape_dict = {'0': 0x0, 'a': 0x7, 'b': 0x8, 'e': 0x1b, 'f': 0xc, 'n': 0xa, 'r': 0xd, 't': 0x9, 'v': 0xb, + '\\': 0x5c, "'": 0x27, '"': 0x22, '?': 0x3f} +escape_chars = ''.join(k for k in char_escape_dict) +char = fr'[ -~]|\\[{escape_chars}]|\\[xX][0-9a-fA-F]{{2}}' + +number_re = fr"({bin_num})|({hex_num})|('({char})')|({dec_num})" +string_re = fr'"({char})*"' + + +def get_char_value_and_length(s: str) -> Tuple[int, int]: + if s[0] != '\\': + return ord(s[0]), 1 + if s[1] in char_escape_dict: + return char_escape_dict[s[1]], 2 + return int(s[2:4], 16), 4 + + +class FJLexer(sly.Lexer): tokens = {NS, DEF, REP, - WFLIP, SEGMENT, RESERVE, + WFLIP, PAD, SEGMENT, RESERVE, ID, DOT_ID, NUMBER, STRING, LE, GE, EQ, NEQ, SHL, SHR, @@ -69,13 +105,11 @@ class FJLexer(Lexer): ID[r'ns'] = NS ID[r'wflip'] = WFLIP + ID[r'pad'] = PAD ID[r'segment'] = SEGMENT ID[r'reserve'] = RESERVE - global reserved_names - reserved_names = {DEF, REP, NS, WFLIP, SEGMENT, RESERVE} - LE = "<=" GE = ">=" @@ -91,7 +125,7 @@ class FJLexer(Lexer): ignore = ' \t' - def NUMBER(self, t): + def NUMBER(self, t: Token) -> Token: n = t.value if len(n) >= 2: if n[0] == "'": @@ -106,7 +140,7 @@ def NUMBER(self, t): t.value = int(t.value) return t - def STRING(self, t): + def STRING(self, t: Token) -> Token: chars = [] s = t.value[1:-1] i = 0 @@ -117,19 +151,23 @@ def STRING(self, t): t.value = sum(val << (i*8) for i, val in enumerate(chars)) return t - def NL(self, t): + def NL(self, t: Token) -> Token: self.lineno += 1 return t - def error(self, t): + def error(self, t: Token) -> None: global error_occurred error_occurred = True print() - print(f"Lexing Error in file {curr_file} line {self.lineno}: {t.value[0]}") + print(f"Lexing Error in {get_position(self.lineno)}: {t.value[0]}") self.index += 1 -class FJParser(Parser): +def next_address() -> Expr: + return Expr('$') + + +class FJParser(sly.Parser): tokens = FJLexer.tokens # TODO add Unary Minus (-), Unary Not (~). Maybe add logical or (||) and logical and (&&). Maybe handle power (**). precedence = ( @@ -146,502 +184,507 @@ class FJParser(Parser): ) # debugfile = 'src/parser.out' - def __init__(self, w, warning_as_errors, verbose=False): - self.verbose = verbose - self.defs = {'w': Expr(w)} - self.warning_as_errors = warning_as_errors + def __init__(self, w: int, warning_as_errors: bool): + self.consts: Dict[str, Expr] = {'w': Expr(w)} + self.warning_as_errors: bool = warning_as_errors + self.macros: Dict[MacroName, Macro] = {initial_macro_name: Macro([], [], [], '', None)} - # [(params, quiet_params), statements, (curr_file, p.lineno, ns_name)] - self.macros = {main_macro: [([], []), [], (None, None, '')]} - - def check_macro_name(self, name, line): - global reserved_names - base_name = self.ns_to_base_name(name[0]) - if base_name in reserved_names: - syntax_error(line, f'macro name can\'t be {name[0]} ({base_name} is a reserved name)!') + def validate_free_macro_name(self, name: MacroName, lineno: int) -> None: if name in self.macros: - _, _, (other_file, other_line, _) = self.macros[name] - syntax_error(line, f'macro {name} is declared twice! ' - f'also declared in file {other_file} (line {other_line}).') + syntax_error(lineno, f'macro {name} is declared twice! ' + f'also declared in {self.macros[name].code_position}.') - def check_params(self, ids, macro_name, line): + def validate_params(self, ids: List[str], macro_name: MacroName, lineno: int) -> None: for param_id in ids: - if param_id in self.defs: - syntax_error(line, f'parameter {param_id} in macro {macro_name[0]}({macro_name[1]}) ' - f'is also defined as a constant variable (with value {self.defs[param_id]})') - for i1 in range(len(ids)): - for i2 in range(i1): - if ids[i1] == ids[i2]: - syntax_error(line, f'parameter {ids[i1]} in macro {macro_name[0]}({macro_name[1]}) ' - f'is declared twice!') - - def check_label_usage(self, labels_used, labels_declared, params, externs, global_labels, line, macro_name): - if global_labels & externs: - syntax_error(line, f"In macro {macro_name[0]}({macro_name[1]}): " - f"extern labels can't be global labels: " + ', '.join(global_labels & externs)) - if global_labels & params: - syntax_error(line, f"In macro {macro_name[0]}({macro_name[1]}): " - f"extern labels can't be regular labels: " + ', '.join(global_labels & params)) - if externs & params: - syntax_error(line, f"In macro {macro_name[0]}({macro_name[1]}): " - f"global labels can't be regular labels: " + ', '.join(externs & params)) - - # params.update([self.ns_full_name(p) for p in params]) - # externs = set([self.ns_full_name(p) for p in externs]) - # globals.update([self.ns_full_name(p) for p in globals]) - - unused_labels = params - labels_used.union(self.ns_to_base_name(label) for label in labels_declared) + if param_id in self.consts: + syntax_error(lineno, f'parameter {param_id} in macro {macro_name}) ' + f'is also defined as a constant variable (with value {self.consts[param_id]})') + seen_ids = set() + for _id in ids: + if _id in seen_ids: + syntax_error(lineno, f'parameter {_id} in macro {macro_name}) is declared twice!') + else: + seen_ids.add(_id) + + def validate_label_usage(self, labels_used: Set[str], labels_declared: Set[str], + regular_labels: Set[str], extern_labels: Set[str], global_labels: Set[str], + lineno: int, macro_name: MacroName) -> None: + self.validate_labels_groups(extern_labels, global_labels, regular_labels, lineno, macro_name) + + self.validate_no_unused_labels(regular_labels, labels_declared, labels_used, lineno, macro_name) + self.validate_no_bad_label_declarations(regular_labels, extern_labels, labels_declared, lineno, macro_name) + self.validate_no_unknown_label_uses(regular_labels, global_labels, labels_declared, labels_used, + lineno, macro_name) + + @staticmethod + def validate_labels_groups(extern_labels: Set[str], global_labels: Set[str], regular_labels: Set[str], + lineno: int, macro_name: MacroName) -> None: + if global_labels & extern_labels: + syntax_error(lineno, f"In macro {macro_name}: " + f"extern labels can't be global labels: " + ', '.join(global_labels & extern_labels)) + if global_labels & regular_labels: + syntax_error(lineno, f"In macro {macro_name}: " + f"extern labels can't be regular labels: " + ', '.join(global_labels & regular_labels)) + if extern_labels & regular_labels: + syntax_error(lineno, f"In macro {macro_name}: " + f"global labels can't be regular labels: " + ', '.join(extern_labels & regular_labels)) + + def validate_no_unused_labels(self, regular_labels: Set[str], + labels_declared: Set[str], labels_used: Set[str], + lineno: int, macro_name: MacroName) -> None: + unused_labels = regular_labels - labels_used.union(self.to_base_name(label) for label in labels_declared) if unused_labels: - syntax_warning(line, self.warning_as_errors, - f"In macro {macro_name[0]}({macro_name[1]}): " + syntax_warning(lineno, self.warning_as_errors, + f"In macro {macro_name}: " f"unused labels: {', '.join(unused_labels)}.") - bad_declarations = labels_declared - set(self.ns_full_name(label) for label in externs.union(params)) + def validate_no_bad_label_declarations(self, regular_labels: Set[str], extern_labels: Set[str], + labels_declared: Set[str], + lineno: int, macro_name: MacroName) -> None: + bad_declarations = labels_declared - set( + self.ns_full_name(label) for label in extern_labels.union(regular_labels)) if bad_declarations: - syntax_warning(line, self.warning_as_errors, - f"In macro {macro_name[0]}({macro_name[1]}): " + syntax_warning(lineno, self.warning_as_errors, + f"In macro {macro_name}: " f"Declared a not extern/parameter label: {', '.join(bad_declarations)}.") - bad_uses = labels_used - global_labels - params - set(labels_declared) - {'$'} + def validate_no_unknown_label_uses(self, regular_labels: Set[str], global_labels: Set[str], + labels_declared: Set[str], labels_used: Set[str], + lineno: int, macro_name: MacroName) -> None: + bad_uses = labels_used - global_labels - regular_labels - set(labels_declared) - {'$'} if bad_uses: - # print('\nused:', labels_used, 'globals:', globals, 'params:', params) - syntax_warning(line, self.warning_as_errors, - f"In macro {macro_name[0]}({macro_name[1]}): " + syntax_warning(lineno, self.warning_as_errors, + f"In macro {macro_name}: " f"Used a not global/parameter/declared-extern label: {', '.join(bad_uses)}.") @staticmethod - def ns_name(): + def validate_no_segment_or_reserve(ops: List[Op], macro_name: MacroName) -> None: + for op in ops: + if isinstance(op, Segment): + syntax_error(op.code_position.line, f"segment can't be declared inside a macro ({macro_name}).") + if isinstance(op, Reserve): + syntax_error(op.code_position.line, f"reserve can't be declared inside a macro ({macro_name}).") + + def validate_macro_declaration(self, name: MacroName, ops: List[Op], lineno: int, + params: List[str], local_params: List[str], + global_params: Set[str], extern_params: Set[str]) -> None: + self.validate_free_macro_name(name, lineno) + + regular_params = params + local_params + self.validate_params(regular_params, name, lineno) + self.validate_label_usage(get_used_labels(ops), get_declared_labels(ops), + set(regular_params), set(extern_params), set(global_params), + lineno, name) + + self.validate_no_segment_or_reserve(ops, name) + + @staticmethod + def ns_name() -> str: return '.'.join(curr_namespace) @staticmethod - def ns_full_name(base_name): + def ns_full_name(base_name: str) -> str: return '.'.join(curr_namespace + [base_name]) @staticmethod - def dot_id_to_ns_full_name(p): - base_name = p.DOT_ID + def base_name_to_ns_full_name(base_name: str, lineno: int) -> str: without_dots = base_name.lstrip('.') if len(without_dots) == len(base_name): return base_name + num_of_dots = len(base_name) - len(without_dots) if num_of_dots - 1 > len(curr_namespace): - syntax_error(p.lineno, f'Used more leading dots than current namespace depth ' + syntax_error(lineno, f'Used more leading dots than current namespace depth ' f'({num_of_dots}-1 > {len(curr_namespace)})') + return '.'.join(curr_namespace[:len(curr_namespace)-(num_of_dots-1)] + [without_dots]) @staticmethod - def ns_to_base_name(name): + def to_base_name(name: str) -> str: return name.split('.')[-1] - def error(self, token): + def error(self, token: Token) -> None: global error_occurred error_occurred = True print() - print(f'Syntax Error in file {curr_file} line {token.lineno}, token=("{token.type}", {token.value})') + print(f'Syntax Error in {get_position(token.lineno)}, token=("{token.type}", {token.value})') @_('definable_line_statements') - def program(self, p): + def program(self, p: ParsedRule) -> None: ops = p.definable_line_statements - self.macros[main_macro][1].extend(ops) - - # labels_used, labels_declared = all_used_labels(ops) - # bad_uses = labels_used - set(labels_declared) - {'$'} - # if bad_uses: - # syntax_warning(None, self.warning_as_errors, - # f"Outside of macros: " - # f"Used a not declared label: {', '.join(bad_uses)}.") + self.macros[initial_macro_name].ops += ops @_('definable_line_statements NL definable_line_statement') - def definable_line_statements(self, p): + def definable_line_statements(self, p: ParsedRule) -> List[Op]: if p.definable_line_statement: - return p.definable_line_statements + p.definable_line_statement + p.definable_line_statements.extend(p.definable_line_statement) return p.definable_line_statements @_('definable_line_statement') - def definable_line_statements(self, p): + def definable_line_statements(self, p: ParsedRule) -> List[Op]: if p.definable_line_statement: return p.definable_line_statement return [] @_('') - def empty(self, p): + def empty(self, p: ParsedRule) -> None: return None @_('line_statement') - def definable_line_statement(self, p): + def definable_line_statement(self, p: ParsedRule) -> List[Op]: return p.line_statement @_('macro_def') - def definable_line_statement(self, p): + def definable_line_statement(self, p: ParsedRule) -> List[Op]: return [] @_('NS ID') - def namespace(self, p): + def namespace(self, p: ParsedRule) -> None: curr_namespace.append(p.ID) @_('namespace "{" NL definable_line_statements NL "}"') - def definable_line_statement(self, p): + def definable_line_statement(self, p: ParsedRule) -> List[Op]: curr_namespace.pop() return p.definable_line_statements @_('DEF ID macro_params "{" NL line_statements NL "}"') - def macro_def(self, p): + def macro_def(self, p: ParsedRule) -> None: params, local_params, global_params, extern_params = p.macro_params - name = (self.ns_full_name(p.ID), len(params)) - self.check_macro_name(name, p.lineno) - self.check_params(params + local_params, name, p.lineno) + name = MacroName(self.ns_full_name(p.ID), len(params)) ops = p.line_statements - self.check_label_usage(*get_all_used_labels(ops), set(params + local_params), set(extern_params), - set(global_params), p.lineno, name) - self.macros[name] = [(params, local_params), ops, (curr_file, p.lineno, self.ns_name())] + + self.validate_macro_declaration(name, ops, p.lineno, params, local_params, global_params, extern_params) + + self.macros[name] = Macro(params, local_params, ops, self.ns_name(), get_position(p.lineno)) return None @_('empty') - def maybe_ids(self, p): + def maybe_ids(self, p: ParsedRule) -> List[str]: return [] @_('IDs') - def maybe_ids(self, p): + def maybe_ids(self, p: ParsedRule) -> List[str]: return p.IDs @_('empty') - def maybe_local_ids(self, p): + def maybe_local_ids(self, p: ParsedRule) -> List[str]: return [] @_('"@" IDs') - def maybe_local_ids(self, p): + def maybe_local_ids(self, p: ParsedRule) -> List[str]: return p.IDs @_('empty') - def maybe_extern_ids(self, p): - return [] - - @_('empty') - def maybe_global_ids(self, p): + def maybe_global_ids(self, p: ParsedRule) -> List[str]: return [] @_('"<" ids') - def maybe_global_ids(self, p): + def maybe_global_ids(self, p: ParsedRule) -> List[str]: return p.ids + @_('empty') + def maybe_extern_ids(self, p: ParsedRule) -> List[str]: + return [] + @_('">" IDs') - def maybe_extern_ids(self, p): + def maybe_extern_ids(self, p: ParsedRule) -> List[str]: return p.IDs @_('maybe_ids maybe_local_ids maybe_global_ids maybe_extern_ids') - def macro_params(self, p): + def macro_params(self, p: ParsedRule) -> Tuple[List[str], List[str], List[str], List[str]]: return p.maybe_ids, p.maybe_local_ids, p.maybe_global_ids, p.maybe_extern_ids @_('IDs "," ID') - def IDs(self, p): + def IDs(self, p: ParsedRule) -> List[str]: return p.IDs + [p.ID] @_('ID') - def IDs(self, p): + def IDs(self, p: ParsedRule) -> List[str]: return [p.ID] @_('line_statements NL line_statement') - def line_statements(self, p): - return p.line_statements + p.line_statement + def line_statements(self, p: ParsedRule) -> List[Op]: + p.line_statements.extend(p.line_statement) + return p.line_statements @_('line_statement') - def line_statements(self, p): + def line_statements(self, p: ParsedRule) -> List[Op]: return p.line_statement # @_('empty') - # def line_statements(self, p): + # def line_statements(self, p: ParsedRule) -> List[Op]: # return [] @_('empty') - def line_statement(self, p): + def line_statement(self, p: ParsedRule) -> List[Op]: return [] @_('statement') - def line_statement(self, p): + def line_statement(self, p: ParsedRule) -> List[Op]: if p.statement: return [p.statement] return [] @_('label statement') - def line_statement(self, p): + def line_statement(self, p: ParsedRule) -> List[Op]: if p.statement: return [p.label, p.statement] return [p.label] @_('label') - def line_statement(self, p): + def line_statement(self, p: ParsedRule) -> List[Op]: return [p.label] @_('ID ":"') - def label(self, p): - return Op(OpType.Label, (self.ns_full_name(p.ID),), curr_file, p.lineno) + def label(self, p: ParsedRule) -> Label: + return Label(self.ns_full_name(p.ID), get_position(p.lineno)) @_('expr SC') - def statement(self, p): - return Op(OpType.FlipJump, (p.expr, next_address()), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> FlipJump: + return FlipJump(p.expr, next_address(), get_position(p.lineno)) @_('expr SC expr') - def statement(self, p): - return Op(OpType.FlipJump, (p.expr0, p.expr1), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> FlipJump: + return FlipJump(p.expr0, p.expr1, get_position(p.lineno)) @_('SC expr') - def statement(self, p): - return Op(OpType.FlipJump, (Expr(0), p.expr), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> FlipJump: + return FlipJump(Expr(0), p.expr, get_position(p.lineno)) @_('SC') - def statement(self, p): - return Op(OpType.FlipJump, (Expr(0), next_address()), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> FlipJump: + return FlipJump(Expr(0), next_address(), get_position(p.lineno)) @_('ID') - def id(self, p): + def id(self, p: ParsedRule) -> Tuple[str, int]: return p.ID, p.lineno @_('DOT_ID') - def id(self, p): - return self.dot_id_to_ns_full_name(p), p.lineno + def id(self, p: ParsedRule) -> Tuple[str, int]: + return self.base_name_to_ns_full_name(p.DOT_ID, p.lineno), p.lineno @_('ids "," id') - def ids(self, p): + def ids(self, p: ParsedRule) -> List[str]: return p.ids + [p.id[0]] @_('id') - def ids(self, p): + def ids(self, p: ParsedRule) -> List[str]: return [p.id[0]] @_('id') - def statement(self, p): + def statement(self, p: ParsedRule) -> MacroCall: macro_name, lineno = p.id - return Op(OpType.Macro, ((macro_name, 0),), curr_file, lineno) + return MacroCall(macro_name, [], get_position(lineno)) @_('id expressions') - def statement(self, p): + def statement(self, p: ParsedRule) -> MacroCall: macro_name, lineno = p.id - return Op(OpType.Macro, ((macro_name, len(p.expressions)), *p.expressions), curr_file, lineno) + return MacroCall(macro_name, p.expressions, get_position(lineno)) @_('WFLIP expr "," expr') - def statement(self, p): - return Op(OpType.WordFlip, (p.expr0, p.expr1, next_address()), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> WordFlip: + return WordFlip(p.expr0, p.expr1, next_address(), get_position(p.lineno)) @_('WFLIP expr "," expr "," expr') - def statement(self, p): - return Op(OpType.WordFlip, (p.expr0, p.expr1, p.expr2), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> WordFlip: + return WordFlip(p.expr0, p.expr1, p.expr2, get_position(p.lineno)) + + @_('PAD expr') + def statement(self, p: ParsedRule) -> Pad: + return Pad(p.expr, get_position(p.lineno)) @_('ID "=" expr') - def statement(self, p): + def statement(self, p: ParsedRule) -> None: name = self.ns_full_name(p.ID) - if name in self.defs: + if name in self.consts: syntax_error(p.lineno, f'Can\'t redeclare the variable "{name}".') - if not p.expr.eval(self.defs, curr_file, p.lineno): - self.defs[name] = p.expr - return None - syntax_error(p.lineno, f'Can\'t evaluate expression: {str(p.expr)}.') + + evaluated = p.expr.eval_new(self.consts) + try: + self.consts[name] = Expr(int(evaluated)) + except FJExprException: + syntax_error(p.lineno, f'Can\'t evaluate expression: {str(evaluated)}.') @_('REP "(" expr "," ID ")" id') - def statement(self, p): + def statement(self, p: ParsedRule) -> RepCall: macro_name, lineno = p.id - return Op(OpType.Rep, - (p.expr, p.ID, Op(OpType.Macro, ((macro_name, 0),), curr_file, lineno)), - curr_file, p.lineno) + code_position = get_position(lineno) + return RepCall(p.expr, p.ID, macro_name, [], code_position) @_('REP "(" expr "," ID ")" id expressions') - def statement(self, p): - exps = p.expressions + def statement(self, p: ParsedRule) -> RepCall: macro_name, lineno = p.id - return Op(OpType.Rep, - (p.expr, p.ID, Op(OpType.Macro, ((macro_name, len(exps)), *exps), curr_file, lineno)), - curr_file, p.lineno) + code_position = get_position(lineno) + return RepCall(p.expr, p.ID, macro_name, p.expressions, code_position) @_('SEGMENT expr') - def statement(self, p): - return Op(OpType.Segment, (p.expr,), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> Segment: + return Segment(p.expr, get_position(p.lineno)) @_('RESERVE expr') - def statement(self, p): - return Op(OpType.Reserve, (p.expr,), curr_file, p.lineno) + def statement(self, p: ParsedRule) -> Reserve: + return Reserve(p.expr, get_position(p.lineno)) @_('expressions "," expr') - def expressions(self, p): + def expressions(self, p: ParsedRule) -> List[Expr]: return p.expressions + [p.expr] @_('expr') - def expressions(self, p): + def expressions(self, p: ParsedRule) -> List[Expr]: return [p.expr] - @_('_expr') - def expr(self, p): - return p._expr[0] - - @_('_expr "+" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a + b), p.lineno - return Expr(('+', (a, b))), p.lineno - - @_('_expr "-" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a - b), p.lineno - return Expr(('-', (a, b))), p.lineno - - @_('_expr "*" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a * b), p.lineno - return Expr(('*', (a, b))), p.lineno - - @_('"#" _expr') - def _expr(self, p): - a = p._expr[0] - if a is int: - return Expr(a.bit_length()), p.lineno - return Expr(('#', (a,))), p.lineno - - @_('_expr "/" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a // b), p.lineno - return Expr(('/', (a, b))), p.lineno - - @_('_expr "%" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a % b), p.lineno - return Expr(('%', (a, b))), p.lineno - - @_('_expr SHL _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a << b), p.lineno - return Expr(('<<', (a, b))), p.lineno - - @_('_expr SHR _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a >> b), p.lineno - return Expr(('>>', (a, b))), p.lineno - - @_('_expr "^" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a ^ b), p.lineno - return Expr(('^', (a, b))), p.lineno - - @_('_expr "|" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a | b), p.lineno - return Expr(('|', (a, b))), p.lineno - - @_('_expr "&" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(a & b), p.lineno - return Expr(('&', (a, b))), p.lineno - - @_('_expr "?" _expr ":" _expr') - def _expr(self, p): - a, b, c = p._expr0[0], p._expr1[0], p._expr2[0] - if a is int and b is int and c is int: - return Expr(b if a else c), p.lineno - return Expr(('?:', (a, b, c))), p.lineno - - @_('_expr "<" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(1 if a < b else 0), p.lineno - return Expr(('<', (a, b))), p.lineno - - @_('_expr ">" _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(1 if a > b else 0), p.lineno - return Expr(('>', (a, b))), p.lineno - - @_('_expr LE _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(1 if a <= b else 0), p.lineno - return Expr(('<=', (a, b))), p.lineno - - @_('_expr GE _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(1 if a >= b else 0), p.lineno - return Expr(('>=', (a, b))), p.lineno - - @_('_expr EQ _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(1 if a == b else 0), p.lineno - return Expr(('==', (a, b))), p.lineno - - @_('_expr NEQ _expr') - def _expr(self, p): - a, b = p._expr0[0], p._expr1[0] - if a is int and b is int: - return Expr(1 if a != b else 0), p.lineno - return Expr(('!=', (a, b))), p.lineno - - @_('"(" _expr ")"') - def _expr(self, p): - return p._expr + @_('expr_') + def expr(self, p: ParsedRule) -> Expr: + return p.expr_[0] + + @_('expr_ "+" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('+', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ "-" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('-', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ "*" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('*', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('"#" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('#', (p.expr_[0],)), p.lineno + + @_('expr_ "/" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('/', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ "%" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('%', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ SHL expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('<<', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ SHR expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('>>', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ "^" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('^', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ "|" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('|', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ "&" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('&', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ "?" expr_ ":" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('?:', (p.expr_0[0], p.expr_1[0], p.expr_2[0])), p.lineno + + @_('expr_ "<" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('<', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ ">" expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('>', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ LE expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('<=', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ GE expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('>=', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ EQ expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('==', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('expr_ NEQ expr_') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return get_minimized_expr('!=', (p.expr_0[0], p.expr_1[0])), p.lineno + + @_('"(" expr_ ")"') + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: + return p.expr_ @_('NUMBER') - def _expr(self, p): + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: return Expr(p.NUMBER), p.lineno @_('STRING') - def _expr(self, p): + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: return Expr(p.STRING), p.lineno @_('"$"') - def _expr(self, p): + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: return next_address(), p.lineno @_('id') - def _expr(self, p): + def expr_(self, p: ParsedRule) -> Tuple[Expr, int]: id_str, lineno = p.id - if id_str in self.defs: - return self.defs[id_str], lineno + if id_str in self.consts: + return self.consts[id_str], lineno return Expr(id_str), lineno -def exit_if_errors(): +def exit_if_errors() -> None: if error_occurred: raise FJParsingException(f'Errors found in file {curr_file}. Assembly stopped.') -def parse_macro_tree(input_files, w, warning_as_errors, verbose=False): - global curr_file, curr_text, error_occurred, curr_namespace +def validate_current_file(files_seen: Set[Union[str, Path]]) -> None: + if not path.isfile(curr_file): + raise FJParsingException(f"No such file {curr_file}.") + + if curr_file_short_name in files_seen: + raise FJParsingException(f"Short file name is repeated: '{curr_file_short_name}'.") + + abs_path = curr_file.absolute() + if abs_path in files_seen: + raise FJParsingException(f".fj file path is repeated: '{abs_path}'.") + + files_seen.add(curr_file_short_name) + files_seen.add(abs_path) + + +def lex_parse_curr_file(lexer: FJLexer, parser: FJParser) -> None: + global curr_text, curr_namespace + curr_text = curr_file.open('r').read() + curr_namespace = [] + + lex_res = lexer.tokenize(curr_text) + exit_if_errors() + + parser.parse(lex_res) + exit_if_errors() + + +def parse_macro_tree(input_files: List[Tuple[str, Path]], w: int, warning_as_errors: bool) \ + -> Dict[MacroName, Macro]: + """ + parse the .fj files and create a macro-dictionary. + The files will be parsed as if they were concatenated. + @param input_files:[in]: a list of (short_file_name, fj_file_path). The files will to be parsed in that given order. + @param w:[in]: the memory-width + @param warning_as_errors:[in]: treat warnings as errors (stop execution on warnings) + @return: the macro-dictionary. + """ + global curr_file, curr_file_short_name, error_occurred error_occurred = False + files_seen: Set[Union[str, Path]] = set() + lexer = FJLexer() - parser = FJParser(w, warning_as_errors, verbose=verbose) - for curr_file in input_files: - if not path.isfile(curr_file): - raise FJParsingException(f"No such file {curr_file}.") - curr_text = open(curr_file, 'r').read() - curr_namespace = [] - - lex_res = lexer.tokenize(curr_text) - exit_if_errors() - - parser.parse(lex_res) - exit_if_errors() + parser = FJParser(w, warning_as_errors) + + for curr_file_short_name, curr_file in input_files: + validate_current_file(files_seen) + lex_parse_curr_file(lexer, parser) return parser.macros diff --git a/src/fja.py b/src/fja.py deleted file mode 100644 index a66bc0e..0000000 --- a/src/fja.py +++ /dev/null @@ -1,49 +0,0 @@ -import argparse -from os.path import isfile, abspath - -from assembler import assemble -from defs import Verbose, FJException, get_stl_paths - - -def main(): - parser = argparse.ArgumentParser(description='Assemble FlipJump programs.') - parser.add_argument('file', help="the FlipJump files.", nargs='+') - parser.add_argument('-s', '--silent', help="don't show assemble times", action='store_true') - parser.add_argument('-o', '--outfile', help="output assembled file.", default="a.fjm") - parser.add_argument('--no-macros', help="output no-macros file.") - parser.add_argument('-d', '--debug', help="output debug file (used for breakpoints).") - parser.add_argument('-v', '--version', help="fjm version", type=int, default=0) - parser.add_argument('-f', '--flags', help="default running flags", type=int, default=0) - parser.add_argument('-w', '--width', help="specify memory-width. 64 by default.", - type=int, default=64, choices=[8, 16, 32, 64]) - parser.add_argument('--Werror', help="make all warnings into errors.", action='store_true') - parser.add_argument('--no-stl', help="don't assemble/link the standard library files.", action='store_true') - parser.add_argument('--stats', help="show macro usage statistics.", action='store_true') - args = parser.parse_args() - - verbose_set = set() - if not args.silent: - verbose_set.add(Verbose.Time) - - if not args.no_stl: - args.file = get_stl_paths() + args.file - for file in args.file: - file = abspath(file) - if not file.endswith('.fj'): - parser.error(f'file {file} is not a .fj file.') - if not isfile(abspath(file)): - parser.error(f'file {file} does not exist.') - try: - assemble(args.file, args.outfile, args.width, - version=args.version, flags=args.flags, - warning_as_errors=args.Werror, - show_statistics=args.stats, verbose=verbose_set, - preprocessed_file=args.no_macros, debugging_file=args.debug) - except FJException as e: - print() - print(e) - exit(1) - - -if __name__ == '__main__': - main() diff --git a/src/fji.py b/src/fji.py deleted file mode 100644 index 6b2dde7..0000000 --- a/src/fji.py +++ /dev/null @@ -1,61 +0,0 @@ -import argparse -from os.path import isfile, abspath - - -from fjm_run import debug_and_run -from defs import Verbose, FJReadFjmException - - -def main(): - parser = argparse.ArgumentParser(description='Run FlipJump programs.') - parser.add_argument('file', help="the FlipJump file.") - parser.add_argument('-s', '--silent', help="don't show run times", action='store_true') - parser.add_argument('-t', '--trace', help="trace the running opcodes.", action='store_true') - parser.add_argument('-f', '--flags', help="running flags", type=int, default=0) - parser.add_argument('-d', '--debug', help='debugging file') - parser.add_argument('-b', '--breakpoint', help="pause when reaching this label", - default=[], action='append') - parser.add_argument('-B', '--any_breakpoint', help="pause when reaching any label containing this", - default=[], action='append') - - args = parser.parse_args() - - verbose_set = {Verbose.PrintOutput} - if not args.silent: - verbose_set.add(Verbose.Time) - if args.trace: - verbose_set.add(Verbose.Run) - - file = abspath(args.file) - if not isfile(file): - parser.error(f'file {file} does not exist.') - if not file.endswith('.fjm'): - parser.error(f'file {file} is not a .fjm file.') - - if args.debug: - debug_file = abspath(args.debug) - if not isfile(debug_file): - parser.error(f'debug-file {debug_file} does not exist.') - - breakpoint_set = set(args.breakpoint) - breakpoint_any_set = set(args.any_breakpoint) - - try: - run_time, ops_executed, flips_executed, output, termination_cause = \ - debug_and_run(file, debugging_file=args.debug, - defined_input=None, - verbose=verbose_set, - breakpoint_labels=breakpoint_set, - breakpoint_any_labels=breakpoint_any_set) - - if not args.silent: - print(f'finished by {termination_cause} after {run_time:.3f}s ({ops_executed:,} ops executed, {flips_executed/ops_executed*100:.2f}% flips)') - print() - except FJReadFjmException as e: - print() - print(e) - exit(1) - - -if __name__ == '__main__': - main() diff --git a/src/fjm.py b/src/fjm.py index be32c82..ca34627 100644 --- a/src/fjm.py +++ b/src/fjm.py @@ -1,10 +1,13 @@ import struct +from enum import IntEnum +from pathlib import Path from struct import pack, unpack -from random import randint from time import sleep -from typing import BinaryIO, List, Tuple +from typing import BinaryIO, List, Tuple, Dict, Optional -from defs import FJReadFjmException, FJWriteFjmException +import lzma + +from exceptions import FJReadFjmException, FJWriteFjmException """ @@ -22,152 +25,252 @@ u64 segment_length; // in memory words (w-bits) u64 data_start; // in the outer-struct.data words (w-bits) u64 data_length; // in the outer-struct.data words (w-bits) - } *segments; // segments[segment_num] - u8* data; // the data + } *segments; // segments[segment_num] + u8* data; // the data (might be compressed in some versions) } fjm_file; // Flip-Jump Memory file """ -fj_magic = ord('F') + (ord('J') << 8) -reserved_dict_threshold = 1000 +FJ_MAGIC = ord('F') + (ord('J') << 8) + +_reserved_dict_threshold = 1000 + +_header_base_format = ' List[Dict[str, int]]: + return [{"id": lzma.FILTER_LZMA2, "preset": preset, "nice_len": dw}] -SUPPORTED_VERSIONS = {0: 'Normal', 1: 'Full'} -# TODO UPCOMING_VERSIONS = {2: 'Zipped', 3: 'RelativeZipped', 4: '7Zipped', 5: 'RelativeZipped'} + +def _new_garbage_val() -> int: + return 0 # The value read when reading a word outside any segment. + + +class GarbageHandling(IntEnum): + """ + What to do when reading garbage memory (memory outside any segment). + """ + Stop = 0 # Stop and finish + SlowRead = 1 # Continue after a small waiting time, very slow and print a warning + OnlyWarning = 2 # Continue and print a warning + Continue = 3 # Continue normally class Reader: - def __init__(self, input_file, *, slow_garbage_read=True, stop_after_garbage=True): - self.slow_garbage_read = slow_garbage_read - self.stop_after_garbage = stop_after_garbage + """ + Used for reading a .fjm file from memory. + """ + def __init__(self, input_file: Path, *, garbage_handling: GarbageHandling = GarbageHandling.Stop): + """ + The .fjm-file reader + @param input_file: the path to the .fjm file + @param garbage_handling: how to handle access to memory not in any segment + """ + self.garbage_handling = garbage_handling with open(input_file, 'rb') as fjm_file: try: self._init_header_fields(fjm_file) self._validate_header() segments = self._init_segments(fjm_file) - data = self._read_data(fjm_file) + data = self._read_decompressed_data(fjm_file) self._init_memory(segments, data) except struct.error as se: raise FJReadFjmException(f"Bad file {input_file}, can't unpack. Maybe it's not a .fjm file?") from se def _init_header_fields(self, fjm_file: BinaryIO) -> None: self.magic, self.w, self.version, self.segment_num = \ - unpack(header_base_format, fjm_file.read(header_base_size)) - if self.version == 0: + unpack(_header_base_format, fjm_file.read(_header_base_size)) + if BaseVersion == self.version: self.flags, self.reserved = 0, 0 else: - self.flags, self.reserved = unpack(header_extension_format, fjm_file.read(header_extension_size)) + self.flags, self.reserved = unpack(_header_extension_format, fjm_file.read(_header_extension_size)) def _init_segments(self, fjm_file: BinaryIO) -> List[Tuple]: - return [unpack(segment_format, fjm_file.read(segment_size)) for _ in range(self.segment_num)] + return [unpack(_segment_format, fjm_file.read(_segment_size)) for _ in range(self.segment_num)] - def _validate_header(self): - if self.magic != fj_magic: - raise FJReadFjmException(f'Error: bad magic code ({hex(self.magic)}, should be {hex(fj_magic)}).') + def _validate_header(self) -> None: + if self.magic != FJ_MAGIC: + raise FJReadFjmException(f'Error: bad magic code ({hex(self.magic)}, should be {hex(FJ_MAGIC)}).') if self.version not in SUPPORTED_VERSIONS: raise FJReadFjmException( f'Error: unsupported version ({self.version}, this program supports {str(SUPPORTED_VERSIONS)}).') if self.reserved != 0: raise FJReadFjmException(f'Error: bad reserved value ({self.reserved}, should be 0).') - def _read_data(self, fjm_file: BinaryIO) -> List[int]: + @staticmethod + def _decompress_data(compressed_data: bytes) -> bytes: + try: + return lzma.decompress(compressed_data, format=_LZMA_FORMAT, filters=_LZMA_DECOMPRESSION_FILTERS) + except lzma.LZMAError as e: + raise FJReadFjmException(f'Error: The compressed data is damaged; Unable to decompress.') from e + + def _read_decompressed_data(self, fjm_file: BinaryIO) -> List[int]: + """ + @param fjm_file: [in]: read from this file the data words. + @return: list of the data words (decompressed if it was compressed). + """ read_tag = '<' + {8: 'B', 16: 'H', 32: 'L', 64: 'Q'}[self.w] word_bytes_size = self.w // 8 - # TODO read file once: - # data = [] - # while True: - # word = fjm_file.read(word_bytes_size) - # if word == '': - # break - # data.append(unpack(read_tag, word)[0]) - file_data = fjm_file.read() + if CompressedVersion == self.version: + file_data = self._decompress_data(file_data) + data = [unpack(read_tag, file_data[i:i + word_bytes_size])[0] for i in range(0, len(file_data), word_bytes_size)] return data - def _init_memory(self, segments: List[Tuple], data: List[int]): + def _init_memory(self, segments: List[Tuple], data: List[int]) -> None: self.memory = {} self.zeros_boundaries = [] for segment_start, segment_length, data_start, data_length in segments: - for i in range(data_length): - self.memory[segment_start + i] = data[data_start + i] + if self.version in (RelativeJumpVersion, CompressedVersion): + word = ((1 << self.w) - 1) + for i in range(0, data_length, 2): + self.memory[segment_start + i] = data[data_start + i] + self.memory[segment_start + i+1] = (data[data_start + i+1] + (segment_start + i+1) * self.w) & word + else: + for i in range(data_length): + self.memory[segment_start + i] = data[data_start + i] if segment_length > data_length: - if segment_length - data_length < reserved_dict_threshold: + if segment_length - data_length < _reserved_dict_threshold: for i in range(data_length, segment_length): self.memory[segment_start + i] = 0 else: self.zeros_boundaries.append((segment_start + data_length, segment_start + segment_length)) - def __getitem__(self, address): - address &= ((1 << self.w) - 1) - if address not in self.memory: + def _get_memory_word(self, word_address: int) -> int: + word_address &= ((1 << self.w) - 1) + if word_address not in self.memory: for start, end in self.zeros_boundaries: - if start <= address < end: - self.memory[address] = 0 + if start <= word_address < end: + self.memory[word_address] = 0 return 0 - garbage_val = randint(0, (1 << self.w) - 1) - garbage_message = f'Reading garbage word at mem[{hex(address << self.w)[2:]}] = {hex(garbage_val)[2:]}' - if self.stop_after_garbage: + + garbage_val = _new_garbage_val() + garbage_message = f'Reading garbage word at mem[{hex(word_address << self.w)[2:]}] = {hex(garbage_val)[2:]}' + + if GarbageHandling.Stop == self.garbage_handling: raise FJReadFjmException(garbage_message) - print(f'\nWarning: {garbage_message}') - if self.slow_garbage_read: + elif GarbageHandling.OnlyWarning == self.garbage_handling: + print(f'\nWarning: {garbage_message}') + elif GarbageHandling.SlowRead == self.garbage_handling: + print(f'\nWarning: {garbage_message}') sleep(0.1) - self.memory[address] = garbage_val - return self.memory[address] - - def __setitem__(self, address, value): - address &= ((1 << self.w) - 1) - value &= ((1 << self.w) - 1) - self.memory[address] = value - def bit_address_decompose(self, bit_address): - address = (bit_address >> (self.w.bit_length() - 1)) & ((1 << self.w) - 1) - bit = bit_address & (self.w - 1) - return address, bit + self.memory[word_address] = garbage_val - def read_bit(self, bit_address): - address, bit = self.bit_address_decompose(bit_address) - return (self[address] >> bit) & 1 + return self.memory[word_address] - def write_bit(self, bit_address, value): - address, bit = self.bit_address_decompose(bit_address) - if value: - self[address] = self[address] | (1 << bit) + def _set_memory_word(self, word_address: int, value: int) -> None: + word_address &= ((1 << self.w) - 1) + value &= ((1 << self.w) - 1) + self.memory[word_address] = value + + def _bit_address_decompose(self, bit_address: int) -> Tuple[int, int]: + """ + @param bit_address: the address + @return: tuple of the word address and the bit offset + """ + word_address = (bit_address >> (self.w.bit_length() - 1)) & ((1 << self.w) - 1) + bit_offset = bit_address & (self.w - 1) + return word_address, bit_offset + + def read_bit(self, bit_address: int) -> bool: + """ + read a bit from memory. + @param bit_address: the address + @return: True/False for 1/0 + """ + word_address, bit_offset = self._bit_address_decompose(bit_address) + return (self._get_memory_word(word_address) >> bit_offset) & 1 == 1 + + def write_bit(self, bit_address: int, bit_value: bool) -> None: + """ + write a bit to memory. + @param bit_address: the address + @param bit_value: True/False for 1/0 + """ + word_address, bit_offset = self._bit_address_decompose(bit_address) + word_value = self._get_memory_word(word_address) + if bit_value: + word_value |= (1 << bit_offset) else: - self[address] = self[address] & ((1 << self.w) - 1 - (1 << bit)) - - def get_word(self, bit_address): - address, bit = self.bit_address_decompose(bit_address) - if bit == 0: - return self[address] - if address == ((1 << self.w) - 1): + word_value &= ((1 << self.w) - 1 - (1 << bit_offset)) + self._set_memory_word(word_address, word_value) + + def get_word(self, bit_address: int) -> int: + """ + read a word from memory (can be unaligned). + @param bit_address: the address + @return: the word value + """ + word_address, bit_offset = self._bit_address_decompose(bit_address) + if bit_offset == 0: + return self._get_memory_word(word_address) + if word_address == ((1 << self.w) - 1): raise FJReadFjmException(f'Accessed outside of memory (beyond the last bit).') - l, m = self[address], self[address+1] - return ((l >> bit) | (m << (self.w - bit))) & ((1 << self.w) - 1) + + lsw = self._get_memory_word(word_address) + msw = self._get_memory_word(word_address + 1) + return ((lsw >> bit_offset) | (msw << (self.w - bit_offset))) & ((1 << self.w) - 1) class Writer: - def __init__(self, output_file, w, *, version=0, flags=0): + """ + Used for creating a .fjm file in memory. + The process is: + 1. add_data(..) + 2. add_segment(..) + repeat steps 1-2 until you finished updating the fjm + 3. write_to_file() + """ + def __init__(self, output_file: Path, w: int, version: int, *, flags: int = 0, lzma_preset: Optional[int] = None): + """ + the .fjm-file writer + @param output_file: [in,out]: the path to the .fjm file + @param w: the memory-width + @param version: the file's version + @param flags: the file's flags + @param lzma_preset: the preset to be used when compressing the .fjm data + """ if w not in (8, 16, 32, 64): raise FJWriteFjmException(f"Word size {w} is not in {{8, 16, 32, 64}}.") - if version < 0 or version >= 1 << 64: - raise FJWriteFjmException(f"version must be a 64bit positive number, not {version}") - if flags < 0 or flags >= 1 << 64: + if version not in SUPPORTED_VERSIONS: + raise FJWriteFjmException( + f'Error: unsupported version ({version}, this program supports {str(SUPPORTED_VERSIONS)}).') + if flags < 0 or flags >= (1 << 64): raise FJWriteFjmException(f"flags must be a 64bit positive number, not {flags}") - if version == 0 and flags != 0: + if BaseVersion == version and flags != 0: raise FJWriteFjmException(f"version 0 does not support the flags option") + if CompressedVersion == version: + if lzma_preset is None or lzma_preset not in range(10): + raise FJWriteFjmException("version 3 requires an LZMA preset (0-9, faster->smaller).") + else: + self.lzma_preset = lzma_preset self.output_file = output_file self.word_size = w @@ -178,30 +281,137 @@ def __init__(self, output_file, w, *, version=0, flags=0): self.segments = [] self.data = [] # words array - def write_to_file(self): + def _compress_data(self, data: bytes) -> bytes: + try: + return lzma.compress(data, format=_LZMA_FORMAT, + filters=_lzma_compression_filters(2 * self.word_size, self.lzma_preset)) + except lzma.LZMAError as e: + raise FJWriteFjmException(f'Error: Unable to compress the data.') from e + + def write_to_file(self) -> None: + """ + writes the .fjm headers, segments and (might be compressed) data into the output_file. + @note call this after finished adding data and segments and editing the Writer. + """ write_tag = '<' + {8: 'B', 16: 'H', 32: 'L', 64: 'Q'}[self.word_size] with open(self.output_file, 'wb') as f: - f.write(pack(header_base_format, fj_magic, self.word_size, self.version, len(self.segments))) - if self.version > 0: - f.write(pack(header_extension_format, self.flags, self.reserved)) + f.write(pack(_header_base_format, FJ_MAGIC, self.word_size, self.version, len(self.segments))) + if BaseVersion != self.version: + f.write(pack(_header_extension_format, self.flags, self.reserved)) for segment in self.segments: - f.write(pack(segment_format, *segment)) + f.write(pack(_segment_format, *segment)) + + fjm_data = b''.join(pack(write_tag, word) for word in self.data) + if CompressedVersion == self.version: + fjm_data = self._compress_data(fjm_data) + + f.write(fjm_data) + + def get_segment_addresses_repr(self, word_start_address: int, word_length: int) -> str: + """ + @param word_start_address: the start address of the segment in memory (in words) + @param word_length: the number of words the segment takes in memory + @return: a nice looking segment-representation string by its addresses + """ + return f'[{hex(self.word_size * word_start_address)}, ' \ + f'{hex(self.word_size * (word_start_address + word_length))})' + + @staticmethod + def _is_collision(start1: int, end1: int, start2: int, end2: int) -> bool: + if any(start2 <= address <= end2 for address in (start1, end1)): + return True + if any(start1 <= address <= end1 for address in (start2, end2)): + return True + return False + + def _validate_segment_addresses_not_overlapping(self, new_segment_start: int, new_segment_length: int) -> None: + new_segment_end = new_segment_start + new_segment_length - 1 + for i, (segment_start, segment_length, _, _) in enumerate(self.segments): + segment_end = segment_start + segment_length - 1 + + if self._is_collision(segment_start, segment_end, new_segment_start, new_segment_end): + raise FJWriteFjmException( + f"Overlapping segments addresses: " + f"seg[{i}]={self.get_segment_addresses_repr(segment_start, segment_length)}" + f" and " + f"seg[{len(self.segments)}]={self.get_segment_addresses_repr(new_segment_start, new_segment_length)}" + ) + + def _validate_segment_data_not_overlapping(self, new_data_start: int, new_data_length: int) -> None: + if new_data_length == 0: + return + new_data_end = new_data_start + new_data_length - 1 + + for i, (_, _, data_start, data_length) in enumerate(self.segments): + if data_length == 0: + continue + data_end = data_start + data_length - 1 + + if self._is_collision(data_start, data_end, new_data_start, new_data_end): + raise FJWriteFjmException( + f"Overlapping segments data: " + f"seg[{i}]=data[{hex(data_start)}, {hex(data_end + 1)})" + f" and " + f"seg[{len(self.segments)}]=data[{hex(new_data_start)}, {hex(new_data_end + 1)})" + ) + + def _validate_segment_not_overlapping(self, segment_start: int, segment_length: int, + data_start: int, data_length: int) -> None: + self._validate_segment_addresses_not_overlapping(segment_start, segment_length) + + if self.version in (RelativeJumpVersion, CompressedVersion): + self._validate_segment_data_not_overlapping(data_start, data_length) + + def _update_to_relative_jumps(self, segment_start: int, data_start: int, data_length: int) -> None: + word = ((1 << self.word_size) - 1) + for i in range(1, data_length, 2): + self.data[data_start + i] = (self.data[data_start + i] - (segment_start + i) * self.word_size) & word + + def add_segment(self, segment_start: int, segment_length: int, data_start: int, data_length: int) -> None: + """ + inserts a new segment to the fjm file. checks that it doesn't overlap with any previously inserted segments. + @param segment_start: the start address of the segment in memory (in words) + @param segment_length: the number of words the segment takes in memory (if bigger that the data_length, + the segment is padded with zeros after the end of the data). + @param data_start: the index of the data's start in the inner data array + @param data_length: the number of words in the segment's data + """ + segment_addresses_str = f'seg[{self.segments}]={self.get_segment_addresses_repr(segment_start, segment_length)}' + + if segment_length <= 0: + raise FJWriteFjmException(f"segment-length must be positive (in {segment_addresses_str}).") - for word in self.data: - f.write(pack(write_tag, word)) - - def add_segment(self, segment_start, segment_length, data_start, data_length): if segment_length < data_length: - raise FJWriteFjmException(f"segment-length must be at-least data-length") + raise FJWriteFjmException(f"segment-length must be at-least data-length (in {segment_addresses_str}).") + + if segment_start % 2 == 1 or segment_length % 2 == 1: + raise FJWriteFjmException(f"segment-start and segment-length must be 2*w aligned " + f"(in {segment_addresses_str}).") + + self._validate_segment_not_overlapping(segment_start, segment_length, data_start, data_length) + + if self.version in (RelativeJumpVersion, CompressedVersion): + self._update_to_relative_jumps(segment_start, data_start, data_length) + self.segments.append((segment_start, segment_length, data_start, data_length)) - def add_data(self, data): - start = len(self.data) + def add_data(self, data: List[int]) -> int: + """ + append the data to the current data + @param data: [in]: a list of words + @return: the data start index + """ + data_start = len(self.data) self.data += data - return start, len(data) - - def add_simple_segment_with_data(self, segment_start, data): - data_start, data_length = self.add_data(data) - self.add_segment(segment_start, data_length, data_start, data_length) + return data_start + + def add_simple_segment_with_data(self, segment_start: int, data: List[int]) -> None: + """ + adds the data and a segment that contains exactly the data, to the fjm + @param segment_start: the start address of the segment in memory (in words) + @param data: [in]: a list of words + """ + data_start = self.add_data(data) + self.add_segment(segment_start, len(data), data_start, len(data)) diff --git a/src/fjm_run.py b/src/fjm_run.py index e7f2624..f81d725 100644 --- a/src/fjm_run.py +++ b/src/fjm_run.py @@ -1,203 +1,123 @@ -import pickle -from os import path -from time import time -from sys import stdin, stdout -from typing import Optional, List - -import easygui +from pathlib import Path +from typing import Optional import fjm -from defs import Verbose, TerminationCause +from defs import TerminationCause, PrintTimer, RunStatistics +from breakpoints import BreakpointHandler, handle_breakpoint -def display_message_box_and_get_answer(msg: str, title: str, choices: List[str]) -> str: - # TODO deprecated warning. use another gui (tkinter? seems not so simple) - return easygui.buttonbox(msg, title, choices) +from io_devices.IODevice import IODevice +from io_devices.BrokenIO import BrokenIO +from io_devices.io_exceptions import IOReadOnEOF -def get_address_str(address, breakpoints, labels_dict): - if address in breakpoints: - return f'{hex(address)[2:]} ({breakpoints[address]})' - else: - if address in labels_dict: - return f'{hex(address)[2:]} ({labels_dict[address]})' - else: - address_before = max([a for a in labels_dict if a <= address]) - return f'{hex(address)[2:]} ({labels_dict[address_before]} + {hex(address - address_before)})' +class TerminationStatistics: + """ + saves the run-statistics and data of the fj program-termination, to be presented nicely. + also saves the program's output. + """ + def __init__(self, run_statistics: RunStatistics, termination_cause: TerminationCause): + self.run_time = run_statistics.get_run_time() + self.op_counter = run_statistics.op_counter + self.flip_counter = run_statistics.flip_counter + self.jump_counter = run_statistics.jump_counter -def run(input_file, breakpoints=None, defined_input: Optional[bytes] = None, verbose=False, time_verbose=False, output_verbose=False, - next_break=None, labels_dict=None): - if labels_dict is None: - labels_dict = {} - if breakpoints is None: - breakpoints = {} + self.termination_cause = termination_cause - if time_verbose: - print(f' loading memory: ', end='', flush=True) - start_time = time() - mem = fjm.Reader(input_file) - if time_verbose: - print(f'{time() - start_time:.3f}s') + def __str__(self): + flips_percentage = self.flip_counter / self.op_counter * 100 + jumps_percentage = self.jump_counter / self.op_counter * 100 + return f'Finished by {str(self.termination_cause)} after {self.run_time:.3f}s ' \ + f'(' \ + f'{self.op_counter:,} ops executed; ' \ + f'{flips_percentage:.2f}% flips, ' \ + f'{jumps_percentage:.2f}% jumps' \ + f').' - ip = 0 + +def handle_input(io_device: IODevice, ip: int, mem: fjm.Reader, statistics: RunStatistics) -> None: w = mem.w - out_addr = 2*w - in_addr = 3*w + w.bit_length() # 3w + dww + in_addr = 3 * w + w.bit_length() # 3w + dww + + if ip <= in_addr < ip + 2 * w: + with statistics.pause_timer: + input_bit = io_device.read_bit() + mem.write_bit(in_addr, input_bit) + + +def handle_output(flip_address: int, io_device: IODevice, w: int): + out_addr = 2 * w + if out_addr <= flip_address <= out_addr + 1: + io_device.write_bit(out_addr + 1 == flip_address) + - input_char, input_size = 0, 0 - output_char, output_size = 0, 0 - output = bytes() +def trace_jump(jump_address: int, show_trace: bool) -> None: + if show_trace: + print(hex(jump_address)[2:]) - if 0 not in labels_dict: - labels_dict[0] = 'memory_start_0x0000' - output_anything_yet = False - ops_executed = 0 - flips_executed = 0 +def trace_flip(ip: int, flip_address: int, show_trace: bool) -> None: + if show_trace: + print(hex(ip)[2:].rjust(7), end=': ') + print(hex(flip_address)[2:], end='; ', flush=True) + + +def run(fjm_path: Path, *, + breakpoint_handler: Optional[BreakpointHandler] = None, + io_device: Optional[IODevice] = None, + show_trace: bool = False, + time_verbose: bool = False) \ + -> TerminationStatistics: + """ + run / debug a .fjm file (a FlipJump interpreter) + @param fjm_path: the path to the .fjm file + @param breakpoint_handler:[in]: the breakpoint handler (if not None - debug, and break on its breakpoints) + @param io_device:[in,out]: the device handling input/output + @param show_trace: if true print every opcode executed + @param time_verbose: if true print running times + @return: the run's termination-statistics + """ + with PrintTimer(' loading memory: ', print_time=time_verbose): + mem = fjm.Reader(fjm_path) + + if io_device is None: + io_device = BrokenIO() + + ip = 0 + w = mem.w - start_time = time() - pause_time = 0 + statistics = RunStatistics(w) while True: - if next_break == ops_executed or ip in breakpoints: - pause_time_start = time() - title = "Breakpoint" if ip in breakpoints else "Single Step" - address = get_address_str(ip, breakpoints, labels_dict) - flip = f'flip: {get_address_str(mem.get_word(ip), breakpoints, labels_dict)}' - jump = f'jump: {get_address_str(mem.get_word(ip + w), breakpoints, labels_dict)}' - body = f'Address {address} ({ops_executed} ops executed):\n {flip}.\n {jump}.' - actions = ['Single Step', 'Skip 10', 'Skip 100', 'Skip 1000', 'Continue', 'Continue All'] - print(' program break', end="", flush=True) - action = display_message_box_and_get_answer(body, title, actions) - - if action is None: - action = 'Continue All' - print(f': {action}') - if action == 'Single Step': - next_break = ops_executed + 1 - elif action == 'Skip 10': - next_break = ops_executed + 10 - elif action == 'Skip 100': - next_break = ops_executed + 100 - elif action == 'Skip 1000': - next_break = ops_executed + 1000 - elif action == 'Continue': - next_break = None - elif action == 'Continue All': - next_break = None - breakpoints.clear() - pause_time += time() - pause_time_start - - f = mem.get_word(ip) - if verbose: - print(f'{hex(ip)[2:].rjust(7)}: {hex(f)[2:]}', end='; ', flush=True) - - ops_executed += 1 - if f >= 2*w: - flips_executed += 1 - - # handle output - if out_addr <= f <= out_addr+1: - output_char |= (f-out_addr) << output_size - output_byte = bytes([output_char]) - output_size += 1 - if output_size == 8: - output += output_byte - if output_verbose: - if verbose: - for _ in range(3): - print() - print(f'Outputted Char: ', end='') - stdout.buffer.write(bytes([output_char])) - stdout.flush() - for _ in range(3): - print() - else: - stdout.buffer.write(bytes([output_char])) - stdout.flush() - output_anything_yet = True - output_char, output_size = 0, 0 - - # handle input - if ip <= in_addr < ip+2*w: - if input_size == 0: - if defined_input is None: - pause_time_start = time() - input_char = stdin.buffer.read(1)[0] - pause_time += time() - pause_time_start - elif len(defined_input) > 0: - input_char = defined_input[0] - defined_input = defined_input[1:] - else: - if output_verbose and output_anything_yet: - print() - run_time = time() - start_time - pause_time - return run_time, ops_executed, flips_executed, output, TerminationCause.Input # no more input - input_size = 8 - mem.write_bit(in_addr, input_char & 1) - input_char = input_char >> 1 - input_size -= 1 - - mem.write_bit(f, 1-mem.read_bit(f)) # Flip! - new_ip = mem.get_word(ip+w) - if verbose: - print(hex(new_ip)[2:]) - - if new_ip == ip and not ip <= f < ip+2*w: - if output_verbose and output_anything_yet and breakpoints: - print() - run_time = time()-start_time-pause_time - return run_time, ops_executed, flips_executed, output, TerminationCause.Looping # infinite simple loop - if new_ip < 2*w: - if output_verbose and output_anything_yet and breakpoints: - print() - run_time = time() - start_time - pause_time - return run_time, ops_executed, flips_executed, output, TerminationCause.NullIP # null ip - ip = new_ip # Jump! - - -def debug_and_run(input_file, debugging_file=None, - defined_input: Optional[bytes] = None, verbose=None, - breakpoint_addresses=None, breakpoint_labels=None, breakpoint_any_labels=None): - if breakpoint_any_labels is None: - breakpoint_any_labels = set() - if breakpoint_labels is None: - breakpoint_labels = set() - if breakpoint_addresses is None: - breakpoint_addresses = set() - if verbose is None: - verbose = set() - - labels = [] - if debugging_file is not None: - if path.isfile(debugging_file): - with open(debugging_file, 'rb') as f: - labels = pickle.load(f) - else: - print(f"Warning: debugging file {debugging_file} can't be found!") - elif breakpoint_labels or breakpoint_addresses or breakpoint_any_labels: - print(f"Warning: debugging labels can't be found! no debugging file specified.") - - # Handle breakpoints - breakpoint_map = {ba: hex(ba) for ba in breakpoint_addresses} - for bl in breakpoint_labels: - if bl not in labels: - print(f"Warning: Breakpoint label {bl} can't be found!") - else: - breakpoint_map[labels[bl]] = bl - for bal in breakpoint_any_labels: - for label in labels: - if bal in label: - breakpoint_map[labels[label]] = f'{bal}@{label}' - - opposite_labels = {labels[label]: label for label in labels} - - run_time, ops_executed, flips_executed, output, termination_cause = run( - input_file, defined_input=defined_input, - verbose=Verbose.Run in verbose, - time_verbose=Verbose.Time in verbose, - output_verbose=Verbose.PrintOutput in verbose, - breakpoints=breakpoint_map, labels_dict=opposite_labels) - - return run_time, ops_executed, flips_executed, output, termination_cause + # handle breakpoints + if breakpoint_handler and breakpoint_handler.should_break(ip, statistics.op_counter): + breakpoint_handler = handle_breakpoint(breakpoint_handler, ip, mem, statistics) + + # read flip word + flip_address = mem.get_word(ip) + trace_flip(ip, flip_address, show_trace) + + # handle IO + handle_output(flip_address, io_device, w) + try: + handle_input(io_device, ip, mem, statistics) + except IOReadOnEOF: + return TerminationStatistics(statistics, TerminationCause.EOF) + + # FLIP! + mem.write_bit(flip_address, not mem.read_bit(flip_address)) + + # read jump word + jump_address = mem.get_word(ip+w) + trace_jump(jump_address, show_trace) + statistics.register_op(ip, flip_address, jump_address) + + # check finish? + if jump_address == ip and not ip <= flip_address < ip+2*w: + return TerminationStatistics(statistics, TerminationCause.Looping) + if jump_address < 2*w: + return TerminationStatistics(statistics, TerminationCause.NullIP) + + # JUMP! + ip = jump_address diff --git a/src/io_devices/BrokenIO.py b/src/io_devices/BrokenIO.py new file mode 100644 index 0000000..b9ca083 --- /dev/null +++ b/src/io_devices/BrokenIO.py @@ -0,0 +1,15 @@ +from .IODevice import IODevice +from .io_exceptions import BrokenIOUsed + + +class BrokenIO(IODevice): + """ + IO device that raises error on any IO action + """ + def read_bit(self) -> bool: + raise BrokenIOUsed("program tried to read a bit from the BrokenIO device") + + def write_bit(self, bit: bool) -> None: + raise BrokenIOUsed(f"program tried to write a bit ({int(bit)}) to the BrokenIO device") + + # default __del__ diff --git a/src/io_devices/FixedIO.py b/src/io_devices/FixedIO.py new file mode 100644 index 0000000..e53e072 --- /dev/null +++ b/src/io_devices/FixedIO.py @@ -0,0 +1,51 @@ +from .IODevice import IODevice +from .io_exceptions import IOReadOnEOF, IncompleteOutput + + +class FixedIO(IODevice): + """ + read from fixed input, don't output (with get_output functionality) + """ + def __init__(self, _input: bytes): + self.remaining_input = _input + self._output = b'' + + self.current_input_byte = 0 + self.bits_to_read_in_input_byte = 0 + + self.current_output_byte = 0 + self.bits_to_write_in_output_byte = 0 + + def read_bit(self) -> bool: + if 0 == self.bits_to_read_in_input_byte: + if not self.remaining_input: + raise IOReadOnEOF("Read an empty input on fixed IO (EOF)") + + self.current_input_byte = self.remaining_input[0] + self.remaining_input = self.remaining_input[1:] + self.bits_to_read_in_input_byte = 8 + + bit = (self.current_input_byte & 1) == 1 + self.current_input_byte >>= 1 + self.bits_to_read_in_input_byte -= 1 + return bit + + def write_bit(self, bit: bool) -> None: + self.current_output_byte |= bit << self.bits_to_write_in_output_byte + self.bits_to_write_in_output_byte += 1 + + if 8 == self.bits_to_write_in_output_byte: + self._output += self.current_output_byte.to_bytes(1, 'little') + self.current_output_byte = 0 + self.bits_to_write_in_output_byte = 0 + + def get_output(self) -> bytes: + """ + @raise IncompleteOutput when the number of outputted bits can't be divided by 8 + @return: full output until now + """ + if 0 != self.bits_to_write_in_output_byte: + raise IncompleteOutput("tries to get output when an unaligned number of bits was outputted " + "(doesn't divide 8)") + + return self._output diff --git a/src/io_devices/IODevice.py b/src/io_devices/IODevice.py new file mode 100644 index 0000000..eaff5f8 --- /dev/null +++ b/src/io_devices/IODevice.py @@ -0,0 +1,16 @@ +from abc import ABC, abstractmethod + + +class IODevice(ABC): + """ + abstract IO device + """ + @abstractmethod + def read_bit(self) -> bool: + return False + + @abstractmethod + def write_bit(self, bit: bool) -> None: + pass + + # Also, each class should implement a "__del__" to flush last changes before it gets deleted. diff --git a/src/io_devices/StandardIO.py b/src/io_devices/StandardIO.py new file mode 100644 index 0000000..278086d --- /dev/null +++ b/src/io_devices/StandardIO.py @@ -0,0 +1,59 @@ +from sys import stdin, stdout + +from .IODevice import IODevice +from .io_exceptions import IOReadOnEOF, IncompleteOutput + + +io_bytes_encoding = 'raw_unicode_escape' + + +class StandardIO(IODevice): + """ + read from stdin, write to stdout + """ + def __init__(self, output_verbose: bool): + """ + @param output_verbose: if true print program's output + """ + self.output_verbose = output_verbose + self._output = b'' + + self.current_input_byte = 0 + self.bits_to_read_in_input_byte = 0 + + self.current_output_byte = 0 + self.bits_to_write_in_output_byte = 0 + + def read_bit(self) -> bool: + if 0 == self.bits_to_read_in_input_byte: + read_bytes = stdin.read(1).encode(encoding=io_bytes_encoding) + if 0 == len(read_bytes): + raise IOReadOnEOF("Read an empty input on standard IO (EOF)") + + self.current_input_byte = read_bytes[0] + self.bits_to_read_in_input_byte = 8 + + bit = (self.current_input_byte & 1) == 1 + self.current_input_byte >>= 1 + self.bits_to_read_in_input_byte -= 1 + return bit + + def write_bit(self, bit: bool) -> None: + self.current_output_byte |= bit << self.bits_to_write_in_output_byte + self.bits_to_write_in_output_byte += 1 + + if 8 == self.bits_to_write_in_output_byte: + curr_output: bytes = self.current_output_byte.to_bytes(1, 'little') + if self.output_verbose: + stdout.write(curr_output.decode(encoding=io_bytes_encoding)) + stdout.flush() + self._output += curr_output + self.current_output_byte = 0 + self.bits_to_write_in_output_byte = 0 + + def get_output(self) -> bytes: + if 0 != self.bits_to_write_in_output_byte: + raise IncompleteOutput("tries to get output when an unaligned number of bits was outputted " + "(doesn't divide 8)") + + return self._output diff --git a/src/io_devices/io_exceptions.py b/src/io_devices/io_exceptions.py new file mode 100644 index 0000000..0ab1601 --- /dev/null +++ b/src/io_devices/io_exceptions.py @@ -0,0 +1,14 @@ +class IODeviceException(IOError): + pass + + +class BrokenIOUsed(IODeviceException): + pass + + +class IOReadOnEOF(IODeviceException): + pass + + +class IncompleteOutput(IODeviceException): + pass diff --git a/src/macro_usage_graph.py b/src/macro_usage_graph.py new file mode 100644 index 0000000..8a6d579 --- /dev/null +++ b/src/macro_usage_graph.py @@ -0,0 +1,85 @@ +import collections +from typing import Dict, Tuple, List + +import plotly.graph_objects as go + +from defs import macro_separator_string + + +def _prepare_first_and_second_level_significant_macros( + child_significance_min_thresh: float, macro_code_size: Dict[str, int], + main_thresh: float, secondary_thresh: float)\ + -> Tuple[Dict[str, int], Dict[str, Dict[str, int]]]: + first_level = {} + second_level = collections.defaultdict(lambda: dict()) + for k, v in macro_code_size.items(): + if macro_separator_string not in k: + if v < main_thresh: + continue + first_level[k] = v + else: + if v < secondary_thresh: + continue + k_split = k.split(macro_separator_string) + if len(k_split) != 2: + continue + parent, name = k_split + if float(v) / macro_code_size[parent] < child_significance_min_thresh: + continue + second_level[parent][name] = v + return first_level, second_level + + +def _clean_name_for_pie_graph(macro_name: str) -> str: + return macro_name + + +def _choose_most_significant_macros(first_level: Dict[str, int], second_level: Dict[str, Dict[str, int]], + secondary_thresh: float, total_code_size: int) -> List[Tuple[str, int]]: + chosen = [] + for k, v in sorted(first_level.items(), key=lambda x: x[1], reverse=True): + k_name = _clean_name_for_pie_graph(k) + if len(second_level[k]) == 0: + chosen.append((k_name, v)) + else: + for k2, v2 in sorted(second_level[k].items(), key=lambda x: x[1], reverse=True): + k2_name = _clean_name_for_pie_graph(k2) + chosen.append((f"{k_name} => {k2_name}", v2)) + v -= v2 + if v >= secondary_thresh: + chosen.append((f"{k_name} others", v)) + others = total_code_size - sum([value for label, value in chosen]) + chosen.append(('all others', others)) + return chosen + + +def _show_macro_usage_graph(chosen_macros: List[Tuple[str, int]]) -> None: + fig = go.Figure(data=[go.Pie(labels=[label for label, value in chosen_macros], + values=[value for label, value in chosen_macros], + textinfo='label+percent' + )]) + fig.show() + + +def show_macro_usage_pie_graph(macro_code_size: Dict[str, int], total_code_size: int, *, + min_main_thresh: float = 0.05, min_secondary_thresh: float = 0.01, + child_significance_min_thresh: float = 0.1) -> None: + """ + choose and present in a pie graph the macros with the most code-usage + @param macro_code_size: dictionary between macro-paths and their code-size. + @param total_code_size: total number of FlipJump ops in the program. + @param min_main_thresh: the fraction of the program's code-usage needed for a 1st-level macro to be chosen. + @param min_secondary_thresh: the fraction of the program's code-usage needed for a 2nd-level macro to be chosen. + @param child_significance_min_thresh: the fraction of the 1st-level macro code-usage needed + for its 2nd-level macro (son) needs to be chosen + """ + main_thresh = min_main_thresh * total_code_size + secondary_thresh = min_secondary_thresh * total_code_size + + first_level, second_level = _prepare_first_and_second_level_significant_macros( + child_significance_min_thresh, macro_code_size, main_thresh, secondary_thresh + ) + + chosen_macros = _choose_most_significant_macros(first_level, second_level, secondary_thresh, total_code_size) + + _show_macro_usage_graph(chosen_macros) diff --git a/src/ops.py b/src/ops.py new file mode 100644 index 0000000..89b31c7 --- /dev/null +++ b/src/ops.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Union, Dict, Set, List, Tuple + +from expr import Expr +from exceptions import FJExprException + + +@dataclass +class CodePosition: + """ + A position in the .fj files. + """ + file: str + file_short_name: str # shortened file name. usually s1,s2,.. for stl, and f1,f2,.. for the rest. + line: int + + def __str__(self) -> str: + return f"file {self.file} (line {self.line})" + + def short_str(self) -> str: + return f"{self.file_short_name}:l{self.line}" + + +class MacroName: + """ + Unique for every macro definition. + """ + def __init__(self, name: str, parameter_num: int = 0): + self.name = name + self.parameter_num = parameter_num + + def __str__(self) -> str: + if 0 == self.parameter_num: + return self.name + return f"{self.name}({self.parameter_num})" + + def to_tuple(self): + return self.name, self.parameter_num + + def __hash__(self): + return hash(self.to_tuple()) + + def __eq__(self, other): + return type(other) == MacroName and self.to_tuple() == other.to_tuple() + + +# The macro that holds the ops that are outside any macro. +initial_macro_name = MacroName('') +initial_args = [] +initial_labels_prefix = '' + + +class FlipJump: + """ + The python representation of the "flip; jump" fj-assembly op. + """ + def __init__(self, flip: Expr, jump: Expr, code_position: CodePosition): + self.flip = flip + self.jump = jump + self.code_position = code_position + + def __str__(self): + return f"Flip: {self.flip}, Jump: {self.jump}, at {self.code_position}" + + def eval_new(self, labels_dict: Dict[str, Expr]) -> FlipJump: + return FlipJump(self.flip.eval_new(labels_dict), self.jump.eval_new(labels_dict), self.code_position) + + def all_unknown_labels(self) -> Set[str]: + return {label + for expr in (self.flip, self.jump) + for label in expr.all_unknown_labels()} + + def get_flip(self, labels: Dict[str, int]) -> int: + return self.flip.exact_eval(labels) + + def get_jump(self, labels: Dict[str, int]) -> int: + return self.jump.exact_eval(labels) + + +class WordFlip: + """ + The python representation of the "wflip address, value [, return_address]" fj-assembly op. + """ + def __init__(self, word_address: Expr, flip_value: Expr, return_address: Expr, code_position: CodePosition): + self.word_address = word_address + self.flip_value = flip_value + self.return_address = return_address + self.code_position = code_position + + def __str__(self): + return f"Flip Word {self.word_address} by {self.flip_value}, and return to {self.return_address}. " \ + f"at {self.code_position}" + + def eval_new(self, labels_dict: Dict[str, Expr]) -> WordFlip: + return WordFlip(self.word_address.eval_new(labels_dict), self.flip_value.eval_new(labels_dict), + self.return_address.eval_new(labels_dict), self.code_position) + + def all_unknown_labels(self) -> Set[str]: + return {label + for expr in (self.word_address, self.flip_value, self.return_address) + for label in expr.all_unknown_labels()} + + def get_word_address(self, labels: Dict[str, int]) -> int: + return self.word_address.exact_eval(labels) + + def get_flip_value(self, labels: Dict[str, int]) -> int: + return self.flip_value.exact_eval(labels) + + def get_return_address(self, labels: Dict[str, int]) -> int: + return self.return_address.exact_eval(labels) + + +class Pad: + """ + The python representation of the "pad ops_alignment" fj-assembly op. + """ + def __init__(self, ops_alignment: Expr, code_position: CodePosition): + self.ops_alignment = ops_alignment + self.code_position = code_position + + def __str__(self): + return f"Pad {self.ops_alignment} ops, at {self.code_position}" + + def eval_new(self, labels_dict: Dict[str, Expr]) -> Pad: + return Pad(self.ops_alignment.eval_new(labels_dict), self.code_position) + + def all_unknown_labels(self) -> Set[str]: + return self.ops_alignment.all_unknown_labels() + + def calculate_ops_alignment(self, labels: Dict[str, int]) -> int: + try: + return self.ops_alignment.exact_eval(labels) + except FJExprException as e: + raise FJExprException(f"Can't calculate pad ops_alignment on {self.code_position}") from e + + +class Segment: + """ + The python representation of the "segment start_address" fj-assembly op. + """ + def __init__(self, start_address: Expr, code_position: CodePosition): + self.start_address = start_address + self.code_position = code_position + + def __str__(self): + return f"Segment {self.start_address}, at {self.code_position}" + + def eval_new(self, labels_dict: Dict[str, Expr]) -> Segment: + return Segment(self.start_address.eval_new(labels_dict), self.code_position) + + def all_unknown_labels(self) -> Set[str]: + return {label for label in self.start_address.all_unknown_labels()} + + def calculate_address(self, labels: Dict[str, int]) -> int: + try: + return self.start_address.exact_eval(labels) + except FJExprException as e: + raise FJExprException(f"Can't calculate segment address on {self.code_position}") from e + + +class Reserve: + """ + The python representation of the "reserve bit_size" fj-assembly op. + """ + def __init__(self, reserved_bit_size: Expr, code_position: CodePosition): + self.reserved_bit_size = reserved_bit_size + self.code_position = code_position + + def __str__(self): + return f"Reserve {self.reserved_bit_size}, at {self.code_position}" + + def eval_new(self, labels_dict: Dict[str, Expr]) -> Reserve: + return Reserve(self.reserved_bit_size.eval_new(labels_dict), self.code_position) + + def all_unknown_labels(self) -> Set[str]: + return {label for label in self.reserved_bit_size.all_unknown_labels()} + + def calculate_reserved_bit_size(self, labels: Dict[str, int]) -> int: + try: + return self.reserved_bit_size.exact_eval(labels) + except FJExprException as e: + raise FJExprException(f"Can't calculate reserved bits size on {self.code_position}") from e + + +class MacroCall: + """ + The python representation of the "macro-call [args..]" fj-assembly op. + """ + def __init__(self, macro_name: str, arguments: List[Expr], code_position: CodePosition): + self.macro_name = MacroName(macro_name, len(arguments)) + self.arguments = arguments + self.code_position = code_position + + def __str__(self): + return f"macro call. {self.macro_name.name} {', '.join(map(str, self.arguments))}. at {self.code_position}" + + def eval_new(self, labels_dict: Dict[str, Expr]) -> MacroCall: + return MacroCall(self.macro_name.name, [arg.eval_new(labels_dict) for arg in self.arguments], self.code_position) + + def all_unknown_labels(self) -> Set[str]: + return {label for expr in self.arguments for label in expr.all_unknown_labels()} + + def trace_str(self) -> str: + return f'macro {self.macro_name} ({self.code_position})' + + +class RepCall: + """ + The python representation of the "rep(n, i) macro_call [args..]" fj-assembly op. + """ + def __init__(self, repeat_times: Expr, iterator_name: str, macro_name: str, arguments: List[Expr], + code_position: CodePosition): + self.repeat_times = repeat_times + self.iterator_name = iterator_name + self.macro_name = MacroName(macro_name, len(arguments)) + self.arguments = arguments + self.code_position = code_position + + def __str__(self): + return f"rep call. rep({self.repeat_times}, {self.iterator_name}) {self.macro_name.name} " \ + f"{', '.join(map(str, self.arguments))}. at {self.code_position}" + + def eval_new(self, labels_dict: Dict[str, Expr]) -> RepCall: + return RepCall(self.repeat_times.eval_new(labels_dict), self.iterator_name, self.macro_name.name, + [expr.eval_new(labels_dict) for expr in self.arguments], self.code_position) + + def all_unknown_labels(self) -> Set[str]: + times = self.repeat_times + arguments = self.arguments + arguments_labels = set(label for e in arguments for label in e.all_unknown_labels()) + return times.all_unknown_labels() | (arguments_labels - {self.iterator_name}) + + def get_times(self) -> int: + try: + return int(self.repeat_times) + except FJExprException as e: + raise FJExprException(f"Can't calculate rep times on {self.code_position}") from e + + def calculate_times(self, labels: Dict[str, int]) -> int: + try: + times = self.repeat_times.exact_eval(labels) + self.repeat_times = Expr(times) + return times + except FJExprException as e: + raise FJExprException(f"Can't calculate rep times on {self.code_position}") from e + + def calculate_arguments(self, iterator_value: int) -> Tuple[Expr, ...]: + iterator_dict = {self.iterator_name: Expr(iterator_value)} + try: + return tuple(expr.eval_new(iterator_dict) for expr in self.arguments) + except FJExprException as e: + raise FJExprException(f"Can't calculate rep arguments on {self.code_position}") from e + + def trace_str(self, iter_value: int) -> str: + """ + @note assumes calculate_times successfully called before + """ + return f'rep({self.iterator_name}={iter_value}, out of 0..{int(self.repeat_times)-1}) ' \ + f'macro {self.macro_name} ({self.code_position})' + + +class Label: + """ + The python representation of the "label:" fj-assembly op. + """ + def __init__(self, name: str, code_position: CodePosition): + self.name = name + self.code_position = code_position + + def __str__(self): + return f'Label "{self.name}:", at {self.code_position}' + + def eval_name(self, labels_dict: Dict[str, Expr]) -> str: + if self.name in labels_dict: + new_name = labels_dict[self.name].value + if isinstance(new_name, str): + return new_name + raise FJExprException(f'Bad label swap (from {self.name} to {labels_dict[self.name]}) in {self.code_position}.') + return self.name + + +def get_used_labels(ops: List[Op]) -> Set[str]: + used_labels = set() + for op in ops: + if not isinstance(op, Label): + used_labels.update(op.all_unknown_labels()) + return used_labels + + +def get_declared_labels(ops: List[Op]) -> Set[str]: + return set(op.name for op in ops if isinstance(op, Label)) + + +# The input for the preprocessor +Op = Union[FlipJump, WordFlip, Pad, Label, MacroCall, RepCall, Segment, Reserve] + + +WFLIP_NOT_INSERTED_YET = -1 + + +class NewSegment: + """ + The python expressions-resolved (all compilation data is known) representation + of the "segment start_address" fj-assembly op. + """ + def __init__(self, start_address: int): + """ + @param start_address: the first address of the new segment + """ + self.start_address = start_address + + # a stub, to be resolved later with the start of the wflip area address + self.wflip_start_address = WFLIP_NOT_INSERTED_YET + + +class ReserveBits: + """ + The python expressions-resolved (all compilation data is known) representation + of the "reserve bit_size" fj-assembly op. + """ + def __init__(self, first_address_after_reserved: int): + """ + @param first_address_after_reserved: the address right after the reserved "segment". + """ + self.first_address_after_reserved = first_address_after_reserved + + +class Padding: + """ + The python expressions-resolved (all compilation data is known) representation + of the "pad ops_alignment" fj-assembly op. + """ + def __init__(self, ops_count: int): + """ + @param ops_count: the number of fj-ops to pad. + """ + self.ops_count = ops_count + + +# The input to the labels-resolve +LastPhaseOp = Union[FlipJump, WordFlip, Padding, NewSegment, ReserveBits] diff --git a/src/preprocessor.py b/src/preprocessor.py index 97e5b35..385022b 100644 --- a/src/preprocessor.py +++ b/src/preprocessor.py @@ -1,232 +1,265 @@ +from __future__ import annotations + import collections -from copy import deepcopy -from itertools import count +from typing import Dict, Tuple, Iterable, Union, Deque + +from expr import Expr +from defs import CodePosition, Macro, macro_separator_string +from exceptions import FJPreprocessorException, FJExprException +from ops import FlipJump, WordFlip, Label, Segment, Reserve, MacroCall, RepCall, \ + LastPhaseOp, MacroName, NewSegment, ReserveBits, Pad, Padding, \ + initial_macro_name, initial_args, initial_labels_prefix +from macro_usage_graph import show_macro_usage_pie_graph + +CurrTree = Deque[Union[MacroCall, RepCall]] + +wflip_start_label = '_.wflip_area_start_' + + +def macro_resolve_error(curr_tree: CurrTree, msg='', *, orig_exception: BaseException = None) -> None: + """ + raise a descriptive error (with the macro-expansion trace). + @param curr_tree: the ops in the macro-calling path to arrive in this macro + @param msg: the message to show on error + @param orig_exception: if not None, raise from this base error. + """ + error_str = f"Macro Resolve Error" + (f':\n {msg}\n' if msg else '.\n') + if curr_tree: + error_str += 'Macro call trace:\n' + for i, op in enumerate(curr_tree): + error_str += f' {i}) {op.trace_str()}\n' + raise FJPreprocessorException(error_str) from orig_exception + + +class PreprocessorData: + """ + maintains the preprocessor "global" data structures, throughout its recursion. + e.g. current address, resulting ops, labels' dictionary, macros' dictionary... + also offer many functions to manipulate its data. + @note should call finish before get_result..(). + """ + class _PrepareMacroCall: + def __init__(self, curr_tree: CurrTree, + calling_op: Union[MacroCall, RepCall], macros: Dict[MacroName, Macro]): + self.curr_tree = curr_tree + self.calling_op = calling_op + self.macros = macros + + def __enter__(self): + macro_name = self.calling_op.macro_name + if macro_name not in self.macros: + macro_resolve_error(self.curr_tree, f"macro {macro_name} is used but isn't defined.") + self.curr_tree.append(self.calling_op) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.curr_tree.pop() + + def __init__(self, w: int, macros: Dict[MacroName, Macro]): + self.w = w + self.macros = macros + + self.curr_address: int = 0 + + self.macro_code_size = collections.defaultdict(lambda: 0) + + self.curr_tree: CurrTree = collections.deque() + + self.curr_segment_index: int = 0 + self.labels_code_positions: Dict[str, CodePosition] = {} -import plotly.graph_objects as go + self.result_ops: Deque[LastPhaseOp] = collections.deque() + self.labels: Dict[str, int] = {} -from defs import main_macro, wflip_start_label, new_label, \ - Op, OpType, SegmentEntry, Expr, FJPreprocessorException, \ - eval_all, id_swap + first_segment: NewSegment = NewSegment(0) + self.last_new_segment: NewSegment = first_segment + self.result_ops.append(first_segment) + def patch_last_wflip_address(self) -> None: + self.last_new_segment.wflip_start_address = self.curr_address -def macro_resolve_error(curr_tree, msg=''): - error_str = f"Macro Resolve Error" + (f':\n {msg}' if msg else '.') + f'\nmacro call trace:\n' - for i, trace_str in enumerate(curr_tree): - error_str += f' {i}) {trace_str}\n' - raise FJPreprocessorException(error_str) + def finish(self, show_statistics: bool) -> None: + self.patch_last_wflip_address() + if show_statistics: + show_macro_usage_pie_graph(dict(self.macro_code_size), self.curr_address) + def prepare_macro_call(self, calling_op: Union[MacroCall, RepCall]) -> PreprocessorData._PrepareMacroCall: + return PreprocessorData._PrepareMacroCall(self.curr_tree, calling_op, self.macros) -def output_ops(ops, output_file): - with open(output_file, 'w') as f: - for op in ops: - eval_all(op) - if op.type == OpType.FlipJump: - f.write(f' {op.data[0]};{op.data[1]}\n') - elif op.type == OpType.WordFlip: - f.write(f' wflip {op.data[0]}, {op.data[1]}, {op.data[2]}\n') - elif op.type == OpType.Label: - f.write(f'{op.data[0]}:\n') + def get_result_ops_and_labels(self) -> Tuple[Deque[LastPhaseOp], Dict[str, int]]: + return self.result_ops, self.labels + def insert_segment(self, next_segment_start: int) -> None: + self.labels[f'{wflip_start_label}{self.curr_segment_index}'] = self.curr_address + self.curr_segment_index += 1 -def clean_name_for_pie_graph(macro_name: str): - if '_rep_' not in macro_name: - return macro_name + self.patch_last_wflip_address() + new_segment = NewSegment(next_segment_start) + self.last_new_segment = new_segment + self.result_ops.append(new_segment) + + self.curr_address = next_segment_start + + def insert_reserve(self, reserved_bits_size: int) -> None: + self.curr_address += reserved_bits_size + self.result_ops.append(ReserveBits(self.curr_address)) + + def insert_label(self, label: str, code_position: CodePosition) -> None: + if label in self.labels: + other_position = self.labels_code_positions[label] + macro_resolve_error(self.curr_tree, f'label declared twice - "{label}" on ' + f'{code_position} and {other_position}') + self.labels_code_positions[label] = code_position + self.labels[label] = self.curr_address + + def register_macro_code_size(self, macro_path: str, init_curr_address: int) -> None: + if 1 <= len(self.curr_tree) <= 2: + self.macro_code_size[macro_path] += self.curr_address - init_curr_address + + def align_current_address(self, ops_alignment: int) -> None: + op_size = 2 * self.w + ops_to_pad = (-self.curr_address // op_size) % ops_alignment + self.curr_address += ops_to_pad * op_size + self.result_ops.append(Padding(ops_to_pad)) + + +def get_rep_times(op: RepCall, preprocessor_data: PreprocessorData) -> int: try: - rep_count = macro_name.split('_')[3] - inner_macro = macro_name.split("'")[1] - arg_count = macro_name.split(', ')[1].split(')')[0] - return f"{inner_macro}({arg_count})*{rep_count}" - except IndexError: - return macro_name - - -def dict_pie_graph(d, total, min_main_thresh=0.05, min_secondary_thresh=0.02): - main_thresh = min_main_thresh * total - secondary_thresh = min_secondary_thresh * total - first_level = {} - second_level = collections.defaultdict(lambda: dict()) - for k, v in d.items(): - if ' => ' not in k: - if v < main_thresh: - continue - first_level[k] = v - else: - if v < secondary_thresh: - continue - k_split = k.split(' => ') - if len(k_split) != 2: - continue - parent, name = k_split - second_level[parent][name] = v - - chosen = [] - for k, v in sorted(first_level.items(), key=lambda x: x[1], reverse=True): - k_name = clean_name_for_pie_graph(k) - if len(second_level[k]) == 0: - chosen.append((k_name, v)) - else: - for k2, v2 in sorted(second_level[k].items(), key=lambda x: x[1], reverse=True): - k2_name = clean_name_for_pie_graph(k2) - chosen.append((f"{k_name} => {k2_name}", v2)) - v -= v2 - if v >= secondary_thresh: - chosen.append((f"{k_name} others", v)) - - others = total - sum([value for label, value in chosen]) - chosen.append(('all others', others)) - - fig = go.Figure(data=[go.Pie(labels=[label for label, value in chosen], - values=[value for label, value in chosen], - textinfo='label+percent' - )]) - fig.show() - - -def resolve_macros(w, macros, output_file=None, show_statistics=False, verbose=False): - curr_address = [0] - rem_ops = [] - labels = {} - last_address_index = [0] - label_places = {} - boundary_addresses = [(SegmentEntry.StartAddress, 0)] # SegEntries - stat_dict = collections.defaultdict(lambda: 0) - - ops = resolve_macro_aux(w, '', [], macros, main_macro, [], {}, count(), stat_dict, - labels, rem_ops, boundary_addresses, curr_address, last_address_index, label_places, - verbose) - if output_file: - output_ops(ops, output_file) - - if show_statistics: - dict_pie_graph(dict(stat_dict), curr_address[0]) - - boundary_addresses.append((SegmentEntry.WflipAddress, curr_address[0])) - return rem_ops, labels, boundary_addresses - - -def try_int(op, expr): - if expr.is_int(): - return expr.val - raise FJPreprocessorException(f"Can't resolve the following name: {expr.eval({}, op.file, op.line)} (in op={op}).") - - -def resolve_macro_aux(w, parent_name, curr_tree, macros, macro_name, args, rep_dict, dollar_count, stat_dict, - labels, rem_ops, boundary_addresses, curr_address, last_address_index, label_places, - verbose=False, file=None, line=None): - commands = [] - init_curr_address = curr_address[0] - this_curr_address = 0 - if macro_name not in macros: - macro_name = f'{macro_name[0]}({macro_name[1]})' - if None in (file, line): - macro_resolve_error(curr_tree, f"macro {macro_name} isn't defined.") - else: - macro_resolve_error(curr_tree, f"macro {macro_name} isn't defined. Used in file {file} (line {line}).") - full_name = (f"{parent_name} => " if parent_name else "") + macro_name[0] + (f"({macro_name[1]})" if macro_name[0] - else "") - (params, dollar_params), ops, (_, _, ns_name) = macros[macro_name] - id_dict = dict(zip(params, args)) - for dp in dollar_params: - id_dict[dp] = new_label(dollar_count, dp) - for k in rep_dict: - id_dict[k] = rep_dict[k] - if ns_name: - for k in list(id_dict.keys()): - id_dict[f'{ns_name}.{k}'] = id_dict[k] - - for op in ops: - # macro-resolve - if type(op) is not Op: - macro_resolve_error(curr_tree, f"bad op (not of Op type)! type {type(op)}, str {str(op)}.") - if verbose: - print(op) - op = deepcopy(op) - eval_all(op, id_dict) - id_swap(op, id_dict) - if op.type == OpType.Macro: - commands += resolve_macro_aux(w, full_name, curr_tree+[op.macro_trace_str()], macros, op.data[0], - list(op.data[1:]), {}, dollar_count, stat_dict, - labels, rem_ops, boundary_addresses, curr_address, last_address_index, - label_places, verbose, file=op.file, line=op.line) - elif op.type == OpType.Rep: - eval_all(op, labels) - n, i_name, macro_call = op.data - if not n.is_int(): - macro_resolve_error(curr_tree, f'Rep used without a number "{str(n)}" ' - f'in file {op.file} line {op.line}.') - times = n.val - if times == 0: + return op.calculate_times(preprocessor_data.labels) + except FJExprException as e: + macro_resolve_error(preprocessor_data.curr_tree, f'rep {op.macro_name} failed.', orig_exception=e) + + +def get_pad_ops_alignment(op: Pad, preprocessor_data: PreprocessorData) -> int: + try: + return op.calculate_ops_alignment(preprocessor_data.labels) + except FJExprException as e: + macro_resolve_error(preprocessor_data.curr_tree, f'pad {op.ops_alignment} failed.', orig_exception=e) + + +def get_next_segment_start(op: Segment, preprocessor_data: PreprocessorData) -> int: + try: + next_segment_start = op.calculate_address(preprocessor_data.labels) + if next_segment_start % preprocessor_data.w != 0: + macro_resolve_error(preprocessor_data.curr_tree, f'segment ops must have a w-aligned address: ' + f'{hex(next_segment_start)}. In {op.code_position}.') + return next_segment_start + except FJExprException as e: + macro_resolve_error(preprocessor_data.curr_tree, f'segment failed.', orig_exception=e) + + +def get_reserved_bits_size(op: Reserve, preprocessor_data: PreprocessorData) -> int: + try: + reserved_bits_size = op.calculate_reserved_bit_size(preprocessor_data.labels) + if reserved_bits_size % preprocessor_data.w != 0: + macro_resolve_error(preprocessor_data.curr_tree, f'reserve ops must have a w-aligned value: ' + f'{hex(reserved_bits_size)}. In {op.code_position}.') + return reserved_bits_size + except FJExprException as e: + macro_resolve_error(preprocessor_data.curr_tree, f'reserve failed.', orig_exception=e) + + +def get_params_dictionary(current_macro: Macro, args: Iterable[Expr], namespace: str, labels_prefix: str) \ + -> Dict[str, Expr]: + """ + generates the dictionary between the labels (params and local-params) defined by the macro, and their Expr-values. + @param current_macro: the current macro + @param args: the macro's arguments (Expressions) + @param namespace: the current namespace + @param labels_prefix: the path to the currently-preprocessed macro + @return: the parameters' dictionary + """ + params_dict: Dict[str, Expr] = dict(zip(current_macro.params, args)) + + for local_param in current_macro.local_params: + params_dict[local_param] = Expr(f'{labels_prefix}---{local_param}') + + if namespace: + for k, v in tuple(params_dict.items()): + params_dict[f'{namespace}.{k}'] = v + + return params_dict + + +def resolve_macro_aux(preprocessor_data: PreprocessorData, + macro_name: MacroName, args: Iterable[Expr], labels_prefix: str) -> None: + """ + recursively unwind the current macro into a serialized stream of ops and add them to the result_ops-queue. + also add every label's value to the labels-dictionary. both saved in preprocessor_data. + @param preprocessor_data: maintains the preprocessor "global" data structures + @param macro_name: the name of the macro to unwind + @param args: the arguments for the macro to unwind + @param labels_prefix: The prefix for all labels defined in this macro + """ + init_curr_address = preprocessor_data.curr_address + current_macro = preprocessor_data.macros[macro_name] + params_dict = get_params_dictionary(current_macro, args, current_macro.namespace, labels_prefix) + + for op in current_macro.ops: + + if isinstance(op, Label): + preprocessor_data.insert_label(op.eval_name(params_dict), op.code_position) + + elif isinstance(op, FlipJump) or isinstance(op, WordFlip): + preprocessor_data.curr_address += 2 * preprocessor_data.w + params_dict['$'] = Expr(preprocessor_data.curr_address) + preprocessor_data.result_ops.append(op.eval_new(params_dict)) + del params_dict['$'] + + elif isinstance(op, Pad): + op = op.eval_new(params_dict) + ops_alignment = get_pad_ops_alignment(op, preprocessor_data) + preprocessor_data.align_current_address(ops_alignment) + + elif isinstance(op, MacroCall): + op = op.eval_new(params_dict) + next_macro_path = (f"{labels_prefix}{macro_separator_string}" if labels_prefix else "") + \ + f"{op.code_position.short_str()}:{op.macro_name}" + with preprocessor_data.prepare_macro_call(op): + resolve_macro_aux(preprocessor_data, + op.macro_name, op.arguments, next_macro_path) + + elif isinstance(op, RepCall): + op = op.eval_new(params_dict) + rep_times = get_rep_times(op, preprocessor_data) + if rep_times == 0: continue - if i_name in rep_dict: - macro_resolve_error(curr_tree, f'Rep index {i_name} is declared twice; maybe an inner rep. ' - f'in file {op.file} line {op.line}.') - macro_name = macro_call.data[0] - pseudo_macro_name = (new_label(dollar_count, f'rep_{times}_{macro_name}').val, 1) # just moved outside (before) the for loop - for i in range(times): - rep_dict[i_name] = Expr(i) # TODO - call the macro_name directly, and do deepcopy(op) beforehand. - macros[pseudo_macro_name] = (([], []), [macro_call], (op.file, op.line, ns_name)) - commands += resolve_macro_aux(w, full_name, curr_tree+[op.rep_trace_str(i, times)], macros, - pseudo_macro_name, [], rep_dict, dollar_count, stat_dict, - labels, rem_ops, boundary_addresses, curr_address, last_address_index, - label_places, verbose, file=op.file, line=op.line) - if i_name in rep_dict: - del rep_dict[i_name] - else: - macro_resolve_error(curr_tree, f'Rep is used but {i_name} index is gone; maybe also declared elsewhere.' - f' in file {op.file} line {op.line}.') - - # labels_resolve - elif op.type == OpType.Segment: - eval_all(op, labels) - value = try_int(op, op.data[0]) - if value % w != 0: - macro_resolve_error(curr_tree, f'segment ops must have a w-aligned address. In {op}.') - - boundary_addresses.append((SegmentEntry.WflipAddress, curr_address[0])) - labels[f'{wflip_start_label}{last_address_index[0]}'] = Expr(curr_address[0]) - last_address_index[0] += 1 - - this_curr_address += value - curr_address[0] - curr_address[0] = value - boundary_addresses.append((SegmentEntry.StartAddress, curr_address[0])) - rem_ops.append(op) - elif op.type == OpType.Reserve: - eval_all(op, labels) - value = try_int(op, op.data[0]) - if value % w != 0: - macro_resolve_error(curr_tree, f'reserve ops must have a w-aligned value. In {op}.') - - this_curr_address += value - curr_address[0] += value - boundary_addresses.append((SegmentEntry.ReserveAddress, curr_address[0])) - labels[f'{wflip_start_label}{last_address_index[0]}'] = Expr(curr_address[0]) - - last_address_index[0] += 1 - rem_ops.append(op) - elif op.type in {OpType.FlipJump, OpType.WordFlip}: - this_curr_address += 2*w - curr_address[0] += 2*w - eval_all(op, {'$': Expr(curr_address[0])}) - if verbose: - print(f'op added: {str(op)}') - rem_ops.append(op) - elif op.type == OpType.Label: - label = op.data[0] - if label in labels: - other_file, other_line = label_places[label] - macro_resolve_error(curr_tree, f'label declared twice - "{label}" on file {op.file} (line {op.line}) ' - f'and file {other_file} (line {other_line})') - if verbose: - print(f'label added: "{label}" in {op.file} line {op.line}') - labels[label] = Expr(curr_address[0]) - label_places[label] = (op.file, op.line) + next_macro_path = (f"{labels_prefix}{macro_separator_string}" if labels_prefix else "") + \ + f"{op.code_position.short_str()}:rep{{}}:{op.macro_name}" + with preprocessor_data.prepare_macro_call(op): + for i in range(rep_times): + resolve_macro_aux(preprocessor_data, + op.macro_name, op.calculate_arguments(i), next_macro_path.format(i)) + + elif isinstance(op, Segment): + op = op.eval_new(params_dict) + next_segment_start = get_next_segment_start(op, preprocessor_data) + preprocessor_data.insert_segment(next_segment_start) + + elif isinstance(op, Reserve): + op = op.eval_new(params_dict) + reserved_bits_size = get_reserved_bits_size(op, preprocessor_data) + preprocessor_data.insert_reserve(reserved_bits_size) + else: - macro_resolve_error(curr_tree, f"Can't assemble this opcode - {str(op)}") - - # if len(curr_tree) == 1: - # stat_dict[macro_name[0]] += curr_address[0] - init_curr_address - # stat_dict[macro_name[0]] += this_curr_address - if 1 <= len(curr_tree) <= 2: - stat_dict[full_name] += curr_address[0] - init_curr_address - return commands + macro_resolve_error(preprocessor_data.curr_tree, f"Can't assemble this opcode - {str(op)}") + + preprocessor_data.register_macro_code_size(labels_prefix, init_curr_address) + + +def resolve_macros(w: int, macros: Dict[MacroName, Macro], *, show_statistics: bool = False) \ + -> Tuple[Deque[LastPhaseOp], Dict[str, int]]: + """ + unwind the macro tree to a serialized-queue of ops, + and creates a dictionary from label's name to its address. + @param w: the memory-width + @param macros: parser's result; the dictionary from the macro names to the macro declaration + @param show_statistics: if True then prints the macro-usage statistics + @return: tuple of the queue of ops, and the labels' dictionary + """ + preprocessor_data = PreprocessorData(w, macros) + resolve_macro_aux(preprocessor_data, + initial_macro_name, initial_args, initial_labels_prefix) + + preprocessor_data.finish(show_statistics) + return preprocessor_data.get_result_ops_and_labels() diff --git a/stl/declib.fj b/stl/declib.fj index d3c2e8f..ab53f37 100644 --- a/stl/declib.fj +++ b/stl/declib.fj @@ -12,6 +12,11 @@ +// TODO implement dec.vec, maths (inc_by, add, sub, mul, div), if/cmp, input/output. +// no need for logics, pointers. + + + // ---------- Memory Variables: // Size Complexity: 1 diff --git a/stl/iolib.fj b/stl/iolib.fj index d357916..67fd3f0 100644 --- a/stl/iolib.fj +++ b/stl/iolib.fj @@ -278,6 +278,7 @@ def bit2hex n, hex, bit { def hex2bit bit, hex { + bit.zero 4, bit hex.exact_xor bit+3*dw+dbit, bit+2*dw+dbit, bit+dw+dbit, bit+dbit, hex } diff --git a/stl/ptrlib.fj b/stl/ptrlib.fj index 3879a4a..414e647 100644 --- a/stl/ptrlib.fj +++ b/stl/ptrlib.fj @@ -14,6 +14,11 @@ +// TODO read 4-bit values (good for bit/hex/dec) values from pointers. +// it might be cool to implement hex pointers if it can improve speed easily, but it's not needed. + + + // ---------- Init diff --git a/stl/runlib.fj b/stl/runlib.fj index e69ef4c..e21ddbb 100644 --- a/stl/runlib.fj +++ b/stl/runlib.fj @@ -59,10 +59,11 @@ def wflip_macro dst, val, jmp_addr { wflip dst, val, jmp_addr } -def pad x @ pad_start { - pad_start: - rep((0-pad_start/(2*w))%x, i) zero_op -} +//// @note - padding can also be implemented in fj itself! (but the saved-word pad is more compile-time efficient) +//def pad x @ pad_start { +// pad_start: +// rep((0-pad_start/(2*w))%x, i) zero_op +//} // ---------- Compilation Time: diff --git a/tests/README.md b/tests/README.md index a7f3b7a..ae44953 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,12 +7,18 @@ run `pytest` to run the fast tests. Run with `--compile` / `--run` for testing only the compilation / the run. Add a combination of `--fast`, `--medium`, `--slow`, `--hexlib` to run tests of different types.
-Use `--all` to run all the tests. The default (no type flags) means `--fast`. +Use `--regulr` to run all the tests of stable parts.
+Use `--all` to run all the tests.
+The default (no type flags) means `--fast`. + +![Running Pytest with --regular](../res/pytest.gif) You can run the tests parallel with `-n auto` (using [xdist](https://github.com/pytest-dev/pytest-xdist)).
note that this option is only allowed while using exactly one of `--compile` / `--run`.
You can execute the `test_parallel` / `test_parallel.bat` to run parallel compile, and afterwords parallel run, with the given flags. +![Running the test_parallel script with --regular](../res/test_parallel.gif) + Please note that the 7 `hexlib-div-*` tests currently fail. ### Filter tests by their name @@ -32,24 +38,24 @@ The python test itself can be found on [test_fj.py](test_fj.py) (and [conftest.p To add a new test, first choose the relevant csv file.
The rule of thumb (for the sum of compile+run times, in seconds): -fast | medium | slow ----|---|--- -0 → 0.5 | 0.5 → 5 | else +| fast | medium | slow | +|--------------|--------------|------| +| 0 → 0.5 | 0.5 → 5 | else | Then add a new line to the relevant compile-csv and run-csv files, according to the next formats. ### Compile CSVs format: -test name | .fj paths | out .fjm path | memory width | version | flags | use stl | treat warnings as errors ----|---|---|---|---|---|---|--- -example_test | path/to/example_1.fj | ... | path/to/example_n.fj | path/to/compiled/example.fjm | 64 | 1 | 0 | True | True +| test name | .fj paths | out .fjm path | memory width | version | flags | use stl | treat warnings as errors | +|--------------|-------------------------------------------------------------|------------------------------|--------------|---------|-------|---------|--------------------------| +| example_test | path/to/example_1.fj | ... | path/to/example_n.fj | path/to/compiled/example.fjm | 64 | 1 | 0 | True | True | Note that you can specify a single file, or a '|' separated list of files in the .fj paths cell. In case of a list, they will be compiled in the inserted order. ### Run CSVs format: -test name | .fjm path | input file path | output file path | is input a binary file | is output a binary file ----|---|---|---|---|--- -example_test | path/to/compiled/example.fjm | path/to/inputs/example.in | path/to/outputs/example.out | False | False +| test name | .fjm path | input file path | output file path | is input a binary file | is output a binary file | +|--------------|------------------------------|---------------------------|-----------------------------|------------------------|-------------------------| +| example_test | path/to/compiled/example.fjm | path/to/inputs/example.in | path/to/outputs/example.out | False | False | Note that you can also emit specifying a file in the input/output cell, and leave it empty. In that case an empty input/output will be used. diff --git a/tests/conf.json b/tests/conf.json index 58fb33a..e2bea44 100644 --- a/tests/conf.json +++ b/tests/conf.json @@ -1,6 +1,11 @@ { "default_type": "fast", - "ordered_speed_list": [ + "regular_speed_ordered": [ + "fast", + "medium", + "slow" + ], + "all_speed_ordered": [ "fast", "medium", "slow", diff --git a/tests/conftest.py b/tests/conftest.py index 10fffa0..bca8656 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,13 +18,20 @@ COMPILE_ARGUMENTS_FIXTURE = "compile_args" RUN_ARGUMENTS_FIXTURE = "run_args" +fixtures_name_to_type = { + COMPILE_ARGUMENTS_FIXTURE: CompileTestArgs, + RUN_ARGUMENTS_FIXTURE: RunTestArgs, +} + TESTS_PATH = Path(__file__).parent with open(TESTS_PATH / 'conf.json', 'r') as tests_json: TESTS_OPTIONS = json.load(tests_json) -TEST_TYPES = TESTS_OPTIONS['ordered_speed_list'] +TEST_TYPES = TESTS_OPTIONS['all_speed_ordered'] assert TEST_TYPES +REGULAR_TYPES = TESTS_OPTIONS['regular_speed_ordered'] +assert REGULAR_TYPES DEFAULT_TYPE = TESTS_OPTIONS['default_type'] assert DEFAULT_TYPE in TEST_TYPES @@ -34,6 +41,7 @@ ALL_FLAG = 'all' +REGULAR_FLAG = 'regular' COMPILE_FLAG = 'compile' RUN_FLAG = 'run' NAME_EXACT_FLAG = 'name' @@ -109,6 +117,7 @@ def pytest_addoption(parser) -> None: for test_type in TEST_TYPES: parser.addoption(f"--{test_type}", action="store_true", help=f"run {test_type} tests") + parser.addoption(f"--{REGULAR_FLAG}", action="store_true", help=f"run all regular tests ({', '.join(REGULAR_TYPES)})") parser.addoption(f"--{ALL_FLAG}", action="store_true", help=f"run all tests") parser.addoption(f"--{COMPILE_FLAG}", action='store_true', help='only test compiling .fj files') @@ -151,12 +160,15 @@ def get_test_types_to_run__heavy_first(get_option: Callable[[str], bool]) -> Lis @param get_option: function that returns the flags values @return: list of the test types to run """ - test_types_heavy_first = TEST_TYPES[::-1] + all_test_types_heavy_first = TEST_TYPES[::-1] + regular_test_types_heavy_first = REGULAR_TYPES[::-1] if get_option(ALL_FLAG): - types_to_run = list(test_types_heavy_first) + types_to_run = list(all_test_types_heavy_first) + elif get_option(REGULAR_FLAG): + types_to_run = list(regular_test_types_heavy_first) else: - types_to_run = list(filter(get_option, test_types_heavy_first)) + types_to_run = list(filter(get_option, all_test_types_heavy_first)) if not types_to_run: types_to_run = [DEFAULT_TYPE] return types_to_run @@ -233,6 +245,27 @@ def get_option(opt): metafunc.parametrize(RUN_ARGUMENTS_FIXTURE, run_tests__heavy_first, ids=repr) +def is_not_skipped(test) -> bool: + if hasattr(test, 'callspec') and hasattr(test.callspec, 'params'): + params = test.callspec.params + for fixture_name, fixture_type in fixtures_name_to_type.items(): + if fixture_name in params: + return isinstance(params[fixture_name], fixture_type) + return True + + +@pytest.hookimpl(hookwrapper=True) +def pytest_collectreport(report): + report.result = filter(is_not_skipped, report.result) + yield + + +@pytest.hookimpl(hookwrapper=True) +def pytest_collection_modifyitems(config, items): + yield + items[:] = filter(is_not_skipped, items) + + def get_tests_from_csvs__heavy_first__execute_once(get_option: Callable[[str], Any]) -> Tuple[List, List]: """ get the tests from the csv. heavy first. diff --git a/tests/inout/calc_tests/calc1.out b/tests/inout/calc_tests/calc1.out index c727f3e..4f674a5 100644 --- a/tests/inout/calc_tests/calc1.out +++ b/tests/inout/calc_tests/calc1.out @@ -1,3 +1,2 @@ -> 111111111*111111111 -12345678987654321 -> q +> 12345678987654321 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc10.out b/tests/inout/calc_tests/calc10.out index 4f1cd80..69cd653 100644 --- a/tests/inout/calc_tests/calc10.out +++ b/tests/inout/calc_tests/calc10.out @@ -1,3 +1,2 @@ -> -15%4 --3 -> q +> -3 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc2.out b/tests/inout/calc_tests/calc2.out index bf46e43..1c613be 100644 --- a/tests/inout/calc_tests/calc2.out +++ b/tests/inout/calc_tests/calc2.out @@ -1,3 +1,2 @@ -> xd0a0c0d0 + x0e0d000e -0xDEADC0DE -> q +> 0xDEADC0DE +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc3.out b/tests/inout/calc_tests/calc3.out index c30064c..6198a1d 100644 --- a/tests/inout/calc_tests/calc3.out +++ b/tests/inout/calc_tests/calc3.out @@ -1,3 +1,2 @@ -> 23456-765 -22691 -> q +> 22691 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc4.out b/tests/inout/calc_tests/calc4.out index 97c3243..789255b 100644 --- a/tests/inout/calc_tests/calc4.out +++ b/tests/inout/calc_tests/calc4.out @@ -1,3 +1,2 @@ -> X777+x34 -0x7AB -> q +> 0x7AB +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc5.out b/tests/inout/calc_tests/calc5.out index 8500d8f..ea6015a 100644 --- a/tests/inout/calc_tests/calc5.out +++ b/tests/inout/calc_tests/calc5.out @@ -1,3 +1,2 @@ -> x1000+128 -0x1080 -> q +> 0x1080 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc6.out b/tests/inout/calc_tests/calc6.out index 9140c73..d794c42 100644 --- a/tests/inout/calc_tests/calc6.out +++ b/tests/inout/calc_tests/calc6.out @@ -1,3 +1,2 @@ -> 2073600/1080 -1920 -> q +> 1920 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc7.out b/tests/inout/calc_tests/calc7.out index 8ff3cf7..5a8ac2b 100644 --- a/tests/inout/calc_tests/calc7.out +++ b/tests/inout/calc_tests/calc7.out @@ -1,3 +1,2 @@ -> 23-100 --77 -> q +> -77 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc8.out b/tests/inout/calc_tests/calc8.out index 173b74c..a786dc8 100644 --- a/tests/inout/calc_tests/calc8.out +++ b/tests/inout/calc_tests/calc8.out @@ -1,3 +1,2 @@ -> 34%7 -6 -> q +> 6 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc9.out b/tests/inout/calc_tests/calc9.out index a9e21c0..69cd653 100644 --- a/tests/inout/calc_tests/calc9.out +++ b/tests/inout/calc_tests/calc9.out @@ -1,3 +1,2 @@ -> -15/4 --3 -> q +> -3 +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc_empty.out b/tests/inout/calc_tests/calc_empty.out index ec322d9..1ad4d1a 100644 --- a/tests/inout/calc_tests/calc_empty.out +++ b/tests/inout/calc_tests/calc_empty.out @@ -1 +1 @@ -> q +> \ No newline at end of file diff --git a/tests/inout/calc_tests/calc_many.in b/tests/inout/calc_tests/calc_many.in index ccead9c..a3077c1 100644 --- a/tests/inout/calc_tests/calc_many.in +++ b/tests/inout/calc_tests/calc_many.in @@ -3,8 +3,10 @@ -6 5 + 7 1 - 2 + 13+2 + 7 0 -2+2 @@ -13,6 +15,9 @@ g 0x +q 8 +q q +q klewfkew x9 xbabe q diff --git a/tests/inout/calc_tests/calc_many.out b/tests/inout/calc_tests/calc_many.out index 3cf6ea1..15cd963 100644 --- a/tests/inout/calc_tests/calc_many.out +++ b/tests/inout/calc_tests/calc_many.out @@ -1,32 +1,18 @@ > 1 -1 > 2 -2 > -6 --6 -> 5 + 7 -12 -> 1 - 2 --1 -> 13+2 -15 -> -> 7 -7 +> 12 +> -1 +> > 15 +> > > 7 > 0 -0 -> -2+2 -0 -> -2-2 --4 -> -> -> g -Error! -> 0x -Error! -> x9 -0x9 -> xbabe -0xBABE -> q +> 0 +> -4 +> > > Error! +> Error! +> Error! +> Error! +> Error! +> 0x9 +> 0xBABE +> \ No newline at end of file diff --git a/tests/inout/simple_math_checks/series_sum.out b/tests/inout/simple_math_checks/series_sum.out new file mode 100644 index 0000000..b1e449e --- /dev/null +++ b/tests/inout/simple_math_checks/series_sum.out @@ -0,0 +1,4 @@ +params: a1 = 1, d = 3, n = 12. + +an = 34. +Sum(a1, a2, ..., an) = 210. diff --git a/tests/test_compile_fast.csv b/tests/test_compile_fast.csv index d45215f..af5ce13 100644 --- a/tests/test_compile_fast.csv +++ b/tests/test_compile_fast.csv @@ -1,21 +1,21 @@ -cat, programs/print_tests/cat.fj,tests/compiled/print_tests/cat.fjm, 64,1,0, True,True -hello_no-stl, programs/print_tests/hello_no-stl.fj,tests/compiled/print_tests/hello_no-stl.fjm, 64,1,0, False,True -hello_world, programs/print_tests/hello_world.fj,tests/compiled/print_tests/hello_world.fjm, 64,1,0, True,True -hexprint, programs/print_tests/hexprint.fj,tests/compiled/print_tests/hexprint.fjm, 64,1,0, True,True -mathbit, programs/sanity_checks/mathbit.fj,tests/compiled/sanity_checks/mathbit.fjm, 64,1,0, True,True -mathvec, programs/sanity_checks/mathvec.fj,tests/compiled/sanity_checks/mathvec.fjm, 64,1,0, True,True -nadd, programs/simple_math_checks/nadd.fj,tests/compiled/simple_math_checks/nadd.fjm, 64,1,0, True,True -ncat, programs/print_tests/ncat.fj,tests/compiled/print_tests/ncat.fjm, 64,1,0, True,True -ncmp, programs/simple_math_checks/ncmp.fj,tests/compiled/simple_math_checks/ncmp.fjm, 64,1,0, True,True -not, programs/sanity_checks/not.fj,tests/compiled/sanity_checks/not.fjm, 64,1,0, True,True -print_as_digit, programs/print_tests/print_as_digit.fj,tests/compiled/print_tests/print_as_digit.fjm, 64,1,0, True,True +cat, programs/print_tests/cat.fj,tests/compiled/print_tests/cat.fjm, 64,3,0, True,True +hello_no-stl, programs/print_tests/hello_no-stl.fj,tests/compiled/print_tests/hello_no-stl.fjm, 64,3,0, False,True +hello_world, programs/print_tests/hello_world.fj,tests/compiled/print_tests/hello_world.fjm, 64,3,0, True,True +hexprint, programs/print_tests/hexprint.fj,tests/compiled/print_tests/hexprint.fjm, 64,3,0, True,True +mathbit, programs/sanity_checks/mathbit.fj,tests/compiled/sanity_checks/mathbit.fjm, 64,3,0, True,True +mathvec, programs/sanity_checks/mathvec.fj,tests/compiled/sanity_checks/mathvec.fjm, 64,3,0, True,True +nadd, programs/simple_math_checks/nadd.fj,tests/compiled/simple_math_checks/nadd.fjm, 64,3,0, True,True +ncat, programs/print_tests/ncat.fj,tests/compiled/print_tests/ncat.fjm, 64,3,0, True,True +ncmp, programs/simple_math_checks/ncmp.fj,tests/compiled/simple_math_checks/ncmp.fjm, 64,3,0, True,True +not, programs/sanity_checks/not.fj,tests/compiled/sanity_checks/not.fjm, 64,3,0, True,True +print_as_digit, programs/print_tests/print_as_digit.fj,tests/compiled/print_tests/print_as_digit.fjm, 64,3,0, True,True quine16, programs/quine16.fj,tests/compiled/quine16.fjm, 16,0,0, True,True -rep, programs/sanity_checks/rep.fj,tests/compiled/sanity_checks/rep.fjm, 64,1,0, True,True -simple, programs/sanity_checks/simple.fj,tests/compiled/sanity_checks/simple.fjm, 64,1,0, True,True -testbit, programs/sanity_checks/testbit.fj,tests/compiled/sanity_checks/testbit.fjm, 64,1,0, True,True -testbit_with_nops, programs/sanity_checks/testbit_with_nops.fj,tests/compiled/sanity_checks/testbit_with_nops.fjm, 64,1,0, True,True +rep, programs/sanity_checks/rep.fj,tests/compiled/sanity_checks/rep.fjm, 64,3,0, True,True +simple, programs/sanity_checks/simple.fj,tests/compiled/sanity_checks/simple.fjm, 64,3,0, True,True +testbit, programs/sanity_checks/testbit.fj,tests/compiled/sanity_checks/testbit.fjm, 64,3,0, True,True +testbit_with_nops, programs/sanity_checks/testbit_with_nops.fj,tests/compiled/sanity_checks/testbit_with_nops.fjm, 64,3,0, True,True -multi_comp_stl_bc, programs/multi_comp/defs.fj | programs/multi_comp/a.fj | programs/multi_comp/b.fj | programs/multi_comp/c.fj, tests/compiled/multi_comp/multi_comp_stl_bc.fjm, 64,1,0, True,True -multi_comp_no_stl_bc, programs/multi_comp/defs.fj | programs/multi_comp/a_no_stl.fj | programs/multi_comp/b.fj | programs/multi_comp/c.fj, tests/compiled/multi_comp/multi_comp_no_stl_bc.fjm, 64,1,0, False,True -multi_comp_stl_cb, programs/multi_comp/defs.fj | programs/multi_comp/a.fj | programs/multi_comp/c.fj | programs/multi_comp/b.fj, tests/compiled/multi_comp/multi_comp_stl_cb.fjm, 64,1,0, True,True -multi_comp_no_stl_cb, programs/multi_comp/defs.fj | programs/multi_comp/a_no_stl.fj | programs/multi_comp/c.fj | programs/multi_comp/b.fj, tests/compiled/multi_comp/multi_comp_no_stl_cb.fjm, 64,1,0, False,True +multi_comp_stl_bc, programs/multi_comp/defs.fj | programs/multi_comp/a.fj | programs/multi_comp/b.fj | programs/multi_comp/c.fj, tests/compiled/multi_comp/multi_comp_stl_bc.fjm, 64,3,0, True,True +multi_comp_no_stl_bc, programs/multi_comp/defs.fj | programs/multi_comp/a_no_stl.fj | programs/multi_comp/b.fj | programs/multi_comp/c.fj, tests/compiled/multi_comp/multi_comp_no_stl_bc.fjm, 64,3,0, False,True +multi_comp_stl_cb, programs/multi_comp/defs.fj | programs/multi_comp/a.fj | programs/multi_comp/c.fj | programs/multi_comp/b.fj, tests/compiled/multi_comp/multi_comp_stl_cb.fjm, 64,3,0, True,True +multi_comp_no_stl_cb, programs/multi_comp/defs.fj | programs/multi_comp/a_no_stl.fj | programs/multi_comp/c.fj | programs/multi_comp/b.fj, tests/compiled/multi_comp/multi_comp_no_stl_cb.fjm, 64,3,0, False,True diff --git a/tests/test_compile_hexlib.csv b/tests/test_compile_hexlib.csv index 9641c27..00320bf 100644 --- a/tests/test_compile_hexlib.csv +++ b/tests/test_compile_hexlib.csv @@ -1,44 +1,44 @@ -hexlib-print_as_digit, programs/hexlib_tests/basics1/print_as_digit.fj,tests/compiled/hexlib_tests/basics1/print_as_digit.fjm, 64,1,0, True,True -hexlib-input, programs/hexlib_tests/basics1/input.fj,tests/compiled/hexlib_tests/basics1/input.fjm, 64,1,0, True,True -hexlib-basic_memory, programs/hexlib_tests/basics1/basic_memory.fj,tests/compiled/hexlib_tests/basics1/basic_memory.fjm, 64,1,0, True,True -hexlib-basic_math, programs/hexlib_tests/basics1/basic_math.fj,tests/compiled/hexlib_tests/basics1/basic_math.fjm, 64,1,0, True,True -hexlib-if, programs/hexlib_tests/basics1/if.fj,tests/compiled/hexlib_tests/basics1/if.fjm, 64,1,0, True,True -hexlib-print_int, programs/hexlib_tests/basics1/print_int.fj,tests/compiled/hexlib_tests/basics1/print_int.fjm, 64,1,0, True,True +hexlib-print_as_digit, programs/hexlib_tests/basics1/print_as_digit.fj,tests/compiled/hexlib_tests/basics1/print_as_digit.fjm, 64,3,0, True,True +hexlib-input, programs/hexlib_tests/basics1/input.fj,tests/compiled/hexlib_tests/basics1/input.fjm, 64,3,0, True,True +hexlib-basic_memory, programs/hexlib_tests/basics1/basic_memory.fj,tests/compiled/hexlib_tests/basics1/basic_memory.fjm, 64,3,0, True,True +hexlib-basic_math, programs/hexlib_tests/basics1/basic_math.fj,tests/compiled/hexlib_tests/basics1/basic_math.fjm, 64,3,0, True,True +hexlib-if, programs/hexlib_tests/basics1/if.fj,tests/compiled/hexlib_tests/basics1/if.fjm, 64,3,0, True,True +hexlib-print_int, programs/hexlib_tests/basics1/print_int.fj,tests/compiled/hexlib_tests/basics1/print_int.fjm, 64,3,0, True,True -hexlib-add_count_bits, programs/hexlib_tests/basics2/add_count_bits.fj, tests/compiled/hexlib_tests/basics2/add_count_bits.fjm, 64,1,0, True,True -hexlib-count_bits, programs/hexlib_tests/basics2/count_bits.fj, tests/compiled/hexlib_tests/basics2/count_bits.fjm, 64,1,0, True,True -hexlib-shl_bit, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_bit.fj, tests/compiled/hexlib_tests/basics2/shl_bit.fjm, 64,1,0, True,True -hexlib-shl_bit_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_bit_n.fj, tests/compiled/hexlib_tests/basics2/shl_bit_n.fjm, 64,1,0, True,True -hexlib-shr_bit, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_bit.fj, tests/compiled/hexlib_tests/basics2/shr_bit.fjm, 64,1,0, True,True -hexlib-shr_bit_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_bit_n.fj, tests/compiled/hexlib_tests/basics2/shr_bit_n.fjm, 64,1,0, True,True -hexlib-shl_hex, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_hex.fj, tests/compiled/hexlib_tests/basics2/shl_hex.fjm, 64,1,0, True,True -hexlib-shl_hex_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_hex_n.fj, tests/compiled/hexlib_tests/basics2/shl_hex_n.fjm, 64,1,0, True,True -hexlib-shr_hex, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_hex.fj, tests/compiled/hexlib_tests/basics2/shr_hex.fjm, 64,1,0, True,True -hexlib-shr_hex_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_hex_n.fj, tests/compiled/hexlib_tests/basics2/shr_hex_n.fjm, 64,1,0, True,True +hexlib-add_count_bits, programs/hexlib_tests/basics2/add_count_bits.fj, tests/compiled/hexlib_tests/basics2/add_count_bits.fjm, 64,3,0, True,True +hexlib-count_bits, programs/hexlib_tests/basics2/count_bits.fj, tests/compiled/hexlib_tests/basics2/count_bits.fjm, 64,3,0, True,True +hexlib-shl_bit, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_bit.fj, tests/compiled/hexlib_tests/basics2/shl_bit.fjm, 64,3,0, True,True +hexlib-shl_bit_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_bit_n.fj, tests/compiled/hexlib_tests/basics2/shl_bit_n.fjm, 64,3,0, True,True +hexlib-shr_bit, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_bit.fj, tests/compiled/hexlib_tests/basics2/shr_bit.fjm, 64,3,0, True,True +hexlib-shr_bit_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_bit_n.fj, tests/compiled/hexlib_tests/basics2/shr_bit_n.fjm, 64,3,0, True,True +hexlib-shl_hex, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_hex.fj, tests/compiled/hexlib_tests/basics2/shl_hex.fjm, 64,3,0, True,True +hexlib-shl_hex_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shl_hex_n.fj, tests/compiled/hexlib_tests/basics2/shl_hex_n.fjm, 64,3,0, True,True +hexlib-shr_hex, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_hex.fj, tests/compiled/hexlib_tests/basics2/shr_hex.fjm, 64,3,0, True,True +hexlib-shr_hex_n, programs/hexlib_tests/basics2/shift_utils.fj | programs/hexlib_tests/basics2/shr_hex_n.fj, tests/compiled/hexlib_tests/basics2/shr_hex_n.fjm, 64,3,0, True,True -hexlib-cmp, programs/hexlib_tests/2params/cmp.fj,tests/compiled/hexlib_tests/2params/cmp.fjm, 64,1,0, True,True -hexlib-cmp_n, programs/hexlib_tests/2params/cmp_n.fj,tests/compiled/hexlib_tests/2params/cmp_n.fjm, 64,1,0, True,True -hexlib-add, programs/hexlib_tests/2params/add.fj,tests/compiled/hexlib_tests/2params/add.fjm, 64,1,0, True,True -hexlib-add_n, programs/hexlib_tests/2params/add_n.fj,tests/compiled/hexlib_tests/2params/add_n.fjm, 64,1,0, True,True -hexlib-sub, programs/hexlib_tests/2params/sub.fj,tests/compiled/hexlib_tests/2params/sub.fjm, 64,1,0, True,True -hexlib-sub_n, programs/hexlib_tests/2params/sub_n.fj,tests/compiled/hexlib_tests/2params/sub_n.fjm, 64,1,0, True,True -hexlib-or, programs/hexlib_tests/2params/or.fj,tests/compiled/hexlib_tests/2params/or.fjm, 64,1,0, True,True -hexlib-or_n, programs/hexlib_tests/2params/or_n.fj,tests/compiled/hexlib_tests/2params/or_n.fjm, 64,1,0, True,True -hexlib-and, programs/hexlib_tests/2params/and.fj,tests/compiled/hexlib_tests/2params/and.fjm, 64,1,0, True,True -hexlib-and_n, programs/hexlib_tests/2params/and_n.fj,tests/compiled/hexlib_tests/2params/and_n.fjm, 64,1,0, True,True +hexlib-cmp, programs/hexlib_tests/2params/cmp.fj,tests/compiled/hexlib_tests/2params/cmp.fjm, 64,3,0, True,True +hexlib-cmp_n, programs/hexlib_tests/2params/cmp_n.fj,tests/compiled/hexlib_tests/2params/cmp_n.fjm, 64,3,0, True,True +hexlib-add, programs/hexlib_tests/2params/add.fj,tests/compiled/hexlib_tests/2params/add.fjm, 64,3,0, True,True +hexlib-add_n, programs/hexlib_tests/2params/add_n.fj,tests/compiled/hexlib_tests/2params/add_n.fjm, 64,3,0, True,True +hexlib-sub, programs/hexlib_tests/2params/sub.fj,tests/compiled/hexlib_tests/2params/sub.fjm, 64,3,0, True,True +hexlib-sub_n, programs/hexlib_tests/2params/sub_n.fj,tests/compiled/hexlib_tests/2params/sub_n.fjm, 64,3,0, True,True +hexlib-or, programs/hexlib_tests/2params/or.fj,tests/compiled/hexlib_tests/2params/or.fjm, 64,3,0, True,True +hexlib-or_n, programs/hexlib_tests/2params/or_n.fj,tests/compiled/hexlib_tests/2params/or_n.fjm, 64,3,0, True,True +hexlib-and, programs/hexlib_tests/2params/and.fj,tests/compiled/hexlib_tests/2params/and.fjm, 64,3,0, True,True +hexlib-and_n, programs/hexlib_tests/2params/and_n.fj,tests/compiled/hexlib_tests/2params/and_n.fjm, 64,3,0, True,True -hexlib-add-mul-4, programs/hexlib_tests/mul/add_mul4.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul4.fjm, 64,1,0, True,True -hexlib-add-mul-8, programs/hexlib_tests/mul/add_mul8.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul8.fjm, 64,1,0, True,True -hexlib-add-mul-32, programs/hexlib_tests/mul/add_mul32.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul32.fjm, 64,1,0, True,True -hexlib-add-mul-64, programs/hexlib_tests/mul/add_mul64.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul64.fjm, 64,1,0, True,True -hexlib-mul-16, programs/hexlib_tests/mul/mul16.fj | programs/hexlib_tests/mul/mul_test.fj,tests/compiled/hexlib_tests/mul/mul16.fjm, 64,1,0, True,True -hexlib-mul-32, programs/hexlib_tests/mul/mul32.fj | programs/hexlib_tests/mul/mul_test.fj,tests/compiled/hexlib_tests/mul/mul32.fjm, 64,1,0, True,True +hexlib-add-mul-4, programs/hexlib_tests/mul/add_mul4.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul4.fjm, 64,3,0, True,True +hexlib-add-mul-8, programs/hexlib_tests/mul/add_mul8.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul8.fjm, 64,3,0, True,True +hexlib-add-mul-32, programs/hexlib_tests/mul/add_mul32.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul32.fjm, 64,3,0, True,True +hexlib-add-mul-64, programs/hexlib_tests/mul/add_mul64.fj | programs/hexlib_tests/mul/add_mul_test.fj,tests/compiled/hexlib_tests/mul/add_mul64.fjm, 64,3,0, True,True +hexlib-mul-16, programs/hexlib_tests/mul/mul16.fj | programs/hexlib_tests/mul/mul_test.fj,tests/compiled/hexlib_tests/mul/mul16.fjm, 64,3,0, True,True +hexlib-mul-32, programs/hexlib_tests/mul/mul32.fj | programs/hexlib_tests/mul/mul_test.fj,tests/compiled/hexlib_tests/mul/mul32.fjm, 64,3,0, True,True -hexlib-div-4_1, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test4_1.fj, tests/compiled/hexlib_tests/div/test4_1.fjm, 64,1,0, True,True -hexlib-div-4_2, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test4_2.fj, tests/compiled/hexlib_tests/div/test4_2.fjm, 64,1,0, True,True -hexlib-div-4_4, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test4_4.fj, tests/compiled/hexlib_tests/div/test4_4.fjm, 64,1,0, True,True -hexlib-div-8_1, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_1.fj, tests/compiled/hexlib_tests/div/test8_1.fjm, 64,1,0, True,True -hexlib-div-8_2, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_2.fj, tests/compiled/hexlib_tests/div/test8_2.fjm, 64,1,0, True,True -hexlib-div-8_4, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_4.fj, tests/compiled/hexlib_tests/div/test8_4.fjm, 64,1,0, True,True -hexlib-div-8_8, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_8.fj, tests/compiled/hexlib_tests/div/test8_8.fjm, 64,1,0, True,True +hexlib-div-4_1, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test4_1.fj, tests/compiled/hexlib_tests/div/test4_1.fjm, 64,3,0, True,True +hexlib-div-4_2, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test4_2.fj, tests/compiled/hexlib_tests/div/test4_2.fjm, 64,3,0, True,True +hexlib-div-4_4, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test4_4.fj, tests/compiled/hexlib_tests/div/test4_4.fjm, 64,3,0, True,True +hexlib-div-8_1, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_1.fj, tests/compiled/hexlib_tests/div/test8_1.fjm, 64,3,0, True,True +hexlib-div-8_2, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_2.fj, tests/compiled/hexlib_tests/div/test8_2.fjm, 64,3,0, True,True +hexlib-div-8_4, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_4.fj, tests/compiled/hexlib_tests/div/test8_4.fjm, 64,3,0, True,True +hexlib-div-8_8, programs/hexlib_tests/div/hexlib_div.fj | programs/hexlib_tests/div/test8_8.fj, tests/compiled/hexlib_tests/div/test8_8.fjm, 64,3,0, True,True diff --git a/tests/test_compile_medium.csv b/tests/test_compile_medium.csv index dd4c5b7..0a6b132 100644 --- a/tests/test_compile_medium.csv +++ b/tests/test_compile_medium.csv @@ -1,7 +1,7 @@ -casting, programs/concept_checks/casting.fj,tests/compiled/concept_checks/casting.fjm, 64,1,0, True,True -div10, programs/simple_math_checks/div10.fj,tests/compiled/simple_math_checks/div10.fjm, 64,1,0, True,True -hello_world_with_str, programs/print_tests/hello_world_with_str.fj,tests/compiled/print_tests/hello_world_with_str.fjm, 64,1,0, True,True -print_dec, programs/print_tests/print_dec.fj,tests/compiled/print_tests/print_dec.fjm, 64,1,0, True,True -print_hex_int, programs/print_tests/print_hex_int.fj,tests/compiled/print_tests/print_hex_int.fjm, 64,1,0, True,True -ptr, programs/concept_checks/ptr.fj,tests/compiled/concept_checks/ptr.fjm, 64,1,0, True,True -segments, programs/concept_checks/segments.fj,tests/compiled/concept_checks/segments.fjm, 64,1,0, True,True +casting, programs/concept_checks/casting.fj,tests/compiled/concept_checks/casting.fjm, 64,3,0, True,True +div10, programs/simple_math_checks/div10.fj,tests/compiled/simple_math_checks/div10.fjm, 64,3,0, True,True +hello_world_with_str, programs/print_tests/hello_world_with_str.fj,tests/compiled/print_tests/hello_world_with_str.fjm, 64,3,0, True,True +print_dec, programs/print_tests/print_dec.fj,tests/compiled/print_tests/print_dec.fjm, 64,3,0, True,True +print_hex_int, programs/print_tests/print_hex_int.fj,tests/compiled/print_tests/print_hex_int.fjm, 64,3,0, True,True +ptr, programs/concept_checks/ptr.fj,tests/compiled/concept_checks/ptr.fjm, 64,3,0, True,True +segments, programs/concept_checks/segments.fj,tests/compiled/concept_checks/segments.fjm, 64,3,0, True,True diff --git a/tests/test_compile_slow.csv b/tests/test_compile_slow.csv index 5a54e11..f7fe493 100644 --- a/tests/test_compile_slow.csv +++ b/tests/test_compile_slow.csv @@ -1,11 +1,14 @@ -calc, programs/calc.fj,tests/compiled/calc.fjm, 64,1,0, True,True +calc, programs/calc.fj,tests/compiled/calc.fjm, 64,3,0, True,True -func1, programs/func_tests/func1.fj,tests/compiled/func_tests/func1.fjm, 64,1,0, True,True -func2, programs/func_tests/func2.fj,tests/compiled/func_tests/func2.fjm, 64,1,0, True,True -func3, programs/func_tests/func3.fj,tests/compiled/func_tests/func3.fjm, 64,1,0, True,True -func4, programs/func_tests/func4.fj,tests/compiled/func_tests/func4.fjm, 64,1,0, True,True -func5, programs/func_tests/func5.fj,tests/compiled/func_tests/func5.fjm, 64,1,0, True,True +func1, programs/func_tests/func1.fj,tests/compiled/func_tests/func1.fjm, 64,3,0, True,True +func2, programs/func_tests/func2.fj,tests/compiled/func_tests/func2.fjm, 64,3,0, True,True +func3, programs/func_tests/func3.fj,tests/compiled/func_tests/func3.fjm, 64,3,0, True,True +func4, programs/func_tests/func4.fj,tests/compiled/func_tests/func4.fjm, 64,3,0, True,True +func5, programs/func_tests/func5.fj,tests/compiled/func_tests/func5.fjm, 64,3,0, True,True + +pair_ns1, programs/concept_checks/pair_ns.fj | programs/pair_ns_tests/test1.fj,tests/compiled/pair_ns_tests/test1.fjm, 64,3,0, True,True +pair_ns2, programs/concept_checks/pair_ns.fj | programs/pair_ns_tests/test2.fj,tests/compiled/pair_ns_tests/test2.fjm, 64,3,0, True,True +pair_ns3, programs/concept_checks/pair_ns.fj | programs/pair_ns_tests/test3.fj,tests/compiled/pair_ns_tests/test3.fjm, 64,3,0, True,True + +series_sum, programs/simple_math_checks/series_sum.fj,tests/compiled/simple_math_checks/series_sum.fjm, 64,3,0, True,True -pair_ns1, programs/concept_checks/pair_ns.fj | programs/pair_ns_tests/test1.fj,tests/compiled/pair_ns_tests/test1.fjm, 64,1,0, True,True -pair_ns2, programs/concept_checks/pair_ns.fj | programs/pair_ns_tests/test2.fj,tests/compiled/pair_ns_tests/test2.fjm, 64,1,0, True,True -pair_ns3, programs/concept_checks/pair_ns.fj | programs/pair_ns_tests/test3.fj,tests/compiled/pair_ns_tests/test3.fjm, 64,1,0, True,True diff --git a/tests/test_fj.py b/tests/test_fj.py index dc0e152..8005a25 100644 --- a/tests/test_fj.py +++ b/tests/test_fj.py @@ -1,11 +1,12 @@ +import lzma from queue import Queue from threading import Lock from pathlib import Path -from src import assembler +from src import assembler, fjm from src import fjm_run -from src.defs import TerminationCause, Verbose, get_stl_paths - +from src.defs import TerminationCause, get_stl_paths, io_bytes_encoding +from src.io_devices.FixedIO import FixedIO CSV_TRUE = 'True' CSV_FALSE = 'False' @@ -42,7 +43,10 @@ def __init__(self, test_name: str, fj_paths: str, fjm_out_path: str, included_files = get_stl_paths() if self.use_stl else [] fj_paths_list = map(str.strip, fj_paths.split('|')) fj_absolute_paths_list = [ROOT_PATH / fj_path for fj_path in fj_paths_list] - self.fj_files = included_files + fj_absolute_paths_list + + included_files_tuples = [(f's{i}', path) for i, path in enumerate(included_files, start=1)] + fj_paths_tuples = [(f'f{i}', path) for i, path in enumerate(fj_absolute_paths_list, start=1)] + self.fj_files_tuples = included_files_tuples + fj_paths_tuples self.fjm_out_path = ROOT_PATH / fjm_out_path @@ -71,10 +75,10 @@ def test_compile(compile_args: CompileTestArgs) -> None: create_parent_directories(compile_args.fjm_out_path) - assembler.assemble(compile_args.fj_files, compile_args.fjm_out_path, compile_args.word_size, - version=compile_args.version, flags=compile_args.flags, - warning_as_errors=compile_args.warning_as_errors, - verbose={Verbose.Time}) + fjm_writer = fjm.Writer(compile_args.fjm_out_path, compile_args.word_size, compile_args.version, + flags=compile_args.flags, lzma_preset=lzma.PRESET_DEFAULT) + assembler.assemble(compile_args.fj_files_tuples, compile_args.word_size, fjm_writer, + warning_as_errors=compile_args.warning_as_errors) class RunTestArgs: @@ -133,7 +137,7 @@ def get_expected_output(self) -> str: if self.read_out_as_binary: with open(self.out_file_path, 'rb') as out_f: - return out_f.read().decode('raw_unicode_escape') + return out_f.read().decode(io_bytes_encoding) else: with open(self.out_file_path, 'r') as out_f: return out_f.read() @@ -149,15 +153,16 @@ def test_run(run_args: RunTestArgs) -> None: """ print(f'Running test {run_args.test_name}:') - run_time, ops_executed, flips_executed, output, termination_cause =\ - fjm_run.run(run_args.fjm_path, defined_input=run_args.get_defined_input(), time_verbose=True) + io_device = FixedIO(run_args.get_defined_input()) + termination_statistics = fjm_run.run(run_args.fjm_path, + io_device=io_device, + time_verbose=True) - print(f'finished by {termination_cause} after {run_time:.3f}s ' - f'({ops_executed:,} ops executed, {flips_executed / ops_executed * 100:.2f}% flips)') + print(termination_statistics) expected_termination_cause = TerminationCause.Looping - assert termination_cause == expected_termination_cause + assert termination_statistics.termination_cause == expected_termination_cause - output = output.decode('raw_unicode_escape') + output = io_device.get_output().decode(io_bytes_encoding) expected_output = run_args.get_expected_output() assert output == expected_output diff --git a/tests/test_run_slow.csv b/tests/test_run_slow.csv index 4089f4d..5a50157 100644 --- a/tests/test_run_slow.csv +++ b/tests/test_run_slow.csv @@ -20,3 +20,6 @@ func5, tests/compiled/func_tests/func5.fjm, ,tests/inout/func_tests/func5.out, F pair_ns1, tests/compiled/pair_ns_tests/test1.fjm, ,tests/inout/pair_ns_tests/pair_ns1.out, False,False pair_ns2, tests/compiled/pair_ns_tests/test2.fjm, ,tests/inout/pair_ns_tests/pair_ns2.out, False,False pair_ns3, tests/compiled/pair_ns_tests/test3.fjm, ,tests/inout/pair_ns_tests/pair_ns3.out, False,False + +series_sum, tests/compiled/simple_math_checks/series_sum.fjm, ,tests/inout/simple_math_checks/series_sum.out, False,False +