Prepare for public commit history

We want commit messages to represent finshed pieces of work.
So try to start with a clean slate.

Author: jlapeyre

Diff for: .gitignore

@@ -1,4 +1,4 @@
-/target
+target/
 /Cargo.lock
 dev_docs/diagrams
 crates/oq3_syntax/examples/oq3_source
@@ -7,6 +7,11 @@ organization.md
 parser_design.md
 restorenodes.sh
 restorgenerated.sh
-crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs
+crates/parser/src/syntax_kind/_syntax_kind_enum.rs
 crates/oq3_syntax/src/ast/generated/_new_nodes.rs
 crates/oq3_syntax/src/ast/generated/_new_tokens.rs        
+.venv/
+.venvs/
+.env/
+.envs/
+.ackrc

Diff for: .rustfmt.toml

@@ -0,0 +1,7 @@
+# See https://rust-lang.github.io/rustfmt/?version=v1.6.0&search=
+
+# Use this when it becomes stable
+# reorder_impl_items = true
+
+# When use_small_heuristics is set to Max, then each granular width setting is set to the same value as max_width.
+# use_small_heuristics = "Max"

Diff for: CODE_OF_CONDUCT.md

@@ -0,0 +1,8 @@
+# Code of Conduct
+All members of this project agree to adhere to the Qiskit Code of Conduct as found
+at [https://github.com/Qiskit/qiskit/blob/master/CODE_OF_CONDUCT.md](https://github.com/Qiskit/qiskit/blob/master/CODE_OF_CONDUCT.md)
+
+----
+
+License: [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/),
+Copyright Contributors to Qiskit.

Diff for: CONTRIBUTING.md

@@ -0,0 +1,6 @@
+### Pull Requests
+
+Running `cargo clippy` on the branch to be merged must show no errors.
+To handle a lot of errors at the command line can use (for unix-like OS) `cargo clippy --color always &| less -R`.
+
+

Diff for: Cargo.toml

@@ -27,16 +27,13 @@ authors = ["OpenQASM3 parser team"]
 # # Set this to 1 or 2 to get more useful backtraces in debugger.
 # debug = 0
 
-[patch.'crates-io']
-# rowan = { path = "../rowan" }
-
 [workspace.dependencies]
 # local crates
 
-oq3_lexer = { path = "./crates/oq3_lexer", version = "0.0.0" }
-oq3_parser = { path = "./crates/oq3_parser", version = "0.1.0" }
+lexer = { path = "./crates/lexer", version = "0.0.0" }
+parser = { path = "./crates/parser", version = "0.1.0" }
 oq3_syntax = { path = "./crates/oq3_syntax", version = "0.0.0" }
-semantic_ast = { path = "./crates/semantic_ast", version = "0.0.0" }
+semantics = { path = "./crates/semantics", version = "0.0.0" }
 sourcegen = { path = "./crates/sourcegen", version = "0.0.0" }
 ast_pyo3 = { path = "./crates/ast_pyo3", version = "0.0.0" }
 source_file = { path = "./crates/source_file", version = "0.0.0" }

Diff for: NOTICE.txt

@@ -0,0 +1,28 @@
+The code in crates `lexer`, `parser`, and `oq3_syntax` are derived
+from crates in the rust-analyzer project, commit (d398a).
+The code in these crates is licensed in this OpenQASM 3 parser project under
+the following MIT license:
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

Diff for: README.md

@@ -1,32 +1,46 @@
 # OpenQASM 3 Parser
 
-This is a parser for the OpenQASM 3 language (OQ3) written in Rust.
+This project provides a compiler front end for OpenQASM 3 language (OQ3).
+
 In this document, this parser is referred to as `openqasm3_parser`.
 
+Differences with the [OpenQASM reference parser](https://github.com/openqasm/openqasm) are
+
+* The parser in `openqasm3_parser` is much more performant.
+  A crude test with large source files showed parse time reduced by a factor of 80.
+* `openqasm3_parser` performs semantic analysis.
+
+<details>
+  <summary>What is a rust "crate"</summary>
+
 We talk about rust ["crates"](https://doc.rust-lang.org/book/ch07-01-packages-and-crates.html).
 A rust library crate is more or less the source for a rust library that is developed, built, and installed with the rust package manager [cargo](https://doc.rust-lang.org/cargo/).
 This single repository contains more than one separately installable crates. In the future, this repository may also be used to generate other artifacts.
 
-The code does not yet perfectly do everything promised below.
+</details>
+
+### Status
+
+There have been no releases of any kind. I'll try to keep something more or less meaningful in this spot.
+For instance, I hope to soon replace "I" with "we".
 
 ### Crates (roughly one installable library per crate)
 
-* [oq3_lexer](./crates/oq3_lexer) -- This is a lightly modified version of the `rustc` (the rust compiler) lexer.
-* [oq3_parser](./crates/oq3_parser) -- This parses OQ3 source as lexed by `oq3_lexer` into a nested sequence of tagged sections of text
-according the OQ3 grammar. This is based on the rust-analyzer parser crate. All nodes are represented by an instance
-of a single Rust `struct` (with some support `structs`).
-* [oq3_syntax](./crates/oq3_syntax) -- This crate transforms the AST produced by `oq3_parser` into an AST implemented
-as standard rust data structures: `enum` and `struct` and provides an interface for working with the AST.
-The types of these data structures reflect the syntactic elements of OQ3. The rust-analyzer
-documentation sometimes refers to this AST by something like "typed AST". This can be confusing. It does not mean that semantic
-analysis has been performed and OQ3 types have been assigned to all expressions. Rather it is meant to highlight the distinction:
-The AST produced by `oq3_syntax` represents syntactic elements using the rust type system. The lower AST produced by
-`oq3_parser` represents syntactic elements via tags.
-* [semantic_ast](./crates/semantic_ast) -- Generates an AST for OQ3 with the types of all typed OQ3 elements resolved and
-the referent of all identifiers resolved.
-* [source_file](./crates/source_file) -- A higher-level interface to the syntactic AST. This sits beetween the syntactic and
-semantic ASTs. This crate manages source files, and incuded source files, and diagnostic messages.
-* [ast_pyo3](./crates/source_file) A Python interface. This is a bit exploratory. But fixing the approach is a development priority.
+The first three crates are based on tools for `rust` and `rust-analyzer`.
+
+* [lexer](./crates/lexer) -- A lightly modified version of the `rustc` (the rust compiler) lexer.
+* [parser](./crates/parser) -- Ingests output of `lexer` and outputs a concrete syntax tree.
+* [oq3_syntax](./crates/oq3_syntax) -- Ingests output of `parser` and outputs an abstract syntax tree (AST).
+The rust-analyzer [documentation](#design) sometimes refers to this AST by something like "typed AST".
+This can be confusing. It does not mean that semantic
+analysis has been performed and OQ3 types have been assigned to all expressions. It means that the rust type system is
+used to encode syntactic elements, in contrast to some lower representations in the same crate.
+* [semantics](./crates/semantics) -- Performs [semantic analysis](https://en.wikipedia.org/wiki/Compiler#Front_end)
+and outputs an [abstract semantic graph (ASG)](https://en.wikipedia.org/wiki/Abstract_semantic_graph)
+There are other names for this structure. But "ASG" is convenient.
+* [source_file](./crates/source_file) -- A higher-level interface to the syntactic AST. This sits beetween the syntactic AST and
+semantic ASG. This crate manages the main source file and incuded source files.
+* [ast_pyo3](./crates/source_file) Experimental code. It will not be used in it's current form.
 
 #### Supporting crates
 
@@ -38,10 +52,27 @@ crate has a bug.
 Do not run `cargo test`. Rather use `./run_tests.sh` or commands found therein. This is because codegen is implemented via
 the test system (you read correctly). If possible, we plan to change this to a more conventional approach.
 
+### Using this front end
+
+A reminder: A front end is not of much use unless you have a back end. Examples showing the entry points and how to use them,
+can be found in [./crates/semantics/examples/semdemo.rs](./crates/semantics/examples/semdemo.rs).
+
+```shell
+shell> export QASM3_PATH=./crates/semantics/examples/qasm/
+shell> cargo run --example semdemo -- semantic scratch1.qasm
+```
+
+Replace `scratch1.qasm` with some file found in [./crates/semantics/examples/qasm/](./crates/semantics/examples/qasm/).
+
+#### Search path
+
+The environment variable `QASM_PATH` is a colon separated list of paths. Note that the name follows the venerable unix tradition of
+ending in `PATH` rather than `PATHS`. The code that retrives the paths uses routines `std::path` which may actually handle
+path specifications on other platforms.
+
 ### Design
 
 Code from rust-analyzer has been borrowed and modified for the lower levels of parsing.
-
 The [developer documents for rust-analyzer](https://github.com/rust-lang/rust-analyzer/tree/master/docs/dev) are very relevant as the
 structure has not been changed in adapting to OQ3.
 
@@ -52,10 +83,9 @@ structure has not been changed in adapting to OQ3.
     * [From Pratt to Dijkstra](https://matklad.github.io/2020/04/15/from-pratt-to-dijkstra.html)
     * [Resilient parsing](https://matklad.github.io/2023/05/21/resilient-ll-parsing-tutorial.html)
 
-
 ## Notes
 
-Some of this code is modified from code in [rust-analyzer](https://github.com/rust-lang/rust-analyzer).
+Some of this code is modified from code found in [rust-analyzer](https://github.com/rust-lang/rust-analyzer).
 It was taken at [this commit](https://github.com/rust-lang/rust-analyzer/pull/15380):
 
     commit d398ad3326780598bbf1480014f4c59fbf6461a7

Diff for: codegen_scripts/README.md

@@ -0,0 +1,16 @@
+## Code generation for the crate oq3_syntax
+
+NOTE: These (rather simple) scripts controlling codegen have been developed under fedora linux. They may need
+to be modified for your OS. The codegen itself is done in rust and should run correctly on most
+platforms, or should be easily made to do so.
+
+The source files in [../crates/oq3_syntax/src/ast/generated](../crates/oq3_syntax/src/ast/generated) are generated via scripts
+run during development. They do not need to be run when building the crates if the input
+files to the codegen have not been modified.
+
+On the other hand, if the input files to codegen have been modified, then codegen scripts
+must be run by hand. Some details are in [./mkgenerated](./mkgenerated).
+
+Run ./mkgenerated to write generated rust source to temporary files.
+Run [./cpnodes.sh](./cpnodes.sh) and [./cpgenerated.sh](./cpgenerated.sh) to overwrite the revision-controlled source
+files with the temporary files.

Diff for: cleangenerated.sh renamed to codegen_scripts/cleangenerated.sh

@@ -5,8 +5,9 @@
 # The generated files that are actually in use, nodes.rs and generated.rs
 # are not touched by this script.
 
-rm crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs
-rm crates/oq3_parser/src/syntax_kind/syntax_kind_enum.rs.~*
+cd ..
+rm crates/parser/src/syntax_kind/_syntax_kind_enum.rs
+rm crates/parser/src/syntax_kind/syntax_kind_enum.rs.~*
 
 rm crates/oq3_syntax/src/ast/generated/_new_nodes.rs
 rm crates/oq3_syntax/src/ast/generated/nodes.rs.~*

Diff for: cpgenerated.sh renamed to codegen_scripts/cpgenerated.sh

@@ -4,5 +4,5 @@
 # Copy the generated code from the temporary files to which it is written
 # to it's final location where it will be compiled into the library.
 
-cp -a --backup=t crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs crates/oq3_parser/src/syntax_kind/syntax_kind_enum.rs
+cd .. && cp -a --backup=t crates/parser/src/syntax_kind/_syntax_kind_enum.rs crates/parser/src/syntax_kind/syntax_kind_enum.rs
 

Diff for: cpnodes.sh renamed to codegen_scripts/cpnodes.sh

@@ -4,4 +4,4 @@
 # Copy the generated code from the temporary files to which it is written
 # to its final location where it will be compiled into the library.
 
-cp -a --backup=t  crates/oq3_syntax/src/ast/generated/_new_nodes.rs crates/oq3_syntax/src/ast/generated/nodes.rs
+cd .. && cp -a --backup=t  crates/oq3_syntax/src/ast/generated/_new_nodes.rs crates/oq3_syntax/src/ast/generated/nodes.rs

Diff for: mk_nodes.sh renamed to codegen_scripts/mk_nodes.sh

@@ -6,7 +6,7 @@ echo
 echo =============== Running sourcegen_ast_nodes once ===============
 echo
 
-cargo test sourcegen_ast_nodes
+cd .. && cargo test sourcegen_ast_nodes
 
 # echo
 # echo =============== Running sourcegen_ast_nodes twice ===============
@@ -15,5 +15,5 @@ cargo test sourcegen_ast_nodes
 # cargo test sourcegen_ast_nodes
 
 # rustfmt crates/oq3_syntax/src/ast/generated/_new_nodes.rs
-# rustfmt crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs
+# rustfmt crates/parser/src/syntax_kind/_syntax_kind_enum.rs
 # rustfmt crates/oq3_syntax/src/ast/generated/_new_tokens.rs

Diff for: mk_syntax_kinds.sh renamed to codegen_scripts/mk_syntax_kinds.sh

@@ -6,7 +6,7 @@ echo
 echo ======================= Running write_syntax_kinds_enum once ===============
 echo
 
-cargo test write_syntax_kinds_enum
+cd .. && cargo test write_syntax_kinds_enum
 
 #echo ======================= Running write_syntax_kinds_enum() twice ===============
 # echo >>>>>>>>>>>>>>> Running write_syntax_kinds_enum() twice
@@ -15,6 +15,6 @@ cargo test write_syntax_kinds_enum
 # cargo test write_syntax_kinds_enum
 
 # rustfmt crates/oq3_syntax/src/ast/generated/_new_nodes.rs
-# rustfmt crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs
+# rustfmt crates/parser/src/syntax_kind/_syntax_kind_enum.rs
 # rustfmt crates/oq3_syntax/src/ast/generated/_new_tokens.rs
 

Diff for: mk_syntax_tokens.sh renamed to codegen_scripts/mk_syntax_tokens.sh

@@ -6,9 +6,9 @@ echo
 echo  =============== Running sourcegen_ast_tokens once ===============
 echo
 
-cargo test sourcegen_ast_tokens
+cd .. && cargo test sourcegen_ast_tokens
 
 # rustfmt crates/oq3_syntax/src/ast/generated/_new_nodes.rs
-# rustfmt crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs
+# rustfmt crates/parser/src/syntax_kind/_syntax_kind_enum.rs
 # rustfmt crates/oq3_syntax/src/ast/generated/_new_tokens.rs
 

Diff for: mkgenerated.sh renamed to codegen_scripts/mkgenerated.sh

@@ -10,16 +10,16 @@
 # This should be enough for codegen.
 
 # But you still need to add the things to the parser grammar
-# 4 You might need to update crates/oq3_parser/src/grammar/expressions/atom.rs
-#   Or crates/oq3_parser/src/grammar/items.rs (eg. for `gate`)
+# 4 You might need to update crates/parser/src/grammar/expressions/atom.rs
+#   Or crates/parser/src/grammar/items.rs (eg. for `gate`)
 # Or other grammar files
 
 # Generated files are not given their final names in order not to clobber exisiting generated code
 # Here are the temporary filenames and the final filenames
 # You have to copy them manually.
 # crates/oq3_syntax/src/ast/generated/_new_tokens.rs --> tokens.rs
 # crates/oq3_syntax/src/ast/generated/_new_nodes.rs --> nodes.rs
-# crates/oq3_parser/src/syntax_kind/_generated.rs --> generated.rs
+# crates/parser/src/syntax_kind/_generated.rs --> generated.rs
 
 # Update: Running this script now seems robust. Originally all codegen was done
 # by a single test. I split it into two tests and run each of them twice.
@@ -38,6 +38,7 @@
 # Warning: Do not format till all files are generated.
 # Don't know why, but code gen will fail otherwise.
 
-rustfmt crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs
+cd ..
+rustfmt crates/parser/src/syntax_kind/_syntax_kind_enum.rs
 rustfmt crates/oq3_syntax/src/ast/generated/_new_nodes.rs
 rustfmt crates/oq3_syntax/src/ast/generated/_new_tokens.rs

Diff for: crates/ast_pyo3/buildlib.sh

@@ -1,5 +1,5 @@
 [package]
-name = "oq3_lexer"
+name = "lexer"
 version = "0.0.0"
 edition = "2021"
 

Diff for: crates/ast_pyo3/src/bytecode.rs

@@ -422,7 +422,7 @@ impl Cursor<'_> {
         }
     }
 
-    fn hardware_ident(&mut self) -> TokenKind { // OQ3
+    fn hardware_ident(&mut self) -> TokenKind {
         // debug_assert!(is_id_start(self.prev()));
         // Start is already eaten, eat the rest of identifier.
         match self.first() {

Diff for: crates/oq3_lexer/Cargo.toml renamed to crates/lexer/Cargo.toml

@@ -27,8 +27,8 @@ triomphe.workspace = true
 # local crates
 
 # rustc_lexer.workspace = true
-oq3_lexer.workspace = true
-oq3_parser.workspace = true
+lexer.workspace = true
+parser.workspace = true
 # profile.workspace = true
 stdx.workspace = true
 # text-edit.workspace = true

Diff for: crates/oq3_lexer/src/cursor.rs renamed to crates/lexer/src/cursor.rs

@@ -3,8 +3,8 @@ use std::path::PathBuf;
 use clap::{Parser, Subcommand};
 
 use rowan::NodeOrToken; // TODO: this can be accessed from a higher level
-use oq3_lexer::{tokenize, Token};
-use oq3_parser::SyntaxKind;
+use lexer::{tokenize, Token};
+use parser::SyntaxKind;
 use oq3_syntax::{ast, parse_text, SourceFile, GreenNode};
 
 #[derive(Parser)]

Diff for: crates/oq3_lexer/src/lib.rs renamed to crates/lexer/src/lib.rs

@@ -40,7 +40,7 @@ int[64] x = 142;
     parse_print_items(code);
 }
 
-//use oq3_parser::syntax_kind::SyntaxKind;
+//use parser::syntax_kind::SyntaxKind;
 fn main () {
     //    parts_testing();
     try_int_def();