Merge pull request #188 from Schottkyc137/179-tool-directives

Feat: Tokenize tool directives

vhdl_lang/src/analysis/tests/mod.rs

@@ -21,6 +21,7 @@ mod resolves_names;
 mod resolves_type_mark;
 mod sensitivity_list;
 mod subprogram_arguments;
+mod tool_directive;
 mod typecheck_expression;
 mod util;
 mod visibility;

vhdl_lang/src/analysis/tests/tool_directive.rs

@@ -0,0 +1,32 @@
+use crate::analysis::tests::{check_no_diagnostics, LibraryBuilder};
+
+#[test]
+fn simple_tool_directive() {
+    let mut builder = LibraryBuilder::new();
+    builder.code("libname", "`protect begin");
+    let (_, diagnostics) = builder.get_analyzed_root();
+
+    check_no_diagnostics(&diagnostics);
+}
+
+#[test]
+fn tool_directive() {
+    let mut builder = LibraryBuilder::new();
+    builder.code(
+        "libname",
+        "\
+entity my_ent is
+end my_ent;
+
+`protect begin_protected
+`protect version = 1
+`protect encrypt_agent = \"XILINX\"
+`protect encrypt_agent_info = \"Xilinx Encryption Tool 2020.2\"
+`protect key_keyowner = \"Cadence Design Systems.\", key_keyname = \"cds_rsa_key\", key_method = \"rsa\"
+`protect encoding = (enctype = \"BASE64\", line_length = 76, bytes = 64)
+        ",
+    );
+    let (_, diagnostics) = builder.get_analyzed_root();
+
+    check_no_diagnostics(&diagnostics);
+}

vhdl_lang/src/syntax/test.rs

@@ -528,6 +528,11 @@ fn substr_range(source: &Source, range: Range, substr: &str, occurence: usize) -
 
 /// Fast forward tokenstream until position
 fn forward(stream: &TokenStream, start: Position) {
+    // short-circuit when start is zero.
+    // Also prevents the case where the token stream is empty
+    if start.line == 0 && start.character == 0 {
+        return;
+    }
     loop {
         let token = stream.peek_expect().unwrap();
         if token.pos.start() >= start {

vhdl_lang/src/syntax/tokens/tokenizer.rs

@@ -164,6 +164,8 @@ pub enum Kind {
     Comma,
     ColonEq,
     RightArrow,
+    GraveAccent, // `
+    Text,        // Raw text that is not processed (i.e. tokenized) further. Used in tool directives
 }
 use self::Kind::*;
 
@@ -413,6 +415,8 @@ pub fn kind_str(kind: Kind) -> &'static str {
         Comma => ",",
         ColonEq => ":=",
         RightArrow => "=>",
+        GraveAccent => "`",
+        Text => "{text}",
     }
 }
 
@@ -442,6 +446,8 @@ pub enum Value {
     BitString(ast::BitString),
     AbstractLiteral(ast::AbstractLiteral),
     Character(u8),
+    // Raw text that is not processed (i.e. tokenized) further. Used in tool directives
+    Text(Latin1String),
     NoValue,
 }
 
@@ -468,6 +474,7 @@ pub struct Comment {
 }
 
 use std::convert::AsRef;
+
 impl AsRef<SrcPos> for Token {
     fn as_ref(&self) -> &SrcPos {
         &self.pos
@@ -1149,6 +1156,25 @@ fn parse_character_literal(
     }
 }
 
+/// Reads into `buffer` until a newline character is observed.
+/// Does not consume the newline character.
+///
+/// Clears the buffer prior to reading
+fn read_until_newline(
+    buffer: &mut Latin1String,
+    reader: &mut ContentReader,
+) -> Result<(), TokenError> {
+    buffer.bytes.clear();
+    while let Some(b) = reader.peek()? {
+        if b == b'\n' {
+            break;
+        }
+        buffer.bytes.push(b);
+        reader.skip();
+    }
+    Ok(())
+}
+
 fn get_leading_comments(reader: &mut ContentReader) -> Result<Vec<Comment>, TokenError> {
     let mut comments: Vec<Comment> = Vec::new();
 
@@ -1703,6 +1729,10 @@ impl<'a> Tokenizer<'a> {
                 let result = Value::Identifier(self.symbols.symtab().insert_extended(&result));
                 (Identifier, result)
             }
+            b'`' => {
+                self.reader.skip();
+                (GraveAccent, Value::NoValue)
+            }
             _ => {
                 self.reader.skip();
                 illegal_token!();
@@ -1764,6 +1794,25 @@ impl<'a> Tokenizer<'a> {
     pub fn get_final_comments(&self) -> Option<Vec<Comment>> {
         self.final_comments.clone()
     }
+
+    pub fn text_until_newline(&mut self) -> DiagnosticResult<Token> {
+        let start_pos = self.reader.pos();
+        if let Err(err) = read_until_newline(&mut self.buffer, &mut self.reader) {
+            self.state.start = self.reader.state();
+            return Err(Diagnostic::error(
+                &self.source.pos(err.range.start, err.range.end),
+                err.message,
+            ));
+        }
+        let text = self.buffer.clone();
+        let end_pos = self.reader.pos();
+        Ok(Token {
+            kind: Text,
+            value: Value::Text(text),
+            pos: self.source.pos(start_pos, end_pos),
+            comments: None,
+        })
+    }
 }
 
 #[cfg(test)]

vhdl_lang/src/syntax/tokens/tokenstream.rs

@@ -19,13 +19,47 @@ pub struct TokenStream<'a> {
 }
 
 impl<'a> TokenStream<'a> {
+    /// Special handling for a tool directive of the form
+    /// ```vhdl
+    /// `identifier { any chars until newline }
+    /// ```
+    /// This needs special handling as the text that follows the identifier is arbitrary.
+    fn handle_tool_directive(
+        grave_accent: Token,
+        tokenizer: &mut Tokenizer,
+        diagnostics: &mut dyn DiagnosticHandler,
+    ) {
+        let start_pos = grave_accent.pos.clone();
+        match tokenizer.pop() {
+            Ok(Some(tok)) => {
+                if tok.kind != Identifier {
+                    diagnostics.error(tok, "Expecting identifier");
+                    let _ = tokenizer.text_until_newline(); // skip potentially invalid tokens
+                    return;
+                }
+            }
+            Err(err) => diagnostics.push(err),
+            Ok(None) => {
+                diagnostics.error(start_pos, "Expecting identifier");
+                return;
+            }
+        }
+        match tokenizer.text_until_newline() {
+            Ok(_) => {}
+            Err(err) => diagnostics.push(err),
+        }
+    }
+
     pub fn new(
         mut tokenizer: Tokenizer<'a>,
         diagnostics: &mut dyn DiagnosticHandler,
     ) -> TokenStream<'a> {
         let mut tokens = Vec::new();
         loop {
             match tokenizer.pop() {
+                Ok(Some(token)) if token.kind == GraveAccent => {
+                    TokenStream::handle_tool_directive(token, &mut tokenizer, diagnostics)
+                }
                 Ok(Some(token)) => tokens.push(token),
                 Ok(None) => break,
                 Err(err) => diagnostics.push(err),
@@ -259,6 +293,12 @@ mod tests {
             let tokenizer = Tokenizer::new(&$code.symbols, source, ContentReader::new(&contents));
             let $stream = TokenStream::new(tokenizer, &mut NoDiagnostics);
         };
+        ($code:ident, $stream:ident, $diagnostics:ident) => {
+            let source = $code.source();
+            let contents = source.contents();
+            let tokenizer = Tokenizer::new(&$code.symbols, source, ContentReader::new(&contents));
+            let $stream = TokenStream::new(tokenizer, &mut $diagnostics);
+        };
     }
 
     #[test]
@@ -388,4 +428,38 @@ mod tests {
         assert!(stream.skip_until(|ref k| matches!(k, Plus)).is_ok());
         assert_eq!(stream.peek().map(|t| t.kind), Some(Plus));
     }
+
+    #[test]
+    fn tokenize_simple_identifier_directive() {
+        let code = Code::new("`protect begin");
+        new_stream!(code, _stream);
+    }
+
+    #[test]
+    fn tokenize_extended_identifier_directive() {
+        let code = Code::new("`\\extended ident\\ begin other words");
+        new_stream!(code, _stream);
+    }
+
+    #[test]
+    fn tokenize_directive_illegal_identifier() {
+        let code = Code::new("`123 begin other words");
+        let mut diagnostics: Vec<Diagnostic> = vec![];
+        new_stream!(code, _stream, diagnostics);
+        assert_eq!(
+            diagnostics,
+            vec![Diagnostic::error(code.s1("123"), "Expecting identifier")]
+        )
+    }
+
+    #[test]
+    fn tokenize_directive_then_end_of_stream() {
+        let code = Code::new("`");
+        let mut diagnostics: Vec<Diagnostic> = vec![];
+        new_stream!(code, _stream, diagnostics);
+        assert_eq!(
+            diagnostics,
+            vec![Diagnostic::error(code.s1("`"), "Expecting identifier")]
+        )
+    }
 }