diff --git a/vhdl_lang/src/analysis/tests/mod.rs b/vhdl_lang/src/analysis/tests/mod.rs index 4b8789cc..cf463600 100644 --- a/vhdl_lang/src/analysis/tests/mod.rs +++ b/vhdl_lang/src/analysis/tests/mod.rs @@ -21,6 +21,7 @@ mod resolves_names; mod resolves_type_mark; mod sensitivity_list; mod subprogram_arguments; +mod tool_directive; mod typecheck_expression; mod util; mod visibility; diff --git a/vhdl_lang/src/analysis/tests/tool_directive.rs b/vhdl_lang/src/analysis/tests/tool_directive.rs new file mode 100644 index 00000000..debc1707 --- /dev/null +++ b/vhdl_lang/src/analysis/tests/tool_directive.rs @@ -0,0 +1,32 @@ +use crate::analysis::tests::{check_no_diagnostics, LibraryBuilder}; + +#[test] +fn simple_tool_directive() { + let mut builder = LibraryBuilder::new(); + builder.code("libname", "`protect begin"); + let (_, diagnostics) = builder.get_analyzed_root(); + + check_no_diagnostics(&diagnostics); +} + +#[test] +fn tool_directive() { + let mut builder = LibraryBuilder::new(); + builder.code( + "libname", + "\ +entity my_ent is +end my_ent; + +`protect begin_protected +`protect version = 1 +`protect encrypt_agent = \"XILINX\" +`protect encrypt_agent_info = \"Xilinx Encryption Tool 2020.2\" +`protect key_keyowner = \"Cadence Design Systems.\", key_keyname = \"cds_rsa_key\", key_method = \"rsa\" +`protect encoding = (enctype = \"BASE64\", line_length = 76, bytes = 64) + ", + ); + let (_, diagnostics) = builder.get_analyzed_root(); + + check_no_diagnostics(&diagnostics); +} diff --git a/vhdl_lang/src/syntax/test.rs b/vhdl_lang/src/syntax/test.rs index 727451c0..fdd11739 100644 --- a/vhdl_lang/src/syntax/test.rs +++ b/vhdl_lang/src/syntax/test.rs @@ -528,6 +528,11 @@ fn substr_range(source: &Source, range: Range, substr: &str, occurence: usize) - /// Fast forward tokenstream until position fn forward(stream: &TokenStream, start: Position) { + // short-circuit when start is zero. + // Also prevents the case where the token stream is empty + if start.line == 0 && start.character == 0 { + return; + } loop { let token = stream.peek_expect().unwrap(); if token.pos.start() >= start { diff --git a/vhdl_lang/src/syntax/tokens/tokenizer.rs b/vhdl_lang/src/syntax/tokens/tokenizer.rs index 883c1dfd..db209a0d 100644 --- a/vhdl_lang/src/syntax/tokens/tokenizer.rs +++ b/vhdl_lang/src/syntax/tokens/tokenizer.rs @@ -164,6 +164,8 @@ pub enum Kind { Comma, ColonEq, RightArrow, + GraveAccent, // ` + Text, // Raw text that is not processed (i.e. tokenized) further. Used in tool directives } use self::Kind::*; @@ -413,6 +415,8 @@ pub fn kind_str(kind: Kind) -> &'static str { Comma => ",", ColonEq => ":=", RightArrow => "=>", + GraveAccent => "`", + Text => "{text}", } } @@ -442,6 +446,8 @@ pub enum Value { BitString(ast::BitString), AbstractLiteral(ast::AbstractLiteral), Character(u8), + // Raw text that is not processed (i.e. tokenized) further. Used in tool directives + Text(Latin1String), NoValue, } @@ -468,6 +474,7 @@ pub struct Comment { } use std::convert::AsRef; + impl AsRef for Token { fn as_ref(&self) -> &SrcPos { &self.pos @@ -1149,6 +1156,25 @@ fn parse_character_literal( } } +/// Reads into `buffer` until a newline character is observed. +/// Does not consume the newline character. +/// +/// Clears the buffer prior to reading +fn read_until_newline( + buffer: &mut Latin1String, + reader: &mut ContentReader, +) -> Result<(), TokenError> { + buffer.bytes.clear(); + while let Some(b) = reader.peek()? { + if b == b'\n' { + break; + } + buffer.bytes.push(b); + reader.skip(); + } + Ok(()) +} + fn get_leading_comments(reader: &mut ContentReader) -> Result, TokenError> { let mut comments: Vec = Vec::new(); @@ -1703,6 +1729,10 @@ impl<'a> Tokenizer<'a> { let result = Value::Identifier(self.symbols.symtab().insert_extended(&result)); (Identifier, result) } + b'`' => { + self.reader.skip(); + (GraveAccent, Value::NoValue) + } _ => { self.reader.skip(); illegal_token!(); @@ -1764,6 +1794,25 @@ impl<'a> Tokenizer<'a> { pub fn get_final_comments(&self) -> Option> { self.final_comments.clone() } + + pub fn text_until_newline(&mut self) -> DiagnosticResult { + let start_pos = self.reader.pos(); + if let Err(err) = read_until_newline(&mut self.buffer, &mut self.reader) { + self.state.start = self.reader.state(); + return Err(Diagnostic::error( + &self.source.pos(err.range.start, err.range.end), + err.message, + )); + } + let text = self.buffer.clone(); + let end_pos = self.reader.pos(); + Ok(Token { + kind: Text, + value: Value::Text(text), + pos: self.source.pos(start_pos, end_pos), + comments: None, + }) + } } #[cfg(test)] diff --git a/vhdl_lang/src/syntax/tokens/tokenstream.rs b/vhdl_lang/src/syntax/tokens/tokenstream.rs index adc41f2d..06359db6 100644 --- a/vhdl_lang/src/syntax/tokens/tokenstream.rs +++ b/vhdl_lang/src/syntax/tokens/tokenstream.rs @@ -19,6 +19,37 @@ pub struct TokenStream<'a> { } impl<'a> TokenStream<'a> { + /// Special handling for a tool directive of the form + /// ```vhdl + /// `identifier { any chars until newline } + /// ``` + /// This needs special handling as the text that follows the identifier is arbitrary. + fn handle_tool_directive( + grave_accent: Token, + tokenizer: &mut Tokenizer, + diagnostics: &mut dyn DiagnosticHandler, + ) { + let start_pos = grave_accent.pos.clone(); + match tokenizer.pop() { + Ok(Some(tok)) => { + if tok.kind != Identifier { + diagnostics.error(tok, "Expecting identifier"); + let _ = tokenizer.text_until_newline(); // skip potentially invalid tokens + return; + } + } + Err(err) => diagnostics.push(err), + Ok(None) => { + diagnostics.error(start_pos, "Expecting identifier"); + return; + } + } + match tokenizer.text_until_newline() { + Ok(_) => {} + Err(err) => diagnostics.push(err), + } + } + pub fn new( mut tokenizer: Tokenizer<'a>, diagnostics: &mut dyn DiagnosticHandler, @@ -26,6 +57,9 @@ impl<'a> TokenStream<'a> { let mut tokens = Vec::new(); loop { match tokenizer.pop() { + Ok(Some(token)) if token.kind == GraveAccent => { + TokenStream::handle_tool_directive(token, &mut tokenizer, diagnostics) + } Ok(Some(token)) => tokens.push(token), Ok(None) => break, Err(err) => diagnostics.push(err), @@ -259,6 +293,12 @@ mod tests { let tokenizer = Tokenizer::new(&$code.symbols, source, ContentReader::new(&contents)); let $stream = TokenStream::new(tokenizer, &mut NoDiagnostics); }; + ($code:ident, $stream:ident, $diagnostics:ident) => { + let source = $code.source(); + let contents = source.contents(); + let tokenizer = Tokenizer::new(&$code.symbols, source, ContentReader::new(&contents)); + let $stream = TokenStream::new(tokenizer, &mut $diagnostics); + }; } #[test] @@ -388,4 +428,38 @@ mod tests { assert!(stream.skip_until(|ref k| matches!(k, Plus)).is_ok()); assert_eq!(stream.peek().map(|t| t.kind), Some(Plus)); } + + #[test] + fn tokenize_simple_identifier_directive() { + let code = Code::new("`protect begin"); + new_stream!(code, _stream); + } + + #[test] + fn tokenize_extended_identifier_directive() { + let code = Code::new("`\\extended ident\\ begin other words"); + new_stream!(code, _stream); + } + + #[test] + fn tokenize_directive_illegal_identifier() { + let code = Code::new("`123 begin other words"); + let mut diagnostics: Vec = vec![]; + new_stream!(code, _stream, diagnostics); + assert_eq!( + diagnostics, + vec![Diagnostic::error(code.s1("123"), "Expecting identifier")] + ) + } + + #[test] + fn tokenize_directive_then_end_of_stream() { + let code = Code::new("`"); + let mut diagnostics: Vec = vec![]; + new_stream!(code, _stream, diagnostics); + assert_eq!( + diagnostics, + vec![Diagnostic::error(code.s1("`"), "Expecting identifier")] + ) + } }