Add function call syntax for builtin functions

This commit paves the way to adding crypto primitives to the surface
language, such as sign(), for painlessly generating signed data, esp. for
testing X.509 parsers.

Author: mcy

cmd/ascii2der/builtins.go

@@ -0,0 +1,67 @@
+// Copyright 2015 The DER ASCII Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+)
+
+// NOTE: If adding a builtin, remember to document it in language.txt!
+
+var builtins = map[string]func(*scanner, [][]byte) ([]byte, error){
+	// define(var, val) sets var = val in the scanner's variable table.
+	// Variables may be redefined. Expands to the empty string.
+	"define": func(scanner *scanner, args [][]byte) ([]byte, error) {
+		if len(args) != 2 {
+			return nil, errors.New("expected two arguments to define()")
+		}
+		scanner.vars[string(args[0])] = args[1]
+		return nil, nil
+	},
+
+	// var(var) expands to whatever var is set to in the scanner's variable table.
+	// Error if var is not defined.
+	//
+	// var(var, default) behaves similarly, except expands to default if var is
+	// not defined.
+	"var": func(scanner *scanner, args [][]byte) ([]byte, error) {
+		switch len(args) {
+		case 1:
+			val, ok := scanner.vars[string(args[0])]
+			if !ok {
+				return nil, fmt.Errorf("var() with undefined name: %q", string(args[0]))
+			}
+			return val, nil
+		case 2:
+			val, ok := scanner.vars[string(args[0])]
+			if !ok {
+				return args[1], nil
+			}
+			return val, nil
+		default:
+			return nil, errors.New("expected one or two arguments to var()")
+		}
+	},
+}
+
+func executeBuiltin(scanner *scanner, name string, args [][]byte) ([]byte, error) {
+	builtin, ok := builtins[name]
+	if !ok {
+		return nil, fmt.Errorf("unrecognized builtin %q", name)
+	}
+
+	return builtin(scanner, args)
+}

cmd/ascii2der/scanner.go

@@ -43,6 +43,10 @@ const (
 	tokenRightCurly
 	tokenIndefinite
 	tokenLongForm
+	tokenComma
+	tokenLeftParen
+	tokenRightParen
+	tokenWord
 	tokenEOF
 )
 
@@ -78,10 +82,11 @@ var (
 type scanner struct {
 	text string
 	pos  position
+	vars map[string][]byte
 }
 
 func newScanner(text string) *scanner {
-	return &scanner{text: text, pos: position{Line: 1}}
+	return &scanner{text: text, pos: position{Line: 1}, vars: make(map[string][]byte)}
 }
 
 func (s *scanner) parseEscapeSequence() (rune, error) {
@@ -274,6 +279,15 @@ again:
 	case '}':
 		s.advance()
 		return token{Kind: tokenRightCurly, Pos: s.pos}, nil
+	case ',':
+		s.advance()
+		return token{Kind: tokenComma, Pos: s.pos}, nil
+	case '(':
+		s.advance()
+		return token{Kind: tokenLeftParen, Pos: s.pos}, nil
+	case ')':
+		s.advance()
+		return token{Kind: tokenRightParen, Pos: s.pos}, nil
 	case '"':
 		return s.parseQuotedString()
 	case 'u':
@@ -366,7 +380,7 @@ again:
 loop:
 	for !s.isEOF() {
 		switch s.text[s.pos.Offset] {
-		case ' ', '\t', '\n', '\r', '{', '}', '[', ']', '`', '"', '#':
+		case ' ', '\t', '\n', '\r', ',', '(', ')', '{', '}', '[', ']', '`', '"', '#':
 			break loop
 		default:
 			s.advance()
@@ -431,7 +445,7 @@ loop:
 		return token{Kind: tokenLongForm, Length: l}, nil
 	}
 
-	return token{}, fmt.Errorf("unrecognized symbol %q", symbol)
+	return token{Kind: tokenWord, Value: []byte(symbol), Pos: s.pos}, nil
 }
 
 func (s *scanner) isEOF() bool {
@@ -469,24 +483,32 @@ func (s *scanner) consumeUpTo(b byte) (string, bool) {
 	return "", false
 }
 
-func asciiToDERImpl(scanner *scanner, leftCurly *token) ([]byte, error) {
+// asciiToDERImpl consumes tokens from |scanner| and returns the concatenation
+// of their byte values, as well as every token processed.
+func asciiToDERImpl(scanner *scanner, left *token) ([]byte, []token, error) {
 	var out []byte
+	var tokens []token
 	var lengthModifier *token
+	var word *token
 	for {
 		token, err := scanner.Next()
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
+		tokens = append(tokens, token)
 		if lengthModifier != nil && token.Kind != tokenLeftCurly {
-			return nil, &parseError{lengthModifier.Pos, errors.New("length modifier was not followed by '{'")}
+			return nil, nil, &parseError{lengthModifier.Pos, errors.New("length modifier was not followed by '{'")}
+		}
+		if word != nil && token.Kind != tokenLeftParen {
+			return nil, nil, &parseError{word.Pos, fmt.Errorf("unrecognized symbol %q", string(token.Value))}
 		}
 		switch token.Kind {
 		case tokenBytes:
 			out = append(out, token.Value...)
 		case tokenLeftCurly:
-			child, err := asciiToDERImpl(scanner, &token)
+			child, _, err := asciiToDERImpl(scanner, &token)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			var lengthOverride int
 			if lengthModifier != nil {
@@ -504,22 +526,68 @@ func asciiToDERImpl(scanner *scanner, leftCurly *token) ([]byte, error) {
 			out, err = appendLength(out, len(child), lengthOverride)
 			if err != nil {
 				// appendLength may fail if the lengthModifier was incompatible.
-				return nil, &parseError{lengthModifier.Pos, err}
+				return nil, tokens, &parseError{lengthModifier.Pos, err}
 			}
 			out = append(out, child...)
 			lengthModifier = nil
+		case tokenLeftParen:
+			if word == nil {
+				return nil, tokens, &parseError{token.Pos, errors.New("missing function name")}
+			}
+			var args [][]byte
+		argLoop:
+			for {
+				arg, prev, err := asciiToDERImpl(scanner, &token)
+				if err != nil {
+					return nil, tokens, err
+				}
+				args = append(args, arg)
+				lastToken := prev[len(prev)-1]
+				switch lastToken.Kind {
+				case tokenComma:
+					if len(prev) < 2 {
+						return nil, nil, &parseError{lastToken.Pos, errors.New("function arguments cannot be empty")}
+					}
+				case tokenRightParen:
+					if len(prev) < 2 {
+						// Actually foo(), so the argument list is nil.
+						args = nil
+					}
+					break argLoop
+				default:
+					return nil, nil, &parseError{lastToken.Pos, errors.New("expected ',' or ')'")}
+				}
+			}
+			bytes, err := executeBuiltin(scanner, string(word.Value), args)
+			if err != nil {
+				return nil, nil, err
+			}
+			word = nil
+			out = append(out, bytes...)
 		case tokenRightCurly:
-			if leftCurly != nil {
-				return out, nil
+			if left != nil && left.Kind == tokenLeftCurly {
+				return out, tokens, nil
+			}
+			return nil, nil, &parseError{token.Pos, errors.New("unmatched '}'")}
+		case tokenRightParen:
+			if left != nil && left.Kind == tokenLeftParen {
+				return out, tokens, nil
 			}
-			return nil, &parseError{token.Pos, errors.New("unmatched '}'")}
+			return nil, nil, &parseError{token.Pos, errors.New("unmatched '('")}
 		case tokenLongForm, tokenIndefinite:
 			lengthModifier = &token
+		case tokenComma:
+			return out, tokens, nil
+		case tokenWord:
+			word = &token
 		case tokenEOF:
-			if leftCurly == nil {
-				return out, nil
+			if left == nil {
+				return out, tokens, nil
+			} else if left.Kind == tokenLeftCurly {
+				return nil, nil, &parseError{left.Pos, errors.New("unmatched '{'")}
+			} else {
+				return nil, nil, &parseError{left.Pos, errors.New("unmatched '('")}
 			}
-			return nil, &parseError{leftCurly.Pos, errors.New("unmatched '{'")}
 		default:
 			panic(token)
 		}
@@ -528,5 +596,6 @@ func asciiToDERImpl(scanner *scanner, leftCurly *token) ([]byte, error) {
 
 func asciiToDER(input string) ([]byte, error) {
 	scanner := newScanner(input)
-	return asciiToDERImpl(scanner, nil)
+	bytes, _, err := asciiToDERImpl(scanner, nil)
+	return bytes, err
 }

cmd/ascii2der/scanner_test.go

@@ -94,11 +94,6 @@ indefinite long-form:2`,
 		},
 		true,
 	},
-	// Garbage tokens.
-	{"SEQUENC", nil, false},
-	{"1...2", nil, false},
-	{"true", nil, false},
-	{"false", nil, false},
 	// Unmatched [.
 	{"[SEQUENCE", nil, false},
 	// Unmatched ".
@@ -426,9 +421,18 @@ var asciiToDERTests = []struct {
 	ok  bool
 }{
 	{"SEQUENCE { INTEGER { 42 } INTEGER { 1 } }", []byte{0x30, 0x06, 0x02, 0x01, 0x2a, 0x02, 0x01, 0x01}, true},
+	// Garbage words.
+	{"SEQUENC", nil, false},
+	{"1...2", nil, false},
+	{"true", nil, false},
+	{"false", nil, false},
 	// Mismatched curlies.
 	{"{", nil, false},
 	{"}", nil, false},
+	{"(", nil, false},
+	{")", nil, false},
+	{"({)}", nil, false},
+	{"{(})", nil, false},
 	// Invalid token.
 	{"BOGUS", nil, false},
 	// Length overrides.
@@ -442,6 +446,28 @@ var asciiToDERTests = []struct {
 	// Too long of length modifiers.
 	{"[long-form:1 99999]", nil, false},
 	{"SEQUENCE long-form:1 { `" + strings.Repeat("a", 1024) + "` }", nil, false},
+	// Function call without function name.
+	{"()", nil, false},
+	// Unknown function.
+	{"bogus()", nil, false},
+	// Basic variable usage.
+	{`define("foo", 42) var("foo") var("foo")`, []byte{42, 42}, true},
+	{
+		`
+		define(42, 42) var(42)
+		define(42, "a") var(42)
+		`,
+		[]byte{42, byte('a')},
+		true,
+	},
+	{`var("missing")`, nil, false},
+	{`var("missing", 42)`, []byte{42}, true},
+	// Empty parens -> zero args.
+	// TODO(mcyoung): if we ever add a zero-argument function, use it in this
+	// test, instead.
+	{"var()", nil, false},
+	// Empty token streams are not valid arguments.
+	{"define(, 42)", nil, false},
 }
 
 func TestASCIIToDER(t *testing.T) {

language.txt

@@ -151,7 +151,7 @@ FALSE
 [PRIVATE 2]
 [UNIVERSAL 16] # This is a SEQUENCE.
 [UNIVERSAL 2 PRIMITIVE] # This is an INTEGER.
-[long-form:2 UNIVERSAL 2 PRIMTIVE] # This is `1f0002` instead of `02`.
+[long-form:2 UNIVERSAL 2 PRIMITIVE] # This is `1f0002` instead of `02`.
 
 # As a shorthand, one may write type names from ASN.1, replacing spaces with
 # underscore. These specify tag, number, and the constructed bit. The
@@ -236,7 +236,6 @@ INTEGER long-form:1 { 5 }
   INTEGER { `00ff` }
 }
 
-
 # Examples.
 
 # These primitives may be combined with raw byte strings to produce other
@@ -278,6 +277,35 @@ SEQUENCE `aabbcc`
   INTEGER { 2 }
 
 
+# DER ASCII provides a selection of builtin functions for building more
+# complex DER structures. Builtins are spelled builtin(arg1, arg2, ...), where
+# each argument is an arbitrary but non-empty sequence of DER ASCII tokens.
+# Tokens are fully expanded (but not emitted) before the builtin executes.
+# Builtin calls expand to a byte string just like any other token.
+#
+# There is explicitly no support for user-defined functions.
+#
+# The syntax and output of anything using builtins is subject to change, and
+# doesn't have the same stability expectations as the rest of DER ASCII.
+
+# define(var, value) defines a variable to the given value. The name of the
+# variable may be an arbitrary byte string. Variables may be redefined at any
+# point. define() always expands to an empty byte string.
+define("payload", SEQUENCE {
+  INTEGER { 1 }
+  OCTET_STRING { "hello, world" }
+})
+define(`ffff`, "payload")
+
+# var(var) expands to a variable previously defined with define(). The main use
+# of var() is to factor a complex structure that will be repeated many times,
+# such as complex issuer and subject fields in a self-signed X.509 cert.
+#
+# It is an error to access a var() that has not been previously defined.
+var("payload")
+var(var(`ffff`)) # Same as above, since var(`ffff`) expands to "payload".
+
+
 # Disassembler.
 
 # Although the conversion from DER ASCII to a byte string is well-defined, the