Курнат Андрей
470 lines
13 KiB
Go
470 lines
13 KiB
Go
package ast
|
|
|
|
import (
|
|
"bytes"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
type grammarOptimizer struct {
|
|
rule string
|
|
protectedRules map[string]struct{}
|
|
rules map[string]*Rule
|
|
ruleUsesRules map[string]map[string]struct{}
|
|
ruleUsedByRules map[string]map[string]struct{}
|
|
visitor func(expr Expression) Visitor
|
|
optimized bool
|
|
}
|
|
|
|
func newGrammarOptimizer(protectedRules []string) *grammarOptimizer {
|
|
pr := make(map[string]struct{}, len(protectedRules))
|
|
for _, nm := range protectedRules {
|
|
pr[nm] = struct{}{}
|
|
}
|
|
|
|
r := grammarOptimizer{
|
|
protectedRules: pr,
|
|
rules: make(map[string]*Rule),
|
|
ruleUsesRules: make(map[string]map[string]struct{}),
|
|
ruleUsedByRules: make(map[string]map[string]struct{}),
|
|
}
|
|
r.visitor = r.init
|
|
return &r
|
|
}
|
|
|
|
// Visit is a generic Visitor to be used with Walk
|
|
// The actual function, which should be used during Walk
|
|
// is held in ruleRefOptimizer.visitor.
|
|
func (r *grammarOptimizer) Visit(expr Expression) Visitor {
|
|
return r.visitor(expr)
|
|
}
|
|
|
|
// init is a Visitor, which is used with the Walk function
|
|
// The purpose of this function is to initialize the reference
|
|
// maps rules, ruleUsesRules and ruleUsedByRules.
|
|
func (r *grammarOptimizer) init(expr Expression) Visitor {
|
|
switch expr := expr.(type) {
|
|
case *Rule:
|
|
// Keep track of current rule, which is processed
|
|
r.rule = expr.Name.Val
|
|
r.rules[expr.Name.Val] = expr
|
|
case *RuleRefExpr:
|
|
// Fill ruleUsesRules and ruleUsedByRules for every RuleRefExpr
|
|
set(r.ruleUsesRules, r.rule, expr.Name.Val)
|
|
set(r.ruleUsedByRules, expr.Name.Val, r.rule)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// Add element to map of maps, initialize the inner map
|
|
// if necessary.
|
|
func set(m map[string]map[string]struct{}, src, dst string) {
|
|
if _, ok := m[src]; !ok {
|
|
m[src] = make(map[string]struct{})
|
|
}
|
|
m[src][dst] = struct{}{}
|
|
}
|
|
|
|
// optimize is a Visitor, which is used with the Walk function
|
|
// The purpose of this function is to perform the actual optimizations.
|
|
// See Optimize for a detailed list of the performed optimizations.
|
|
func (r *grammarOptimizer) optimize(expr0 Expression) Visitor {
|
|
switch expr := expr0.(type) {
|
|
case *ActionExpr:
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
case *AndExpr:
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
case *ChoiceExpr:
|
|
expr.Alternatives = r.optimizeRules(expr.Alternatives)
|
|
|
|
// Optimize choice nested in choice
|
|
for i := 0; i < len(expr.Alternatives); i++ {
|
|
if choice, ok := expr.Alternatives[i].(*ChoiceExpr); ok {
|
|
r.optimized = true
|
|
if i+1 < len(expr.Alternatives) {
|
|
expr.Alternatives = append(expr.Alternatives[:i], append(choice.Alternatives, expr.Alternatives[i+1:]...)...)
|
|
} else {
|
|
expr.Alternatives = append(expr.Alternatives[:i], choice.Alternatives...)
|
|
}
|
|
}
|
|
|
|
// Combine sequence of single char LitMatcher to CharClassMatcher
|
|
if i > 0 {
|
|
l0, lok0 := expr.Alternatives[i-1].(*LitMatcher)
|
|
l1, lok1 := expr.Alternatives[i].(*LitMatcher)
|
|
c0, cok0 := expr.Alternatives[i-1].(*CharClassMatcher)
|
|
c1, cok1 := expr.Alternatives[i].(*CharClassMatcher)
|
|
|
|
combined := false
|
|
|
|
switch {
|
|
// Combine two LitMatcher to CharClassMatcher
|
|
// "a" / "b" => [ab]
|
|
case lok0 && lok1 && len([]rune(l0.Val)) == 1 && len([]rune(l1.Val)) == 1 && l0.IgnoreCase == l1.IgnoreCase:
|
|
combined = true
|
|
cm := CharClassMatcher{
|
|
Chars: append([]rune(l0.Val), []rune(l1.Val)...),
|
|
IgnoreCase: l0.IgnoreCase,
|
|
posValue: l0.posValue,
|
|
}
|
|
expr.Alternatives[i-1] = &cm
|
|
|
|
// Combine LitMatcher with CharClassMatcher
|
|
// "a" / [bc] => [abc]
|
|
case lok0 && cok1 && len([]rune(l0.Val)) == 1 && l0.IgnoreCase == c1.IgnoreCase && !c1.Inverted:
|
|
combined = true
|
|
c1.Chars = append(c1.Chars, []rune(l0.Val)...)
|
|
expr.Alternatives[i-1] = c1
|
|
|
|
// Combine CharClassMatcher with LitMatcher
|
|
// [ab] / "c" => [abc]
|
|
case cok0 && lok1 && len([]rune(l1.Val)) == 1 && c0.IgnoreCase == l1.IgnoreCase && !c0.Inverted:
|
|
combined = true
|
|
c0.Chars = append(c0.Chars, []rune(l1.Val)...)
|
|
|
|
// Combine CharClassMatcher with CharClassMatcher
|
|
// [ab] / [cd] => [abcd]
|
|
case cok0 && cok1 && c0.IgnoreCase == c1.IgnoreCase && c0.Inverted == c1.Inverted:
|
|
combined = true
|
|
c0.Chars = append(c0.Chars, c1.Chars...)
|
|
c0.Ranges = append(c0.Ranges, c1.Ranges...)
|
|
c0.UnicodeClasses = append(c0.UnicodeClasses, c1.UnicodeClasses...)
|
|
}
|
|
|
|
// If one of the optimizations was applied, remove the second element from Alternatives
|
|
if combined {
|
|
r.optimized = true
|
|
if i+1 < len(expr.Alternatives) {
|
|
expr.Alternatives = append(expr.Alternatives[:i], expr.Alternatives[i+1:]...)
|
|
} else {
|
|
expr.Alternatives = expr.Alternatives[:i]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
case *Grammar:
|
|
// Reset optimized at the start of each Walk.
|
|
r.optimized = false
|
|
for i := 0; i < len(expr.Rules); i++ {
|
|
rule := expr.Rules[i]
|
|
// Remove Rule, if it is no longer used by any other Rule and it is not the first Rule.
|
|
_, used := r.ruleUsedByRules[rule.Name.Val]
|
|
_, protected := r.protectedRules[rule.Name.Val]
|
|
if !used && !protected {
|
|
expr.Rules = append(expr.Rules[:i], expr.Rules[i+1:]...)
|
|
// Compensate for the removed item
|
|
i--
|
|
|
|
for k, v := range r.ruleUsedByRules {
|
|
for kk := range v {
|
|
if kk == rule.Name.Val {
|
|
delete(r.ruleUsedByRules[k], kk)
|
|
if len(r.ruleUsedByRules[k]) == 0 {
|
|
delete(r.ruleUsedByRules, k)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
r.optimized = true
|
|
continue
|
|
}
|
|
}
|
|
case *LabeledExpr:
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
case *NotExpr:
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
case *OneOrMoreExpr:
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
case *Rule:
|
|
r.rule = expr.Name.Val
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
case *SeqExpr:
|
|
expr.Exprs = r.optimizeRules(expr.Exprs)
|
|
|
|
for i := 0; i < len(expr.Exprs); i++ {
|
|
// Optimize nested sequences
|
|
if seq, ok := expr.Exprs[i].(*SeqExpr); ok {
|
|
r.optimized = true
|
|
if i+1 < len(expr.Exprs) {
|
|
expr.Exprs = append(expr.Exprs[:i], append(seq.Exprs, expr.Exprs[i+1:]...)...)
|
|
} else {
|
|
expr.Exprs = append(expr.Exprs[:i], seq.Exprs...)
|
|
}
|
|
}
|
|
|
|
// Combine sequence of LitMatcher
|
|
if i > 0 {
|
|
l0, ok0 := expr.Exprs[i-1].(*LitMatcher)
|
|
l1, ok1 := expr.Exprs[i].(*LitMatcher)
|
|
if ok0 && ok1 && l0.IgnoreCase == l1.IgnoreCase {
|
|
r.optimized = true
|
|
l0.Val += l1.Val
|
|
expr.Exprs[i-1] = l0
|
|
if i+1 < len(expr.Exprs) {
|
|
expr.Exprs = append(expr.Exprs[:i], expr.Exprs[i+1:]...)
|
|
} else {
|
|
expr.Exprs = expr.Exprs[:i]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
case *ZeroOrMoreExpr:
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
case *ZeroOrOneExpr:
|
|
expr.Expr = r.optimizeRule(expr.Expr)
|
|
}
|
|
return r
|
|
}
|
|
|
|
func (r *grammarOptimizer) optimizeRules(exprs []Expression) []Expression {
|
|
for i := 0; i < len(exprs); i++ {
|
|
exprs[i] = r.optimizeRule(exprs[i])
|
|
}
|
|
return exprs
|
|
}
|
|
|
|
func (r *grammarOptimizer) optimizeRule(expr Expression) Expression {
|
|
// Optimize RuleRefExpr
|
|
if ruleRef, ok := expr.(*RuleRefExpr); ok {
|
|
if _, ok := r.ruleUsesRules[ruleRef.Name.Val]; !ok {
|
|
r.optimized = true
|
|
delete(r.ruleUsedByRules[ruleRef.Name.Val], r.rule)
|
|
if len(r.ruleUsedByRules[ruleRef.Name.Val]) == 0 {
|
|
delete(r.ruleUsedByRules, ruleRef.Name.Val)
|
|
}
|
|
delete(r.ruleUsesRules[r.rule], ruleRef.Name.Val)
|
|
if len(r.ruleUsesRules[r.rule]) == 0 {
|
|
delete(r.ruleUsesRules, r.rule)
|
|
}
|
|
// TODO: Check if reference exists, otherwise raise an error, which reference is missing!
|
|
return cloneExpr(r.rules[ruleRef.Name.Val].Expr)
|
|
}
|
|
}
|
|
|
|
// Remove Choices with only one Alternative left
|
|
if choice, ok := expr.(*ChoiceExpr); ok {
|
|
if len(choice.Alternatives) == 1 {
|
|
r.optimized = true
|
|
return choice.Alternatives[0]
|
|
}
|
|
}
|
|
|
|
// Remove Sequence with only one Expression
|
|
if seq, ok := expr.(*SeqExpr); ok {
|
|
if len(seq.Exprs) == 1 {
|
|
r.optimized = true
|
|
return seq.Exprs[0]
|
|
}
|
|
}
|
|
|
|
return expr
|
|
}
|
|
|
|
// cloneExpr takes an Expression and deep clones it (including all children)
|
|
// This is necessary because referenced Rules are denormalized and therefore
|
|
// have to become independent from their original Expression.
|
|
func cloneExpr(expr Expression) Expression {
|
|
switch expr := expr.(type) {
|
|
case *ActionExpr:
|
|
return &ActionExpr{
|
|
Code: expr.Code,
|
|
Expr: cloneExpr(expr.Expr),
|
|
FuncIx: expr.FuncIx,
|
|
p: expr.p,
|
|
}
|
|
case *AndExpr:
|
|
return &AndExpr{
|
|
Expr: cloneExpr(expr.Expr),
|
|
p: expr.p,
|
|
}
|
|
case *AndCodeExpr:
|
|
return &AndCodeExpr{
|
|
Code: expr.Code,
|
|
FuncIx: expr.FuncIx,
|
|
p: expr.p,
|
|
}
|
|
case *CharClassMatcher:
|
|
return &CharClassMatcher{
|
|
Chars: append([]rune{}, expr.Chars...),
|
|
IgnoreCase: expr.IgnoreCase,
|
|
Inverted: expr.Inverted,
|
|
posValue: expr.posValue,
|
|
Ranges: append([]rune{}, expr.Ranges...),
|
|
UnicodeClasses: append([]string{}, expr.UnicodeClasses...),
|
|
}
|
|
case *ChoiceExpr:
|
|
alts := make([]Expression, 0, len(expr.Alternatives))
|
|
for i := 0; i < len(expr.Alternatives); i++ {
|
|
alts = append(alts, cloneExpr(expr.Alternatives[i]))
|
|
}
|
|
return &ChoiceExpr{
|
|
Alternatives: alts,
|
|
p: expr.p,
|
|
}
|
|
case *LabeledExpr:
|
|
return &LabeledExpr{
|
|
Expr: cloneExpr(expr.Expr),
|
|
Label: expr.Label,
|
|
p: expr.p,
|
|
}
|
|
case *NotExpr:
|
|
return &NotExpr{
|
|
Expr: cloneExpr(expr.Expr),
|
|
p: expr.p,
|
|
}
|
|
case *NotCodeExpr:
|
|
return &NotCodeExpr{
|
|
Code: expr.Code,
|
|
FuncIx: expr.FuncIx,
|
|
p: expr.p,
|
|
}
|
|
case *OneOrMoreExpr:
|
|
return &OneOrMoreExpr{
|
|
Expr: cloneExpr(expr.Expr),
|
|
p: expr.p,
|
|
}
|
|
case *SeqExpr:
|
|
exprs := make([]Expression, 0, len(expr.Exprs))
|
|
for i := 0; i < len(expr.Exprs); i++ {
|
|
exprs = append(exprs, cloneExpr(expr.Exprs[i]))
|
|
}
|
|
return &SeqExpr{
|
|
Exprs: exprs,
|
|
p: expr.p,
|
|
}
|
|
case *StateCodeExpr:
|
|
return &StateCodeExpr{
|
|
p: expr.p,
|
|
Code: expr.Code,
|
|
FuncIx: expr.FuncIx,
|
|
}
|
|
case *ZeroOrMoreExpr:
|
|
return &ZeroOrMoreExpr{
|
|
Expr: cloneExpr(expr.Expr),
|
|
p: expr.p,
|
|
}
|
|
case *ZeroOrOneExpr:
|
|
return &ZeroOrOneExpr{
|
|
Expr: cloneExpr(expr.Expr),
|
|
p: expr.p,
|
|
}
|
|
}
|
|
return expr
|
|
}
|
|
|
|
// cleanupCharClassMatcher is a Visitor, which is used with the Walk function
|
|
// The purpose of this function is to cleanup the redundancies created by the
|
|
// optimize Visitor. This includes to remove redundant entries in Chars, Ranges
|
|
// and UnicodeClasses of the given CharClassMatcher as well as regenerating the
|
|
// correct content for the Val field (string representation of the CharClassMatcher).
|
|
func (r *grammarOptimizer) cleanupCharClassMatcher(expr0 Expression) Visitor {
|
|
// We are only interested in nodes of type *CharClassMatcher
|
|
if chr, ok := expr0.(*CharClassMatcher); ok {
|
|
// Remove redundancies in Chars
|
|
chars := make([]rune, 0, len(chr.Chars))
|
|
charsMap := make(map[rune]struct{})
|
|
for _, c := range chr.Chars {
|
|
if _, ok := charsMap[c]; !ok {
|
|
charsMap[c] = struct{}{}
|
|
chars = append(chars, c)
|
|
}
|
|
}
|
|
if len(chars) > 0 {
|
|
chr.Chars = chars
|
|
} else {
|
|
chr.Chars = nil
|
|
}
|
|
|
|
// Remove redundancies in Ranges
|
|
ranges := make([]rune, 0, len(chr.Ranges))
|
|
rangesMap := make(map[string]struct{})
|
|
for i := 0; i < len(chr.Ranges); i += 2 {
|
|
rangeKey := string(chr.Ranges[i]) + "-" + string(chr.Ranges[i+1])
|
|
if _, ok := rangesMap[rangeKey]; !ok {
|
|
rangesMap[rangeKey] = struct{}{}
|
|
ranges = append(ranges, chr.Ranges[i], chr.Ranges[i+1])
|
|
}
|
|
}
|
|
if len(ranges) > 0 {
|
|
chr.Ranges = ranges
|
|
} else {
|
|
chr.Ranges = nil
|
|
}
|
|
|
|
// Remove redundancies in UnicodeClasses
|
|
unicodeClasses := make([]string, 0, len(chr.UnicodeClasses))
|
|
unicodeClassesMap := make(map[string]struct{})
|
|
for _, u := range chr.UnicodeClasses {
|
|
if _, ok := unicodeClassesMap[u]; !ok {
|
|
unicodeClassesMap[u] = struct{}{}
|
|
unicodeClasses = append(unicodeClasses, u)
|
|
}
|
|
}
|
|
if len(unicodeClasses) > 0 {
|
|
chr.UnicodeClasses = unicodeClasses
|
|
} else {
|
|
chr.UnicodeClasses = nil
|
|
}
|
|
|
|
// Regenerate the content for Val
|
|
var val bytes.Buffer
|
|
val.WriteString("[")
|
|
if chr.Inverted {
|
|
val.WriteString("^")
|
|
}
|
|
for _, c := range chr.Chars {
|
|
val.WriteString(escapeRune(c))
|
|
}
|
|
for i := 0; i < len(chr.Ranges); i += 2 {
|
|
val.WriteString(escapeRune(chr.Ranges[i]))
|
|
val.WriteString("-")
|
|
val.WriteString(escapeRune(chr.Ranges[i+1]))
|
|
}
|
|
for _, u := range chr.UnicodeClasses {
|
|
val.WriteString("\\p" + u)
|
|
}
|
|
val.WriteString("]")
|
|
if chr.IgnoreCase {
|
|
val.WriteString("i")
|
|
}
|
|
chr.posValue.Val = val.String()
|
|
}
|
|
return r
|
|
}
|
|
|
|
func escapeRune(r rune) string {
|
|
return strings.Trim(strconv.QuoteRune(r), `'`)
|
|
}
|
|
|
|
// Optimize walks a given grammar and optimizes the grammar in regards
|
|
// of parsing performance. This is done with several optimizations:
|
|
// - removal of unreferenced rules
|
|
// - replace rule references with a copy of the referenced Rule, if the
|
|
// referenced rule it self has no references.
|
|
// - resolve nested choice expressions
|
|
// - resolve choice expressions with only one alternative
|
|
// - resolve nested sequences expression
|
|
// - resolve sequence expressions with only one element
|
|
// - combine character class matcher and literal matcher, where possible
|
|
func Optimize(g *Grammar, alternateEntrypoints ...string) {
|
|
entrypoints := alternateEntrypoints
|
|
if len(g.Rules) > 0 {
|
|
entrypoints = append(entrypoints, g.Rules[0].Name.Val)
|
|
}
|
|
|
|
r := newGrammarOptimizer(entrypoints)
|
|
Walk(r, g)
|
|
|
|
r.visitor = r.optimize
|
|
r.optimized = true
|
|
for r.optimized {
|
|
Walk(r, g)
|
|
}
|
|
|
|
r.visitor = r.cleanupCharClassMatcher
|
|
Walk(r, g)
|
|
}
|