Initial QSfera import
This commit is contained in:
Курнат Андрей
+96
@@ -0,0 +1,96 @@
|
||||
An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode 17.
|
||||
|
||||
[](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
|
||||
|
||||
|
||||
|
||||
## Quick start
|
||||
|
||||
```
|
||||
go get "github.com/clipperhouse/uax29/v2/graphemes"
|
||||
```
|
||||
|
||||
```go
|
||||
import "github.com/clipperhouse/uax29/v2/graphemes"
|
||||
|
||||
text := "Hello, 世界. Nice dog! 👍🐶"
|
||||
|
||||
tokens := graphemes.FromString(text)
|
||||
|
||||
for tokens.Next() { // Next() returns true until end of data
|
||||
fmt.Println(tokens.Value()) // Do something with the current grapheme
|
||||
}
|
||||
```
|
||||
|
||||
_A grapheme is a “single visible character”, which might be a simple as a single letter, or a complex emoji that consists of several Unicode code points._
|
||||
|
||||
## Conformance
|
||||
|
||||
We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-36.html#Tests29).
|
||||
|
||||
|
||||
|
||||
|
||||
## APIs
|
||||
|
||||
### If you have a `string`
|
||||
|
||||
```go
|
||||
text := "Hello, 世界. Nice dog! 👍🐶"
|
||||
|
||||
tokens := graphemes.FromString(text)
|
||||
|
||||
for tokens.Next() { // Next() returns true until end of data
|
||||
fmt.Println(tokens.Value()) // Do something with the current grapheme
|
||||
}
|
||||
```
|
||||
|
||||
### If you have an `io.Reader`
|
||||
|
||||
`FromReader` embeds a [`bufio.Scanner`](https://pkg.go.dev/bufio#Scanner), so just use those methods.
|
||||
|
||||
```go
|
||||
r := getYourReader() // from a file or network maybe
|
||||
tokens := graphemes.FromReader(r)
|
||||
|
||||
for tokens.Scan() { // Scan() returns true until error or EOF
|
||||
fmt.Println(tokens.Text()) // Do something with the current grapheme
|
||||
}
|
||||
|
||||
if tokens.Err() != nil { // Check the error
|
||||
log.Fatal(tokens.Err())
|
||||
}
|
||||
```
|
||||
|
||||
### If you have a `[]byte`
|
||||
|
||||
```go
|
||||
b := []byte("Hello, 世界. Nice dog! 👍🐶")
|
||||
|
||||
tokens := graphemes.FromBytes(b)
|
||||
|
||||
for tokens.Next() { // Next() returns true until end of data
|
||||
fmt.Println(tokens.Value()) // Do something with the current grapheme
|
||||
}
|
||||
```
|
||||
|
||||
### Benchmarks
|
||||
|
||||
```
|
||||
goos: darwin
|
||||
goarch: arm64
|
||||
pkg: github.com/clipperhouse/uax29/graphemes/comparative
|
||||
cpu: Apple M2
|
||||
|
||||
BenchmarkGraphemesMixed/clipperhouse/uax29-8 142635 ns/op 245.12 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkGraphemesMixed/rivo/uniseg-8 2018284 ns/op 17.32 MB/s 0 B/op 0 allocs/op
|
||||
|
||||
BenchmarkGraphemesASCII/clipperhouse/uax29-8 8846 ns/op 508.73 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkGraphemesASCII/rivo/uniseg-8 366760 ns/op 12.27 MB/s 0 B/op 0 allocs/op
|
||||
```
|
||||
|
||||
### Invalid inputs
|
||||
|
||||
Invalid UTF-8 input is considered undefined behavior. We test to ensure that bad inputs will not cause pathological outcomes, such as a panic or infinite loop. Callers should expect “garbage-in, garbage-out”.
|
||||
|
||||
Your pipeline should probably include a call to [`utf8.Valid()`](https://pkg.go.dev/unicode/utf8#Valid).
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
package graphemes
|
||||
|
||||
// ansiEscapeLength returns the byte length of a valid ANSI escape sequence at the
|
||||
// start of data, or 0 if none. Input is UTF-8; only 7-bit ESC sequences are
|
||||
// recognized (C1 0x80–0x9F can be UTF-8 continuation bytes).
|
||||
//
|
||||
// Recognized forms (ECMA-48 / ISO 6429):
|
||||
// - CSI: ESC [ then parameter bytes (0x30–0x3F), intermediate (0x20–0x2F), final (0x40–0x7E)
|
||||
// - OSC: ESC ] then payload until ST (ESC \) or BEL (0x07)
|
||||
// - DCS, SOS, PM, APC: ESC P / X / ^ / _ then payload until ST (ESC \)
|
||||
// - Two-byte: ESC + Fe (0x40–0x5F excluding above), or Fp (0x30–0x3F), or nF (0x20–0x2F then final)
|
||||
func ansiEscapeLength[T ~string | ~[]byte](data T) int {
|
||||
n := len(data)
|
||||
if n < 2 {
|
||||
return 0
|
||||
}
|
||||
if data[0] != esc {
|
||||
return 0
|
||||
}
|
||||
|
||||
b1 := data[1]
|
||||
switch b1 {
|
||||
case '[': // CSI
|
||||
body := csiLength(data[2:])
|
||||
if body == 0 {
|
||||
return 0
|
||||
}
|
||||
return 2 + body
|
||||
case ']': // OSC – allows BEL or ST as terminator
|
||||
body := oscLength(data[2:])
|
||||
if body == 0 {
|
||||
return 0
|
||||
}
|
||||
return 2 + body
|
||||
case 'P', 'X', '^', '_': // DCS, SOS, PM, APC – require ST (ESC \) only
|
||||
body := stSequenceLength(data[2:])
|
||||
if body == 0 {
|
||||
return 0
|
||||
}
|
||||
return 2 + body
|
||||
}
|
||||
if b1 >= 0x40 && b1 <= 0x5F {
|
||||
// Fe (C1) two-byte; [ ] P X ^ _ handled above
|
||||
return 2
|
||||
}
|
||||
if b1 >= 0x30 && b1 <= 0x3F {
|
||||
// Fp (private) two-byte
|
||||
return 2
|
||||
}
|
||||
if b1 >= 0x20 && b1 <= 0x2F {
|
||||
// nF: intermediates then one final (0x30–0x7E)
|
||||
i := 2
|
||||
for i < n && data[i] >= 0x20 && data[i] <= 0x2F {
|
||||
i++
|
||||
}
|
||||
if i < n && data[i] >= 0x30 && data[i] <= 0x7E {
|
||||
return i + 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// csiLength returns the length of the CSI body (param/intermediate/final bytes).
|
||||
// data is the slice after "ESC [".
|
||||
// Per ECMA-48, the CSI body has the form:
|
||||
//
|
||||
// parameters (0x30–0x3F)*, intermediates (0x20–0x2F)*, final (0x40–0x7E)
|
||||
//
|
||||
// Once an intermediate byte is seen, subsequent parameter bytes are invalid.
|
||||
func csiLength[T ~string | ~[]byte](data T) int {
|
||||
seenIntermediate := false
|
||||
for i := 0; i < len(data); i++ {
|
||||
b := data[i]
|
||||
if b >= 0x30 && b <= 0x3F {
|
||||
if seenIntermediate {
|
||||
return 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
if b >= 0x20 && b <= 0x2F {
|
||||
seenIntermediate = true
|
||||
continue
|
||||
}
|
||||
if b >= 0x40 && b <= 0x7E {
|
||||
return i + 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// oscLength returns the length of the OSC body up to and including
|
||||
// the terminator. OSC accepts either BEL (0x07) or ST (ESC \) per
|
||||
// widespread terminal convention. data is the slice after "ESC ]".
|
||||
func oscLength[T ~string | ~[]byte](data T) int {
|
||||
for i := 0; i < len(data); i++ {
|
||||
b := data[i]
|
||||
if b == bel {
|
||||
return i + 1
|
||||
}
|
||||
if b == esc && i+1 < len(data) && data[i+1] == '\\' {
|
||||
return i + 2
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// stSequenceLength returns the length of a control-string body up to and
|
||||
// including the ST (ESC \) terminator. Used for DCS, SOS, PM, and APC, which
|
||||
// per ECMA-48 require ST and do not accept BEL. data is the slice after "ESC x".
|
||||
func stSequenceLength[T ~string | ~[]byte](data T) int {
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] == esc && i+1 < len(data) && data[i+1] == '\\' {
|
||||
return i + 2
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package graphemes
|
||||
|
||||
import "unicode/utf8"
|
||||
|
||||
// FromString returns an iterator for the grapheme clusters in the input string.
|
||||
// Iterate while Next() is true, and access the grapheme via Value().
|
||||
func FromString(s string) *Iterator[string] {
|
||||
return &Iterator[string]{
|
||||
split: splitFuncString,
|
||||
data: s,
|
||||
}
|
||||
}
|
||||
|
||||
// FromBytes returns an iterator for the grapheme clusters in the input bytes.
|
||||
// Iterate while Next() is true, and access the grapheme via Value().
|
||||
func FromBytes(b []byte) *Iterator[[]byte] {
|
||||
return &Iterator[[]byte]{
|
||||
split: splitFuncBytes,
|
||||
data: b,
|
||||
}
|
||||
}
|
||||
|
||||
// Iterator is a generic iterator for grapheme clusters in strings or byte slices,
|
||||
// with an ASCII hot path optimization.
|
||||
type Iterator[T ~string | ~[]byte] struct {
|
||||
split func(T, bool) (int, T, error)
|
||||
data T
|
||||
pos int
|
||||
start int
|
||||
// AnsiEscapeSequences treats ANSI escape sequences (ECMA-48) as single grapheme
|
||||
// clusters when true. Default is false.
|
||||
AnsiEscapeSequences bool
|
||||
}
|
||||
|
||||
var (
|
||||
splitFuncString = splitFunc[string]
|
||||
splitFuncBytes = splitFunc[[]byte]
|
||||
)
|
||||
|
||||
const (
|
||||
esc = 0x1B
|
||||
cr = 0x0D
|
||||
bel = 0x07
|
||||
)
|
||||
|
||||
// Next advances the iterator to the next grapheme cluster.
|
||||
// Returns false when there are no more grapheme clusters.
|
||||
func (iter *Iterator[T]) Next() bool {
|
||||
if iter.pos >= len(iter.data) {
|
||||
return false
|
||||
}
|
||||
iter.start = iter.pos
|
||||
|
||||
if iter.AnsiEscapeSequences && iter.data[iter.pos] == esc {
|
||||
if a := ansiEscapeLength(iter.data[iter.pos:]); a > 0 {
|
||||
iter.pos += a
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// ASCII hot path: any ASCII is one grapheme when next byte is ASCII or end.
|
||||
// Fall through on CR so splitfunc can handle CR+LF as a single cluster.
|
||||
b := iter.data[iter.pos]
|
||||
if b < utf8.RuneSelf && b != cr {
|
||||
if iter.pos+1 >= len(iter.data) || iter.data[iter.pos+1] < utf8.RuneSelf {
|
||||
iter.pos++
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to actual grapheme parsing
|
||||
remaining := iter.data[iter.pos:]
|
||||
advance, _, err := iter.split(remaining, true)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if advance <= 0 {
|
||||
panic("splitFunc returned a zero or negative advance")
|
||||
}
|
||||
iter.pos += advance
|
||||
if iter.pos > len(iter.data) {
|
||||
panic("splitFunc advanced beyond end of data")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Value returns the current grapheme cluster.
|
||||
func (iter *Iterator[T]) Value() T {
|
||||
return iter.data[iter.start:iter.pos]
|
||||
}
|
||||
|
||||
// Start returns the byte position of the current grapheme in the original data.
|
||||
func (iter *Iterator[T]) Start() int {
|
||||
return iter.start
|
||||
}
|
||||
|
||||
// End returns the byte position after the current grapheme in the original data.
|
||||
func (iter *Iterator[T]) End() int {
|
||||
return iter.pos
|
||||
}
|
||||
|
||||
// Reset resets the iterator to the beginning of the data.
|
||||
func (iter *Iterator[T]) Reset() {
|
||||
iter.start = 0
|
||||
iter.pos = 0
|
||||
}
|
||||
|
||||
// SetText sets the data for the iterator to operate on, and resets all state.
|
||||
func (iter *Iterator[T]) SetText(data T) {
|
||||
iter.data = data
|
||||
iter.start = 0
|
||||
iter.pos = 0
|
||||
}
|
||||
|
||||
// First returns the first grapheme cluster without advancing the iterator.
|
||||
func (iter *Iterator[T]) First() T {
|
||||
if len(iter.data) == 0 {
|
||||
return iter.data
|
||||
}
|
||||
|
||||
// Use a copy to leverage Next()'s ASCII optimization
|
||||
cp := *iter
|
||||
cp.pos = 0
|
||||
cp.start = 0
|
||||
cp.Next()
|
||||
return cp.Value()
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// Package graphemes implements Unicode grapheme cluster boundaries: https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
|
||||
package graphemes
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
*bufio.Scanner
|
||||
}
|
||||
|
||||
// FromReader returns a Scanner, to split graphemes per
|
||||
// https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
|
||||
//
|
||||
// It embeds a [bufio.Scanner], so you can use its methods.
|
||||
//
|
||||
// Iterate through graphemes by calling Scan() until false, then check Err().
|
||||
func FromReader(r io.Reader) *Scanner {
|
||||
sc := bufio.NewScanner(r)
|
||||
sc.Split(SplitFunc)
|
||||
return &Scanner{
|
||||
Scanner: sc,
|
||||
}
|
||||
}
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
package graphemes
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
)
|
||||
|
||||
// is determines if lookup intersects propert(ies)
|
||||
func (lookup property) is(properties property) bool {
|
||||
return (lookup & properties) != 0
|
||||
}
|
||||
|
||||
const _Ignore = _Extend
|
||||
|
||||
// incbState tracks state for GB9c rule (Indic conjunct clusters)
|
||||
// Pattern: Consonant (Extend|Linker)* Linker (Extend|Linker)* × Consonant
|
||||
type incbState int
|
||||
|
||||
const (
|
||||
incbNone incbState = iota // initial/reset
|
||||
incbConsonant // seen Consonant, awaiting Linker
|
||||
incbLinker // seen Consonant and Linker (conjunct ready)
|
||||
)
|
||||
|
||||
// SplitFunc is a bufio.SplitFunc implementation of Unicode grapheme cluster segmentation, for use with bufio.Scanner.
|
||||
//
|
||||
// See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
|
||||
var SplitFunc bufio.SplitFunc = splitFunc[[]byte]
|
||||
|
||||
func splitFunc[T ~string | ~[]byte](data T, atEOF bool) (advance int, token T, err error) {
|
||||
var empty T
|
||||
if len(data) == 0 {
|
||||
return 0, empty, nil
|
||||
}
|
||||
|
||||
// These vars are stateful across loop iterations
|
||||
var pos int
|
||||
var lastExIgnore property = 0 // "last excluding ignored categories"
|
||||
var lastLastExIgnore property = 0 // "last one before that"
|
||||
var regionalIndicatorCount int
|
||||
|
||||
// GB9c state: tracking Indic conjunct clusters
|
||||
var incb incbState
|
||||
|
||||
// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
|
||||
// to the right of the ×, from which we look back or forward
|
||||
|
||||
current, w := lookup(data[pos:])
|
||||
if w == 0 {
|
||||
if !atEOF {
|
||||
// Rune extends past current data, request more
|
||||
return 0, empty, nil
|
||||
}
|
||||
pos = len(data)
|
||||
return pos, data[:pos], nil
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB1
|
||||
// Start of text always advances
|
||||
pos += w
|
||||
|
||||
for {
|
||||
eot := pos == len(data) // "end of text"
|
||||
|
||||
if eot {
|
||||
if !atEOF {
|
||||
// Token extends past current data, request more
|
||||
return 0, empty, nil
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB2
|
||||
break
|
||||
}
|
||||
|
||||
/*
|
||||
We've switched the evaluation order of GB1↓ and GB2↑. It's ok:
|
||||
because we've checked for len(data) at the top of this function,
|
||||
sot and eot are mutually exclusive, order doesn't matter.
|
||||
*/
|
||||
|
||||
// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
|
||||
// to the right of the ×, from which we look back or forward
|
||||
|
||||
// Remember previous properties to avoid lookups/lookbacks
|
||||
last := current
|
||||
if !last.is(_Ignore) {
|
||||
lastLastExIgnore = lastExIgnore
|
||||
lastExIgnore = last
|
||||
}
|
||||
|
||||
// Update GB9c state based on what we just advanced past
|
||||
if last.is(_InCBConsonant | _InCBLinker | _InCBExtend) {
|
||||
switch {
|
||||
case last.is(_InCBConsonant):
|
||||
if incb != incbLinker {
|
||||
incb = incbConsonant
|
||||
}
|
||||
case last.is(_InCBLinker):
|
||||
if incb >= incbConsonant {
|
||||
incb = incbLinker
|
||||
}
|
||||
// case last.is(_InCBExtend): stay in current state
|
||||
}
|
||||
} else {
|
||||
incb = incbNone
|
||||
}
|
||||
|
||||
current, w = lookup(data[pos:])
|
||||
if w == 0 {
|
||||
if atEOF {
|
||||
// Just return the bytes, we can't do anything with them
|
||||
pos = len(data)
|
||||
break
|
||||
}
|
||||
// Rune extends past current data, request more
|
||||
return 0, empty, nil
|
||||
}
|
||||
|
||||
// Optimization: no rule can possibly apply
|
||||
if current|last == 0 { // i.e. both are zero
|
||||
break
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB3
|
||||
if current.is(_LF) && last.is(_CR) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB4
|
||||
// https://unicode.org/reports/tr29/#GB5
|
||||
if (current | last).is(_Control | _CR | _LF) {
|
||||
break
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB6
|
||||
if current.is(_L|_V|_LV|_LVT) && last.is(_L) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB7
|
||||
if current.is(_V|_T) && last.is(_LV|_V) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB8
|
||||
if current.is(_T) && last.is(_LVT|_T) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB9
|
||||
if current.is(_Extend | _ZWJ) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB9a
|
||||
if current.is(_SpacingMark) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB9b
|
||||
if last.is(_Prepend) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB9c
|
||||
// Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker.
|
||||
if incb == incbLinker && current.is(_InCBConsonant) {
|
||||
// After matching the pattern, reset state to start tracking a new pattern
|
||||
// The current Consonant becomes the start of the new pattern
|
||||
incb = incbConsonant
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB11
|
||||
if current.is(_ExtendedPictographic) && last.is(_ZWJ) && lastLastExIgnore.is(_ExtendedPictographic) {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB12
|
||||
// https://unicode.org/reports/tr29/#GB13
|
||||
if (current & last).is(_RegionalIndicator) {
|
||||
regionalIndicatorCount++
|
||||
|
||||
odd := regionalIndicatorCount%2 == 1
|
||||
if odd {
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// If we fall through all the above rules, it's a grapheme cluster break
|
||||
break
|
||||
}
|
||||
|
||||
// Return token
|
||||
return pos, data[:pos], nil
|
||||
}
|
||||
+1717
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user