mirror of
https://github.com/letic/terraform-provider-google.git
synced 2024-09-18 17:19:58 +00:00
961c878e0d
Switch to using Go modules. This migrates our vendor.json to use Go 1.11's modules system, and replaces the vendor folder with the output of go mod vendor. The vendored code should remain basically the same; I believe some tree shaking of packages and support scripts/licenses/READMEs/etc. happened. This also fixes Travis and our Makefile to no longer use govendor.
509 lines
14 KiB
Go
509 lines
14 KiB
Go
// Copyright 2011 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package norm
|
|
|
|
import "unicode/utf8"
|
|
|
|
const (
|
|
maxNonStarters = 30
|
|
// The maximum number of characters needed for a buffer is
|
|
// maxNonStarters + 1 for the starter + 1 for the GCJ
|
|
maxBufferSize = maxNonStarters + 2
|
|
maxNFCExpansion = 3 // NFC(0x1D160)
|
|
maxNFKCExpansion = 18 // NFKC(0xFDFA)
|
|
|
|
maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128
|
|
)
|
|
|
|
// ssState is used for reporting the segment state after inserting a rune.
|
|
// It is returned by streamSafe.next.
|
|
type ssState int
|
|
|
|
const (
|
|
// Indicates a rune was successfully added to the segment.
|
|
ssSuccess ssState = iota
|
|
// Indicates a rune starts a new segment and should not be added.
|
|
ssStarter
|
|
// Indicates a rune caused a segment overflow and a CGJ should be inserted.
|
|
ssOverflow
|
|
)
|
|
|
|
// streamSafe implements the policy of when a CGJ should be inserted.
|
|
type streamSafe uint8
|
|
|
|
// first inserts the first rune of a segment. It is a faster version of next if
|
|
// it is known p represents the first rune in a segment.
|
|
func (ss *streamSafe) first(p Properties) {
|
|
*ss = streamSafe(p.nTrailingNonStarters())
|
|
}
|
|
|
|
// insert returns a ssState value to indicate whether a rune represented by p
|
|
// can be inserted.
|
|
func (ss *streamSafe) next(p Properties) ssState {
|
|
if *ss > maxNonStarters {
|
|
panic("streamSafe was not reset")
|
|
}
|
|
n := p.nLeadingNonStarters()
|
|
if *ss += streamSafe(n); *ss > maxNonStarters {
|
|
*ss = 0
|
|
return ssOverflow
|
|
}
|
|
// The Stream-Safe Text Processing prescribes that the counting can stop
|
|
// as soon as a starter is encountered. However, there are some starters,
|
|
// like Jamo V and T, that can combine with other runes, leaving their
|
|
// successive non-starters appended to the previous, possibly causing an
|
|
// overflow. We will therefore consider any rune with a non-zero nLead to
|
|
// be a non-starter. Note that it always hold that if nLead > 0 then
|
|
// nLead == nTrail.
|
|
if n == 0 {
|
|
*ss = streamSafe(p.nTrailingNonStarters())
|
|
return ssStarter
|
|
}
|
|
return ssSuccess
|
|
}
|
|
|
|
// backwards is used for checking for overflow and segment starts
|
|
// when traversing a string backwards. Users do not need to call first
|
|
// for the first rune. The state of the streamSafe retains the count of
|
|
// the non-starters loaded.
|
|
func (ss *streamSafe) backwards(p Properties) ssState {
|
|
if *ss > maxNonStarters {
|
|
panic("streamSafe was not reset")
|
|
}
|
|
c := *ss + streamSafe(p.nTrailingNonStarters())
|
|
if c > maxNonStarters {
|
|
return ssOverflow
|
|
}
|
|
*ss = c
|
|
if p.nLeadingNonStarters() == 0 {
|
|
return ssStarter
|
|
}
|
|
return ssSuccess
|
|
}
|
|
|
|
func (ss streamSafe) isMax() bool {
|
|
return ss == maxNonStarters
|
|
}
|
|
|
|
// GraphemeJoiner is inserted after maxNonStarters non-starter runes.
|
|
const GraphemeJoiner = "\u034F"
|
|
|
|
// reorderBuffer is used to normalize a single segment. Characters inserted with
|
|
// insert are decomposed and reordered based on CCC. The compose method can
|
|
// be used to recombine characters. Note that the byte buffer does not hold
|
|
// the UTF-8 characters in order. Only the rune array is maintained in sorted
|
|
// order. flush writes the resulting segment to a byte array.
|
|
type reorderBuffer struct {
|
|
rune [maxBufferSize]Properties // Per character info.
|
|
byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos.
|
|
nbyte uint8 // Number or bytes.
|
|
ss streamSafe // For limiting length of non-starter sequence.
|
|
nrune int // Number of runeInfos.
|
|
f formInfo
|
|
|
|
src input
|
|
nsrc int
|
|
tmpBytes input
|
|
|
|
out []byte
|
|
flushF func(*reorderBuffer) bool
|
|
}
|
|
|
|
func (rb *reorderBuffer) init(f Form, src []byte) {
|
|
rb.f = *formTable[f]
|
|
rb.src.setBytes(src)
|
|
rb.nsrc = len(src)
|
|
rb.ss = 0
|
|
}
|
|
|
|
func (rb *reorderBuffer) initString(f Form, src string) {
|
|
rb.f = *formTable[f]
|
|
rb.src.setString(src)
|
|
rb.nsrc = len(src)
|
|
rb.ss = 0
|
|
}
|
|
|
|
func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
|
|
rb.out = out
|
|
rb.flushF = f
|
|
}
|
|
|
|
// reset discards all characters from the buffer.
|
|
func (rb *reorderBuffer) reset() {
|
|
rb.nrune = 0
|
|
rb.nbyte = 0
|
|
}
|
|
|
|
func (rb *reorderBuffer) doFlush() bool {
|
|
if rb.f.composing {
|
|
rb.compose()
|
|
}
|
|
res := rb.flushF(rb)
|
|
rb.reset()
|
|
return res
|
|
}
|
|
|
|
// appendFlush appends the normalized segment to rb.out.
|
|
func appendFlush(rb *reorderBuffer) bool {
|
|
for i := 0; i < rb.nrune; i++ {
|
|
start := rb.rune[i].pos
|
|
end := start + rb.rune[i].size
|
|
rb.out = append(rb.out, rb.byte[start:end]...)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// flush appends the normalized segment to out and resets rb.
|
|
func (rb *reorderBuffer) flush(out []byte) []byte {
|
|
for i := 0; i < rb.nrune; i++ {
|
|
start := rb.rune[i].pos
|
|
end := start + rb.rune[i].size
|
|
out = append(out, rb.byte[start:end]...)
|
|
}
|
|
rb.reset()
|
|
return out
|
|
}
|
|
|
|
// flushCopy copies the normalized segment to buf and resets rb.
|
|
// It returns the number of bytes written to buf.
|
|
func (rb *reorderBuffer) flushCopy(buf []byte) int {
|
|
p := 0
|
|
for i := 0; i < rb.nrune; i++ {
|
|
runep := rb.rune[i]
|
|
p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size])
|
|
}
|
|
rb.reset()
|
|
return p
|
|
}
|
|
|
|
// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class.
|
|
// It returns false if the buffer is not large enough to hold the rune.
|
|
// It is used internally by insert and insertString only.
|
|
func (rb *reorderBuffer) insertOrdered(info Properties) {
|
|
n := rb.nrune
|
|
b := rb.rune[:]
|
|
cc := info.ccc
|
|
if cc > 0 {
|
|
// Find insertion position + move elements to make room.
|
|
for ; n > 0; n-- {
|
|
if b[n-1].ccc <= cc {
|
|
break
|
|
}
|
|
b[n] = b[n-1]
|
|
}
|
|
}
|
|
rb.nrune += 1
|
|
pos := uint8(rb.nbyte)
|
|
rb.nbyte += utf8.UTFMax
|
|
info.pos = pos
|
|
b[n] = info
|
|
}
|
|
|
|
// insertErr is an error code returned by insert. Using this type instead
|
|
// of error improves performance up to 20% for many of the benchmarks.
|
|
type insertErr int
|
|
|
|
const (
|
|
iSuccess insertErr = -iota
|
|
iShortDst
|
|
iShortSrc
|
|
)
|
|
|
|
// insertFlush inserts the given rune in the buffer ordered by CCC.
|
|
// If a decomposition with multiple segments are encountered, they leading
|
|
// ones are flushed.
|
|
// It returns a non-zero error code if the rune was not inserted.
|
|
func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr {
|
|
if rune := src.hangul(i); rune != 0 {
|
|
rb.decomposeHangul(rune)
|
|
return iSuccess
|
|
}
|
|
if info.hasDecomposition() {
|
|
return rb.insertDecomposed(info.Decomposition())
|
|
}
|
|
rb.insertSingle(src, i, info)
|
|
return iSuccess
|
|
}
|
|
|
|
// insertUnsafe inserts the given rune in the buffer ordered by CCC.
|
|
// It is assumed there is sufficient space to hold the runes. It is the
|
|
// responsibility of the caller to ensure this. This can be done by checking
|
|
// the state returned by the streamSafe type.
|
|
func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
|
|
if rune := src.hangul(i); rune != 0 {
|
|
rb.decomposeHangul(rune)
|
|
}
|
|
if info.hasDecomposition() {
|
|
// TODO: inline.
|
|
rb.insertDecomposed(info.Decomposition())
|
|
} else {
|
|
rb.insertSingle(src, i, info)
|
|
}
|
|
}
|
|
|
|
// insertDecomposed inserts an entry in to the reorderBuffer for each rune
|
|
// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes.
|
|
// It flushes the buffer on each new segment start.
|
|
func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
|
|
rb.tmpBytes.setBytes(dcomp)
|
|
// As the streamSafe accounting already handles the counting for modifiers,
|
|
// we don't have to call next. However, we do need to keep the accounting
|
|
// intact when flushing the buffer.
|
|
for i := 0; i < len(dcomp); {
|
|
info := rb.f.info(rb.tmpBytes, i)
|
|
if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {
|
|
return iShortDst
|
|
}
|
|
i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)])
|
|
rb.insertOrdered(info)
|
|
}
|
|
return iSuccess
|
|
}
|
|
|
|
// insertSingle inserts an entry in the reorderBuffer for the rune at
|
|
// position i. info is the runeInfo for the rune at position i.
|
|
func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) {
|
|
src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size))
|
|
rb.insertOrdered(info)
|
|
}
|
|
|
|
// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb.
|
|
func (rb *reorderBuffer) insertCGJ() {
|
|
rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))})
|
|
}
|
|
|
|
// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
|
|
func (rb *reorderBuffer) appendRune(r rune) {
|
|
bn := rb.nbyte
|
|
sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
|
|
rb.nbyte += utf8.UTFMax
|
|
rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)}
|
|
rb.nrune++
|
|
}
|
|
|
|
// assignRune sets a rune at position pos. It is used for Hangul and recomposition.
|
|
func (rb *reorderBuffer) assignRune(pos int, r rune) {
|
|
bn := rb.rune[pos].pos
|
|
sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
|
|
rb.rune[pos] = Properties{pos: bn, size: uint8(sz)}
|
|
}
|
|
|
|
// runeAt returns the rune at position n. It is used for Hangul and recomposition.
|
|
func (rb *reorderBuffer) runeAt(n int) rune {
|
|
inf := rb.rune[n]
|
|
r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size])
|
|
return r
|
|
}
|
|
|
|
// bytesAt returns the UTF-8 encoding of the rune at position n.
|
|
// It is used for Hangul and recomposition.
|
|
func (rb *reorderBuffer) bytesAt(n int) []byte {
|
|
inf := rb.rune[n]
|
|
return rb.byte[inf.pos : int(inf.pos)+int(inf.size)]
|
|
}
|
|
|
|
// For Hangul we combine algorithmically, instead of using tables.
|
|
const (
|
|
hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80
|
|
hangulBase0 = 0xEA
|
|
hangulBase1 = 0xB0
|
|
hangulBase2 = 0x80
|
|
|
|
hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4
|
|
hangulEnd0 = 0xED
|
|
hangulEnd1 = 0x9E
|
|
hangulEnd2 = 0xA4
|
|
|
|
jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00
|
|
jamoLBase0 = 0xE1
|
|
jamoLBase1 = 0x84
|
|
jamoLEnd = 0x1113
|
|
jamoVBase = 0x1161
|
|
jamoVEnd = 0x1176
|
|
jamoTBase = 0x11A7
|
|
jamoTEnd = 0x11C3
|
|
|
|
jamoTCount = 28
|
|
jamoVCount = 21
|
|
jamoVTCount = 21 * 28
|
|
jamoLVTCount = 19 * 21 * 28
|
|
)
|
|
|
|
const hangulUTF8Size = 3
|
|
|
|
func isHangul(b []byte) bool {
|
|
if len(b) < hangulUTF8Size {
|
|
return false
|
|
}
|
|
b0 := b[0]
|
|
if b0 < hangulBase0 {
|
|
return false
|
|
}
|
|
b1 := b[1]
|
|
switch {
|
|
case b0 == hangulBase0:
|
|
return b1 >= hangulBase1
|
|
case b0 < hangulEnd0:
|
|
return true
|
|
case b0 > hangulEnd0:
|
|
return false
|
|
case b1 < hangulEnd1:
|
|
return true
|
|
}
|
|
return b1 == hangulEnd1 && b[2] < hangulEnd2
|
|
}
|
|
|
|
func isHangulString(b string) bool {
|
|
if len(b) < hangulUTF8Size {
|
|
return false
|
|
}
|
|
b0 := b[0]
|
|
if b0 < hangulBase0 {
|
|
return false
|
|
}
|
|
b1 := b[1]
|
|
switch {
|
|
case b0 == hangulBase0:
|
|
return b1 >= hangulBase1
|
|
case b0 < hangulEnd0:
|
|
return true
|
|
case b0 > hangulEnd0:
|
|
return false
|
|
case b1 < hangulEnd1:
|
|
return true
|
|
}
|
|
return b1 == hangulEnd1 && b[2] < hangulEnd2
|
|
}
|
|
|
|
// Caller must ensure len(b) >= 2.
|
|
func isJamoVT(b []byte) bool {
|
|
// True if (rune & 0xff00) == jamoLBase
|
|
return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1
|
|
}
|
|
|
|
func isHangulWithoutJamoT(b []byte) bool {
|
|
c, _ := utf8.DecodeRune(b)
|
|
c -= hangulBase
|
|
return c < jamoLVTCount && c%jamoTCount == 0
|
|
}
|
|
|
|
// decomposeHangul writes the decomposed Hangul to buf and returns the number
|
|
// of bytes written. len(buf) should be at least 9.
|
|
func decomposeHangul(buf []byte, r rune) int {
|
|
const JamoUTF8Len = 3
|
|
r -= hangulBase
|
|
x := r % jamoTCount
|
|
r /= jamoTCount
|
|
utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
|
|
utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
|
|
if x != 0 {
|
|
utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
|
|
return 3 * JamoUTF8Len
|
|
}
|
|
return 2 * JamoUTF8Len
|
|
}
|
|
|
|
// decomposeHangul algorithmically decomposes a Hangul rune into
|
|
// its Jamo components.
|
|
// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul.
|
|
func (rb *reorderBuffer) decomposeHangul(r rune) {
|
|
r -= hangulBase
|
|
x := r % jamoTCount
|
|
r /= jamoTCount
|
|
rb.appendRune(jamoLBase + r/jamoVCount)
|
|
rb.appendRune(jamoVBase + r%jamoVCount)
|
|
if x != 0 {
|
|
rb.appendRune(jamoTBase + x)
|
|
}
|
|
}
|
|
|
|
// combineHangul algorithmically combines Jamo character components into Hangul.
|
|
// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul.
|
|
func (rb *reorderBuffer) combineHangul(s, i, k int) {
|
|
b := rb.rune[:]
|
|
bn := rb.nrune
|
|
for ; i < bn; i++ {
|
|
cccB := b[k-1].ccc
|
|
cccC := b[i].ccc
|
|
if cccB == 0 {
|
|
s = k - 1
|
|
}
|
|
if s != k-1 && cccB >= cccC {
|
|
// b[i] is blocked by greater-equal cccX below it
|
|
b[k] = b[i]
|
|
k++
|
|
} else {
|
|
l := rb.runeAt(s) // also used to compare to hangulBase
|
|
v := rb.runeAt(i) // also used to compare to jamoT
|
|
switch {
|
|
case jamoLBase <= l && l < jamoLEnd &&
|
|
jamoVBase <= v && v < jamoVEnd:
|
|
// 11xx plus 116x to LV
|
|
rb.assignRune(s, hangulBase+
|
|
(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount)
|
|
case hangulBase <= l && l < hangulEnd &&
|
|
jamoTBase < v && v < jamoTEnd &&
|
|
((l-hangulBase)%jamoTCount) == 0:
|
|
// ACxx plus 11Ax to LVT
|
|
rb.assignRune(s, l+v-jamoTBase)
|
|
default:
|
|
b[k] = b[i]
|
|
k++
|
|
}
|
|
}
|
|
}
|
|
rb.nrune = k
|
|
}
|
|
|
|
// compose recombines the runes in the buffer.
|
|
// It should only be used to recompose a single segment, as it will not
|
|
// handle alternations between Hangul and non-Hangul characters correctly.
|
|
func (rb *reorderBuffer) compose() {
|
|
// UAX #15, section X5 , including Corrigendum #5
|
|
// "In any character sequence beginning with starter S, a character C is
|
|
// blocked from S if and only if there is some character B between S
|
|
// and C, and either B is a starter or it has the same or higher
|
|
// combining class as C."
|
|
bn := rb.nrune
|
|
if bn == 0 {
|
|
return
|
|
}
|
|
k := 1
|
|
b := rb.rune[:]
|
|
for s, i := 0, 1; i < bn; i++ {
|
|
if isJamoVT(rb.bytesAt(i)) {
|
|
// Redo from start in Hangul mode. Necessary to support
|
|
// U+320E..U+321E in NFKC mode.
|
|
rb.combineHangul(s, i, k)
|
|
return
|
|
}
|
|
ii := b[i]
|
|
// We can only use combineForward as a filter if we later
|
|
// get the info for the combined character. This is more
|
|
// expensive than using the filter. Using combinesBackward()
|
|
// is safe.
|
|
if ii.combinesBackward() {
|
|
cccB := b[k-1].ccc
|
|
cccC := ii.ccc
|
|
blocked := false // b[i] blocked by starter or greater or equal CCC?
|
|
if cccB == 0 {
|
|
s = k - 1
|
|
} else {
|
|
blocked = s != k-1 && cccB >= cccC
|
|
}
|
|
if !blocked {
|
|
combined := combine(rb.runeAt(s), rb.runeAt(i))
|
|
if combined != 0 {
|
|
rb.assignRune(s, combined)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
b[k] = b[i]
|
|
k++
|
|
}
|
|
rb.nrune = k
|
|
}
|