Matthias Andreas Benkard | 832a54e | 2019-01-29 09:27:38 +0100 | [diff] [blame] | 1 | // Copyright 2014 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | // Package runes provide transforms for UTF-8 encoded text. |
| 6 | package runes // import "golang.org/x/text/runes" |
| 7 | |
| 8 | import ( |
| 9 | "unicode" |
| 10 | "unicode/utf8" |
| 11 | |
| 12 | "golang.org/x/text/transform" |
| 13 | ) |
| 14 | |
| 15 | // A Set is a collection of runes. |
| 16 | type Set interface { |
| 17 | // Contains returns true if r is contained in the set. |
| 18 | Contains(r rune) bool |
| 19 | } |
| 20 | |
| 21 | type setFunc func(rune) bool |
| 22 | |
| 23 | func (s setFunc) Contains(r rune) bool { |
| 24 | return s(r) |
| 25 | } |
| 26 | |
| 27 | // Note: using funcs here instead of wrapping types result in cleaner |
| 28 | // documentation and a smaller API. |
| 29 | |
| 30 | // In creates a Set with a Contains method that returns true for all runes in |
| 31 | // the given RangeTable. |
| 32 | func In(rt *unicode.RangeTable) Set { |
| 33 | return setFunc(func(r rune) bool { return unicode.Is(rt, r) }) |
| 34 | } |
| 35 | |
| 36 | // In creates a Set with a Contains method that returns true for all runes not |
| 37 | // in the given RangeTable. |
| 38 | func NotIn(rt *unicode.RangeTable) Set { |
| 39 | return setFunc(func(r rune) bool { return !unicode.Is(rt, r) }) |
| 40 | } |
| 41 | |
| 42 | // Predicate creates a Set with a Contains method that returns f(r). |
| 43 | func Predicate(f func(rune) bool) Set { |
| 44 | return setFunc(f) |
| 45 | } |
| 46 | |
| 47 | // Transformer implements the transform.Transformer interface. |
| 48 | type Transformer struct { |
| 49 | t transform.SpanningTransformer |
| 50 | } |
| 51 | |
| 52 | func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { |
| 53 | return t.t.Transform(dst, src, atEOF) |
| 54 | } |
| 55 | |
| 56 | func (t Transformer) Span(b []byte, atEOF bool) (n int, err error) { |
| 57 | return t.t.Span(b, atEOF) |
| 58 | } |
| 59 | |
| 60 | func (t Transformer) Reset() { t.t.Reset() } |
| 61 | |
| 62 | // Bytes returns a new byte slice with the result of converting b using t. It |
| 63 | // calls Reset on t. It returns nil if any error was found. This can only happen |
| 64 | // if an error-producing Transformer is passed to If. |
| 65 | func (t Transformer) Bytes(b []byte) []byte { |
| 66 | b, _, err := transform.Bytes(t, b) |
| 67 | if err != nil { |
| 68 | return nil |
| 69 | } |
| 70 | return b |
| 71 | } |
| 72 | |
| 73 | // String returns a string with the result of converting s using t. It calls |
| 74 | // Reset on t. It returns the empty string if any error was found. This can only |
| 75 | // happen if an error-producing Transformer is passed to If. |
| 76 | func (t Transformer) String(s string) string { |
| 77 | s, _, err := transform.String(t, s) |
| 78 | if err != nil { |
| 79 | return "" |
| 80 | } |
| 81 | return s |
| 82 | } |
| 83 | |
| 84 | // TODO: |
| 85 | // - Copy: copying strings and bytes in whole-rune units. |
| 86 | // - Validation (maybe) |
| 87 | // - Well-formed-ness (maybe) |
| 88 | |
| 89 | const runeErrorString = string(utf8.RuneError) |
| 90 | |
| 91 | // Remove returns a Transformer that removes runes r for which s.Contains(r). |
| 92 | // Illegal input bytes are replaced by RuneError before being passed to f. |
| 93 | func Remove(s Set) Transformer { |
| 94 | if f, ok := s.(setFunc); ok { |
| 95 | // This little trick cuts the running time of BenchmarkRemove for sets |
| 96 | // created by Predicate roughly in half. |
| 97 | // TODO: special-case RangeTables as well. |
| 98 | return Transformer{remove(f)} |
| 99 | } |
| 100 | return Transformer{remove(s.Contains)} |
| 101 | } |
| 102 | |
| 103 | // TODO: remove transform.RemoveFunc. |
| 104 | |
| 105 | type remove func(r rune) bool |
| 106 | |
| 107 | func (remove) Reset() {} |
| 108 | |
| 109 | // Span implements transform.Spanner. |
| 110 | func (t remove) Span(src []byte, atEOF bool) (n int, err error) { |
| 111 | for r, size := rune(0), 0; n < len(src); { |
| 112 | if r = rune(src[n]); r < utf8.RuneSelf { |
| 113 | size = 1 |
| 114 | } else if r, size = utf8.DecodeRune(src[n:]); size == 1 { |
| 115 | // Invalid rune. |
| 116 | if !atEOF && !utf8.FullRune(src[n:]) { |
| 117 | err = transform.ErrShortSrc |
| 118 | } else { |
| 119 | err = transform.ErrEndOfSpan |
| 120 | } |
| 121 | break |
| 122 | } |
| 123 | if t(r) { |
| 124 | err = transform.ErrEndOfSpan |
| 125 | break |
| 126 | } |
| 127 | n += size |
| 128 | } |
| 129 | return |
| 130 | } |
| 131 | |
| 132 | // Transform implements transform.Transformer. |
| 133 | func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { |
| 134 | for r, size := rune(0), 0; nSrc < len(src); { |
| 135 | if r = rune(src[nSrc]); r < utf8.RuneSelf { |
| 136 | size = 1 |
| 137 | } else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 { |
| 138 | // Invalid rune. |
| 139 | if !atEOF && !utf8.FullRune(src[nSrc:]) { |
| 140 | err = transform.ErrShortSrc |
| 141 | break |
| 142 | } |
| 143 | // We replace illegal bytes with RuneError. Not doing so might |
| 144 | // otherwise turn a sequence of invalid UTF-8 into valid UTF-8. |
| 145 | // The resulting byte sequence may subsequently contain runes |
| 146 | // for which t(r) is true that were passed unnoticed. |
| 147 | if !t(utf8.RuneError) { |
| 148 | if nDst+3 > len(dst) { |
| 149 | err = transform.ErrShortDst |
| 150 | break |
| 151 | } |
| 152 | dst[nDst+0] = runeErrorString[0] |
| 153 | dst[nDst+1] = runeErrorString[1] |
| 154 | dst[nDst+2] = runeErrorString[2] |
| 155 | nDst += 3 |
| 156 | } |
| 157 | nSrc++ |
| 158 | continue |
| 159 | } |
| 160 | if t(r) { |
| 161 | nSrc += size |
| 162 | continue |
| 163 | } |
| 164 | if nDst+size > len(dst) { |
| 165 | err = transform.ErrShortDst |
| 166 | break |
| 167 | } |
| 168 | for i := 0; i < size; i++ { |
| 169 | dst[nDst] = src[nSrc] |
| 170 | nDst++ |
| 171 | nSrc++ |
| 172 | } |
| 173 | } |
| 174 | return |
| 175 | } |
| 176 | |
| 177 | // Map returns a Transformer that maps the runes in the input using the given |
| 178 | // mapping. Illegal bytes in the input are converted to utf8.RuneError before |
| 179 | // being passed to the mapping func. |
| 180 | func Map(mapping func(rune) rune) Transformer { |
| 181 | return Transformer{mapper(mapping)} |
| 182 | } |
| 183 | |
| 184 | type mapper func(rune) rune |
| 185 | |
| 186 | func (mapper) Reset() {} |
| 187 | |
| 188 | // Span implements transform.Spanner. |
| 189 | func (t mapper) Span(src []byte, atEOF bool) (n int, err error) { |
| 190 | for r, size := rune(0), 0; n < len(src); n += size { |
| 191 | if r = rune(src[n]); r < utf8.RuneSelf { |
| 192 | size = 1 |
| 193 | } else if r, size = utf8.DecodeRune(src[n:]); size == 1 { |
| 194 | // Invalid rune. |
| 195 | if !atEOF && !utf8.FullRune(src[n:]) { |
| 196 | err = transform.ErrShortSrc |
| 197 | } else { |
| 198 | err = transform.ErrEndOfSpan |
| 199 | } |
| 200 | break |
| 201 | } |
| 202 | if t(r) != r { |
| 203 | err = transform.ErrEndOfSpan |
| 204 | break |
| 205 | } |
| 206 | } |
| 207 | return n, err |
| 208 | } |
| 209 | |
| 210 | // Transform implements transform.Transformer. |
| 211 | func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { |
| 212 | var replacement rune |
| 213 | var b [utf8.UTFMax]byte |
| 214 | |
| 215 | for r, size := rune(0), 0; nSrc < len(src); { |
| 216 | if r = rune(src[nSrc]); r < utf8.RuneSelf { |
| 217 | if replacement = t(r); replacement < utf8.RuneSelf { |
| 218 | if nDst == len(dst) { |
| 219 | err = transform.ErrShortDst |
| 220 | break |
| 221 | } |
| 222 | dst[nDst] = byte(replacement) |
| 223 | nDst++ |
| 224 | nSrc++ |
| 225 | continue |
| 226 | } |
| 227 | size = 1 |
| 228 | } else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 { |
| 229 | // Invalid rune. |
| 230 | if !atEOF && !utf8.FullRune(src[nSrc:]) { |
| 231 | err = transform.ErrShortSrc |
| 232 | break |
| 233 | } |
| 234 | |
| 235 | if replacement = t(utf8.RuneError); replacement == utf8.RuneError { |
| 236 | if nDst+3 > len(dst) { |
| 237 | err = transform.ErrShortDst |
| 238 | break |
| 239 | } |
| 240 | dst[nDst+0] = runeErrorString[0] |
| 241 | dst[nDst+1] = runeErrorString[1] |
| 242 | dst[nDst+2] = runeErrorString[2] |
| 243 | nDst += 3 |
| 244 | nSrc++ |
| 245 | continue |
| 246 | } |
| 247 | } else if replacement = t(r); replacement == r { |
| 248 | if nDst+size > len(dst) { |
| 249 | err = transform.ErrShortDst |
| 250 | break |
| 251 | } |
| 252 | for i := 0; i < size; i++ { |
| 253 | dst[nDst] = src[nSrc] |
| 254 | nDst++ |
| 255 | nSrc++ |
| 256 | } |
| 257 | continue |
| 258 | } |
| 259 | |
| 260 | n := utf8.EncodeRune(b[:], replacement) |
| 261 | |
| 262 | if nDst+n > len(dst) { |
| 263 | err = transform.ErrShortDst |
| 264 | break |
| 265 | } |
| 266 | for i := 0; i < n; i++ { |
| 267 | dst[nDst] = b[i] |
| 268 | nDst++ |
| 269 | } |
| 270 | nSrc += size |
| 271 | } |
| 272 | return |
| 273 | } |
| 274 | |
| 275 | // ReplaceIllFormed returns a transformer that replaces all input bytes that are |
| 276 | // not part of a well-formed UTF-8 code sequence with utf8.RuneError. |
| 277 | func ReplaceIllFormed() Transformer { |
| 278 | return Transformer{&replaceIllFormed{}} |
| 279 | } |
| 280 | |
| 281 | type replaceIllFormed struct{ transform.NopResetter } |
| 282 | |
| 283 | func (t replaceIllFormed) Span(src []byte, atEOF bool) (n int, err error) { |
| 284 | for n < len(src) { |
| 285 | // ASCII fast path. |
| 286 | if src[n] < utf8.RuneSelf { |
| 287 | n++ |
| 288 | continue |
| 289 | } |
| 290 | |
| 291 | r, size := utf8.DecodeRune(src[n:]) |
| 292 | |
| 293 | // Look for a valid non-ASCII rune. |
| 294 | if r != utf8.RuneError || size != 1 { |
| 295 | n += size |
| 296 | continue |
| 297 | } |
| 298 | |
| 299 | // Look for short source data. |
| 300 | if !atEOF && !utf8.FullRune(src[n:]) { |
| 301 | err = transform.ErrShortSrc |
| 302 | break |
| 303 | } |
| 304 | |
| 305 | // We have an invalid rune. |
| 306 | err = transform.ErrEndOfSpan |
| 307 | break |
| 308 | } |
| 309 | return n, err |
| 310 | } |
| 311 | |
| 312 | func (t replaceIllFormed) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { |
| 313 | for nSrc < len(src) { |
| 314 | // ASCII fast path. |
| 315 | if r := src[nSrc]; r < utf8.RuneSelf { |
| 316 | if nDst == len(dst) { |
| 317 | err = transform.ErrShortDst |
| 318 | break |
| 319 | } |
| 320 | dst[nDst] = r |
| 321 | nDst++ |
| 322 | nSrc++ |
| 323 | continue |
| 324 | } |
| 325 | |
| 326 | // Look for a valid non-ASCII rune. |
| 327 | if _, size := utf8.DecodeRune(src[nSrc:]); size != 1 { |
| 328 | if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { |
| 329 | err = transform.ErrShortDst |
| 330 | break |
| 331 | } |
| 332 | nDst += size |
| 333 | nSrc += size |
| 334 | continue |
| 335 | } |
| 336 | |
| 337 | // Look for short source data. |
| 338 | if !atEOF && !utf8.FullRune(src[nSrc:]) { |
| 339 | err = transform.ErrShortSrc |
| 340 | break |
| 341 | } |
| 342 | |
| 343 | // We have an invalid rune. |
| 344 | if nDst+3 > len(dst) { |
| 345 | err = transform.ErrShortDst |
| 346 | break |
| 347 | } |
| 348 | dst[nDst+0] = runeErrorString[0] |
| 349 | dst[nDst+1] = runeErrorString[1] |
| 350 | dst[nDst+2] = runeErrorString[2] |
| 351 | nDst += 3 |
| 352 | nSrc++ |
| 353 | } |
| 354 | return nDst, nSrc, err |
| 355 | } |