Matthias Andreas Benkard | 832a54e | 2019-01-29 09:27:38 +0100 | [diff] [blame] | 1 | // Copyright 2013 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | // +build ignore |
| 6 | |
| 7 | package main |
| 8 | |
| 9 | // This program generates tables.go: |
| 10 | // go run maketables.go | gofmt > tables.go |
| 11 | |
| 12 | import ( |
| 13 | "bufio" |
| 14 | "fmt" |
| 15 | "log" |
| 16 | "net/http" |
| 17 | "sort" |
| 18 | "strings" |
| 19 | ) |
| 20 | |
| 21 | func main() { |
| 22 | fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") |
| 23 | fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n") |
| 24 | fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n") |
| 25 | |
| 26 | printGB18030() |
| 27 | printGBK() |
| 28 | } |
| 29 | |
| 30 | func printGB18030() { |
| 31 | res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt") |
| 32 | if err != nil { |
| 33 | log.Fatalf("Get: %v", err) |
| 34 | } |
| 35 | defer res.Body.Close() |
| 36 | |
| 37 | fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n") |
| 38 | fmt.Printf("var gb18030 = [...][2]uint16{\n") |
| 39 | scanner := bufio.NewScanner(res.Body) |
| 40 | for scanner.Scan() { |
| 41 | s := strings.TrimSpace(scanner.Text()) |
| 42 | if s == "" || s[0] == '#' { |
| 43 | continue |
| 44 | } |
| 45 | x, y := uint32(0), uint32(0) |
| 46 | if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
| 47 | log.Fatalf("could not parse %q", s) |
| 48 | } |
| 49 | if x < 0x10000 && y < 0x10000 { |
| 50 | fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y) |
| 51 | } |
| 52 | } |
| 53 | fmt.Printf("}\n\n") |
| 54 | } |
| 55 | |
| 56 | func printGBK() { |
| 57 | res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt") |
| 58 | if err != nil { |
| 59 | log.Fatalf("Get: %v", err) |
| 60 | } |
| 61 | defer res.Body.Close() |
| 62 | |
| 63 | mapping := [65536]uint16{} |
| 64 | reverse := [65536]uint16{} |
| 65 | |
| 66 | scanner := bufio.NewScanner(res.Body) |
| 67 | for scanner.Scan() { |
| 68 | s := strings.TrimSpace(scanner.Text()) |
| 69 | if s == "" || s[0] == '#' { |
| 70 | continue |
| 71 | } |
| 72 | x, y := uint16(0), uint16(0) |
| 73 | if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
| 74 | log.Fatalf("could not parse %q", s) |
| 75 | } |
| 76 | if x < 0 || 126*190 <= x { |
| 77 | log.Fatalf("GBK code %d is out of range", x) |
| 78 | } |
| 79 | mapping[x] = y |
| 80 | if reverse[y] == 0 { |
| 81 | c0, c1 := x/190, x%190 |
| 82 | if c1 >= 0x3f { |
| 83 | c1++ |
| 84 | } |
| 85 | reverse[y] = (0x81+c0)<<8 | (0x40 + c1) |
| 86 | } |
| 87 | } |
| 88 | if err := scanner.Err(); err != nil { |
| 89 | log.Fatalf("scanner error: %v", err) |
| 90 | } |
| 91 | |
| 92 | fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n") |
| 93 | fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n") |
| 94 | fmt.Printf("var decode = [...]uint16{\n") |
| 95 | for i, v := range mapping { |
| 96 | if v != 0 { |
| 97 | fmt.Printf("\t%d: 0x%04X,\n", i, v) |
| 98 | } |
| 99 | } |
| 100 | fmt.Printf("}\n\n") |
| 101 | |
| 102 | // Any run of at least separation continuous zero entries in the reverse map will |
| 103 | // be a separate encode table. |
| 104 | const separation = 1024 |
| 105 | |
| 106 | intervals := []interval(nil) |
| 107 | low, high := -1, -1 |
| 108 | for i, v := range reverse { |
| 109 | if v == 0 { |
| 110 | continue |
| 111 | } |
| 112 | if low < 0 { |
| 113 | low = i |
| 114 | } else if i-high >= separation { |
| 115 | if high >= 0 { |
| 116 | intervals = append(intervals, interval{low, high}) |
| 117 | } |
| 118 | low = i |
| 119 | } |
| 120 | high = i + 1 |
| 121 | } |
| 122 | if high >= 0 { |
| 123 | intervals = append(intervals, interval{low, high}) |
| 124 | } |
| 125 | sort.Sort(byDecreasingLength(intervals)) |
| 126 | |
| 127 | fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) |
| 128 | fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n") |
| 129 | fmt.Printf("// sorted by decreasing length.\n") |
| 130 | for i, v := range intervals { |
| 131 | fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) |
| 132 | } |
| 133 | fmt.Printf("\n") |
| 134 | |
| 135 | for i, v := range intervals { |
| 136 | fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) |
| 137 | fmt.Printf("var encode%d = [...]uint16{\n", i) |
| 138 | for j := v.low; j < v.high; j++ { |
| 139 | x := reverse[j] |
| 140 | if x == 0 { |
| 141 | continue |
| 142 | } |
| 143 | fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) |
| 144 | } |
| 145 | fmt.Printf("}\n\n") |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | // interval is a half-open interval [low, high). |
| 150 | type interval struct { |
| 151 | low, high int |
| 152 | } |
| 153 | |
| 154 | func (i interval) len() int { return i.high - i.low } |
| 155 | |
| 156 | // byDecreasingLength sorts intervals by decreasing length. |
| 157 | type byDecreasingLength []interval |
| 158 | |
| 159 | func (b byDecreasingLength) Len() int { return len(b) } |
| 160 | func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } |
| 161 | func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |