mirror of
https://gitee.com/johng/gf
synced 2026-06-06 02:25:47 +08:00
增加对字符集转换的封装，gxml包中使用新增的字符集转换包来做处理
This commit is contained in:
wenzi1
57
g/encoding/gcharset/gcharset.go
Normal file
57
g/encoding/gcharset/gcharset.go
Normal file
@ -0,0 +1,57 @@
|
||||
// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the MIT License.
|
||||
// If a copy of the MIT was not distributed with this file,
|
||||
// You can obtain one at https://gitee.com/johng/gf.
|
||||
|
||||
//gcharset
|
||||
package gcharset
|
||||
|
||||
import (
|
||||
"github.com/axgle/mahonia"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
//获取支持的字符集
|
||||
func GetCharset(name string) (*mahonia.Charset, error) {
|
||||
s := mahonia.GetCharset(name)
|
||||
if s == nil {
|
||||
return nil, errors.New(fmt.Sprintf("not support charset: %s", name))
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
//2个字符集之间的转换
|
||||
func Convert(dstCharset string, srcCharset string, src string) (dst string, err error) {
|
||||
s := mahonia.GetCharset(srcCharset)
|
||||
if s == nil {
|
||||
return "", errors.New(fmt.Sprintf("not support charset:%s", srcCharset))
|
||||
}
|
||||
|
||||
d := mahonia.GetCharset(dstCharset)
|
||||
if d == nil {
|
||||
return "", errors.New(fmt.Sprintf("not support charset:%s", dstCharset))
|
||||
}
|
||||
|
||||
srctmp := src
|
||||
if s.Name != "UTF-8" {
|
||||
srctmp = s.NewDecoder().ConvertString(srctmp)
|
||||
}
|
||||
|
||||
dst = srctmp
|
||||
if d.Name != "UTF-8" {
|
||||
dst = d.NewEncoder().ConvertString(dst)
|
||||
}
|
||||
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
//指定字符集转UTF8
|
||||
func ToUTF8(charset string, src string) (dst string, err error) {
|
||||
return Convert("UTF-8", charset, src)
|
||||
}
|
||||
|
||||
//UTF8转指定字符集
|
||||
func UTF8To(charset string, src string) (dst string, err error) {
|
||||
return Convert(charset, "UTF-8", src)
|
||||
}
|
||||
99
g/encoding/gcharset/gcharset_test.go
Normal file
99
g/encoding/gcharset/gcharset_test.go
Normal file
@ -0,0 +1,99 @@
|
||||
package gcharset_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"gitee.com/wenzi1/gf/g/encoding/gcharset"
|
||||
)
|
||||
|
||||
|
||||
var testData = []struct {
|
||||
utf8, other, otherEncoding string
|
||||
}{
|
||||
{"Résumé", "Résumé", "utf8"},
|
||||
{"Résumé", "R\xe9sum\xe9", "latin-1"},
|
||||
{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
|
||||
{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
|
||||
{"これは漢字です。", "\xfe\xff0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16"},
|
||||
{"𝄢𝄞𝄪𝄫", "\xfe\xff\xd8\x34\xdd\x22\xd8\x34\xdd\x1e\xd8\x34\xdd\x2a\xd8\x34\xdd\x2b", "UTF-16"},
|
||||
{"Hello, world", "Hello, world", "ASCII"},
|
||||
{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
|
||||
{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
|
||||
{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
|
||||
{"latviešu", "latvie\xf0u", "ISO-8859-13"},
|
||||
{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
|
||||
{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
|
||||
{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
|
||||
{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
|
||||
{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
|
||||
{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
|
||||
{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
|
||||
{"Kağan", "Ka\xf0an", "ISO-8859-9"},
|
||||
{"Résumé", "R\x8esum\x8e", "macintosh"},
|
||||
{"Gdańsk", "Gda\xf1sk", "windows-1250"},
|
||||
{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
|
||||
{"Résumé", "R\xe9sum\xe9", "windows-1252"},
|
||||
{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
|
||||
{"Kağan", "Ka\xf0an", "windows-1254"},
|
||||
{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
|
||||
{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
|
||||
{"latviešu", "latvie\xf0u", "windows-1257"},
|
||||
{"Việt", "Vi\xea\xf2t", "windows-1258"},
|
||||
{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
|
||||
{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
|
||||
{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
|
||||
{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
|
||||
{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
|
||||
{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
|
||||
{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
|
||||
{"㧯", "\x82\x31\x89\x38", "gb18030"},
|
||||
{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
|
||||
{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
|
||||
}
|
||||
|
||||
func TestDecode(t *testing.T) {
|
||||
for _, data := range testData {
|
||||
str := ""
|
||||
str, err := gcharset.Convert("UTF-8", data.otherEncoding, data.other)
|
||||
if err != nil {
|
||||
t.Errorf("Could not create decoder for %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if str != data.utf8 {
|
||||
t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func TestEncode(t *testing.T) {
|
||||
for _, data := range testData {
|
||||
str := ""
|
||||
str, err := gcharset.Convert(data.otherEncoding, "UTF-8", data.utf8)
|
||||
if err != nil {
|
||||
t.Errorf("Could not create decoder for %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if str != data.other {
|
||||
t.Errorf("Unexpected value: %#v (expected %#v)", str, data.other)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvert(t *testing.T) {
|
||||
srcCharset := "big5"
|
||||
src := "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed"
|
||||
dstCharset := "gbk"
|
||||
dst := "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed"
|
||||
|
||||
str, err := gcharset.Convert(dstCharset, srcCharset, src)
|
||||
if err != nil {
|
||||
t.Errorf("convert error. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if str != dst {
|
||||
t.Errorf("unexpected value:%#v (expected %#v)", str, dst)
|
||||
}
|
||||
}
|
||||
@ -11,32 +11,13 @@ import (
|
||||
"github.com/clbanning/mxj"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"strings"
|
||||
"golang.org/x/text/transform"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"gitee.com/johng/gf/g/encoding/gcharset"
|
||||
"gitee.com/johng/gf/g/util/gregx"
|
||||
)
|
||||
|
||||
// 将XML内容解析为map变量
|
||||
func Decode(xmlbyte []byte) (map[string]interface{}, error) {
|
||||
//@author wenzi1
|
||||
//@date 20180529
|
||||
//XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法
|
||||
//所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法
|
||||
if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 {
|
||||
charsetReader := func(charset string, input io.Reader) (io.Reader, error) {
|
||||
reader := input
|
||||
switch charset {
|
||||
case "GBK":
|
||||
reader = transform.NewReader(input,simplifiedchinese.GBK.NewDecoder())
|
||||
case "GB18030":
|
||||
reader = transform.NewReader(input, simplifiedchinese.GB18030.NewDecoder())
|
||||
default:
|
||||
reader = input
|
||||
}
|
||||
return reader, nil
|
||||
}
|
||||
mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader}
|
||||
}
|
||||
Prepare(xmlbyte)
|
||||
return mxj.NewMapXml(xmlbyte)
|
||||
}
|
||||
|
||||
@ -51,30 +32,40 @@ func EncodeWithIndent(v map[string]interface{}, rootTag...string) ([]byte, error
|
||||
|
||||
// XML格式内容直接转换为JSON格式内容
|
||||
func ToJson(xmlbyte []byte) ([]byte, error) {
|
||||
//@author wenzi1
|
||||
//@date 20180529
|
||||
//XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法
|
||||
//所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法
|
||||
if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 {
|
||||
charsetReader := func(charset string, input io.Reader) (io.Reader, error) {
|
||||
reader := input
|
||||
switch charset {
|
||||
case "GBK":
|
||||
reader = transform.NewReader(input,simplifiedchinese.GBK.NewDecoder())
|
||||
case "GB18030":
|
||||
reader = transform.NewReader(input, simplifiedchinese.GB18030.NewDecoder())
|
||||
default:
|
||||
reader = input
|
||||
}
|
||||
return reader, nil
|
||||
}
|
||||
mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader}
|
||||
}
|
||||
Prepare(xmlbyte)
|
||||
mv, err := mxj.NewMapXml(xmlbyte)
|
||||
|
||||
if err == nil {
|
||||
return mv.Json()
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
//XML字符集预处理
|
||||
//@author wenzi1
|
||||
//@date 20180604
|
||||
func Prepare(xmlbyte []byte) error {
|
||||
patten := "<\\?xml\\s+.*?\\s+encoding=\"(.*?)\""
|
||||
charsetReader := func(charset string, input io.Reader) (io.Reader, error) {
|
||||
reader, err := gcharset.GetCharset(charset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reader.NewDecoder().NewReader(input), nil
|
||||
}
|
||||
|
||||
matchStr, err := gregx.MatchString(patten, string(xmlbyte))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
charset, err := gcharset.GetCharset(matchStr[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if charset.Name != "UTF-8" {
|
||||
mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user