From 0e889877094eda4725e3bc9247a3bc4b7998d728 Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Tue, 29 May 2018 23:23:49 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BD=93xml=E4=B8=ADenco?= =?UTF-8?q?ding=E5=AD=97=E7=AC=A6=E9=9B=86=E4=B8=BA=E9=9D=9EUTF-8=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E9=9B=86=E6=97=B6=E6=8A=A5=E9=94=99=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gxml/gxml.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/g/encoding/gxml/gxml.go b/g/encoding/gxml/gxml.go index 4a132546b..e8e90bc5f 100644 --- a/g/encoding/gxml/gxml.go +++ b/g/encoding/gxml/gxml.go @@ -9,10 +9,20 @@ package gxml import ( "github.com/clbanning/mxj" + "encoding/xml" + "io" ) -// 将XML内容解析为map变量 +// 将XML内容解析为map变量,入口参数使用的字符集必须为UTF-8的 func Decode(xmlbyte []byte) (map[string]interface{}, error) { + //@author wenzi1 + //@date 2018-05-29 + //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 + //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 + charsetReader := func(charset string, input io.Reader) (io.Reader, error) { + return input, nil + } + mxj.CustomDecoder = &xml.Decoder{CharsetReader:charsetReader} return mxj.NewMapXml(xmlbyte) } @@ -25,8 +35,16 @@ func EncodeWithIndent(v map[string]interface{}, rootTag...string) ([]byte, error return mxj.Map(v).XmlIndent("", "\t", rootTag...) } -// XML格式内容直接转换为JSON格式内容 +// XML格式内容直接转换为JSON格式内容,入口参数使用的字符集必须为UTF-8的 func ToJson(xmlbyte []byte) ([]byte, error) { + //@author wenzi1 + //@date 2018-05-29 + //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 + //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 + charsetReader := func(charset string, input io.Reader) (io.Reader, error) { + return input, nil + } + mxj.CustomDecoder = &xml.Decoder{CharsetReader:charsetReader} if mv, err := mxj.NewMapXml(xmlbyte); err == nil { return mv.Json() } else { From 1c3e8cb9e944e53d98e6aa04839986d8f33f689b Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Fri, 1 Jun 2018 20:52:17 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E6=94=AF=E6=8C=81GBK/GB18030=E7=9A=84?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E9=9B=86=E8=BD=AC=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gxml/gxml.go | 59 +++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/g/encoding/gxml/gxml.go b/g/encoding/gxml/gxml.go index e8e90bc5f..b2e441057 100644 --- a/g/encoding/gxml/gxml.go +++ b/g/encoding/gxml/gxml.go @@ -11,18 +11,28 @@ import ( "github.com/clbanning/mxj" "encoding/xml" "io" + "strings" + "golang.org/x/text/transform" + "golang.org/x/text/encoding/simplifiedchinese" ) -// 将XML内容解析为map变量,入口参数使用的字符集必须为UTF-8的 +// 将XML内容解析为map变量 func Decode(xmlbyte []byte) (map[string]interface{}, error) { - //@author wenzi1 - //@date 2018-05-29 - //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 - //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 - charsetReader := func(charset string, input io.Reader) (io.Reader, error) { - return input, nil - } - mxj.CustomDecoder = &xml.Decoder{CharsetReader:charsetReader} + if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 { + charsetReader := func(charset string, input io.Reader) (io.Reader, error) { + reader := input + switch charset { + case "GBK": + reader = transform.NewReader(input,simplifiedchinese.GBK.NewDecoder()) + case "GB18030": + reader = transform.NewReader(input, simplifiedchinese.GB18030.NewDecoder()) + default: + reader = input + } + return reader, nil + } + mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader} + } return mxj.NewMapXml(xmlbyte) } @@ -35,17 +45,32 @@ func EncodeWithIndent(v map[string]interface{}, rootTag...string) ([]byte, error return mxj.Map(v).XmlIndent("", "\t", rootTag...) } -// XML格式内容直接转换为JSON格式内容,入口参数使用的字符集必须为UTF-8的 +// XML格式内容直接转换为JSON格式内容 func ToJson(xmlbyte []byte) ([]byte, error) { - //@author wenzi1 - //@date 2018-05-29 + //input, _ := ioutil.ReadAll(bytes.NewReader(xmlbyte)) + //input:= bytes.NewReader(xmlbyte) + //charset := "UTF-8" + //@wenzi1 20180529 //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 - charsetReader := func(charset string, input io.Reader) (io.Reader, error) { - return input, nil - } - mxj.CustomDecoder = &xml.Decoder{CharsetReader:charsetReader} - if mv, err := mxj.NewMapXml(xmlbyte); err == nil { + if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 { + charsetReader := func(charset string, input io.Reader) (io.Reader, error) { + reader := input + switch charset { + case "GBK": + reader = transform.NewReader(input,simplifiedchinese.GBK.NewDecoder()) + case "GB18030": + reader = transform.NewReader(input, simplifiedchinese.GB18030.NewDecoder()) + default: + reader = input + } + return reader, nil + } + mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader} + } + mv, err := mxj.NewMapXml(xmlbyte) + + if err == nil { return mv.Json() } else { return nil, err From 5bf65f1476a68940d947cf47faf43a059c3a667d Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Fri, 1 Jun 2018 21:10:28 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E5=A2=9E=E5=8A=A0gzip=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E7=9A=84=E5=B0=81=E8=A3=85=EF=BC=8C=E4=BF=AE=E6=94=B9gxml?= =?UTF-8?q?=E8=83=BD=E5=A4=9F=E6=94=AF=E6=8C=81GBK/GB18030=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E8=BD=ACUTF-8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gcompress/gcompress.go | 27 +++++++++++++++++++++++++++ g/encoding/gxml/gxml.go | 10 ++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/g/encoding/gcompress/gcompress.go b/g/encoding/gcompress/gcompress.go index 57633a7eb..859f5b8cd 100644 --- a/g/encoding/gcompress/gcompress.go +++ b/g/encoding/gcompress/gcompress.go @@ -10,6 +10,7 @@ package gcompress import ( "bytes" "compress/zlib" + "compress/gzip" "io" ) @@ -38,4 +39,30 @@ func UnZlib(data []byte) []byte { } io.Copy(&out, r) return out.Bytes() +} + +//做gzip解压缩 +func UnGzip(data []byte) []byte { + var buf bytes.Buffer + content := bytes.NewReader(data) + zipdata, err := gzip.NewReader(content) + if err != nil { + return nil + } + io.Copy(&buf, zipdata) + zipdata.Close() + return buf.Bytes() +} + +//做gzip压缩 +func Gzip(data []byte) []byte { + var buf bytes.Buffer + zip := gzip.NewWriter(&buf) + _, err := zip.Write(data) + if err != nil { + return nil + } + zip.Close() + + return buf.Bytes() } \ No newline at end of file diff --git a/g/encoding/gxml/gxml.go b/g/encoding/gxml/gxml.go index b2e441057..f36639106 100644 --- a/g/encoding/gxml/gxml.go +++ b/g/encoding/gxml/gxml.go @@ -18,6 +18,10 @@ import ( // 将XML内容解析为map变量 func Decode(xmlbyte []byte) (map[string]interface{}, error) { + //@author wenzi1 + //@date 20180529 + //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 + //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 { charsetReader := func(charset string, input io.Reader) (io.Reader, error) { reader := input @@ -47,10 +51,8 @@ func EncodeWithIndent(v map[string]interface{}, rootTag...string) ([]byte, error // XML格式内容直接转换为JSON格式内容 func ToJson(xmlbyte []byte) ([]byte, error) { - //input, _ := ioutil.ReadAll(bytes.NewReader(xmlbyte)) - //input:= bytes.NewReader(xmlbyte) - //charset := "UTF-8" - //@wenzi1 20180529 + //@author wenzi1 + //@date 20180529 //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 { From 4029e4ab8746fe50751ca6b5fe9d2aaba5da904f Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Mon, 4 Jun 2018 12:32:32 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E9=9B=86=E8=BD=AC=E6=8D=A2=E7=9A=84=E5=B0=81=E8=A3=85?= =?UTF-8?q?=EF=BC=8Cgxml=E5=8C=85=E4=B8=AD=E4=BD=BF=E7=94=A8=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E7=9A=84=E5=AD=97=E7=AC=A6=E9=9B=86=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E5=8C=85=E6=9D=A5=E5=81=9A=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gcharset/gcharset.go | 57 ++++++++++++++++ g/encoding/gcharset/gcharset_test.go | 99 ++++++++++++++++++++++++++++ g/encoding/gxml/gxml.go | 75 ++++++++++----------- 3 files changed, 189 insertions(+), 42 deletions(-) create mode 100644 g/encoding/gcharset/gcharset.go create mode 100644 g/encoding/gcharset/gcharset_test.go diff --git a/g/encoding/gcharset/gcharset.go b/g/encoding/gcharset/gcharset.go new file mode 100644 index 000000000..63e53f644 --- /dev/null +++ b/g/encoding/gcharset/gcharset.go @@ -0,0 +1,57 @@ +// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. + +//gcharset +package gcharset + +import ( + "github.com/axgle/mahonia" + "errors" + "fmt" +) + +//获取支持的字符集 +func GetCharset(name string) (*mahonia.Charset, error) { + s := mahonia.GetCharset(name) + if s == nil { + return nil, errors.New(fmt.Sprintf("not support charset: %s", name)) + } + return s, nil +} +//2个字符集之间的转换 +func Convert(dstCharset string, srcCharset string, src string) (dst string, err error) { + s := mahonia.GetCharset(srcCharset) + if s == nil { + return "", errors.New(fmt.Sprintf("not support charset:%s", srcCharset)) + } + + d := mahonia.GetCharset(dstCharset) + if d == nil { + return "", errors.New(fmt.Sprintf("not support charset:%s", dstCharset)) + } + + srctmp := src + if s.Name != "UTF-8" { + srctmp = s.NewDecoder().ConvertString(srctmp) + } + + dst = srctmp + if d.Name != "UTF-8" { + dst = d.NewEncoder().ConvertString(dst) + } + + return dst, nil +} + +//指定字符集转UTF8 +func ToUTF8(charset string, src string) (dst string, err error) { + return Convert("UTF-8", charset, src) +} + +//UTF8转指定字符集 +func UTF8To(charset string, src string) (dst string, err error) { + return Convert(charset, "UTF-8", src) +} \ No newline at end of file diff --git a/g/encoding/gcharset/gcharset_test.go b/g/encoding/gcharset/gcharset_test.go new file mode 100644 index 000000000..287f60ce7 --- /dev/null +++ b/g/encoding/gcharset/gcharset_test.go @@ -0,0 +1,99 @@ +package gcharset_test + +import ( + "testing" + "gitee.com/wenzi1/gf/g/encoding/gcharset" +) + + +var testData = []struct { + utf8, other, otherEncoding string +}{ + {"Résumé", "Résumé", "utf8"}, + {"Résumé", "R\xe9sum\xe9", "latin-1"}, + {"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"}, + {"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"}, + {"これは漢字です。", "\xfe\xff0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16"}, + {"𝄢𝄞𝄪𝄫", "\xfe\xff\xd8\x34\xdd\x22\xd8\x34\xdd\x1e\xd8\x34\xdd\x2a\xd8\x34\xdd\x2b", "UTF-16"}, + {"Hello, world", "Hello, world", "ASCII"}, + {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"}, + {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"}, + {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"}, + {"latviešu", "latvie\xf0u", "ISO-8859-13"}, + {"Seònaid", "Se\xf2naid", "ISO-8859-14"}, + {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"}, + {"românește", "rom\xe2ne\xbate", "ISO-8859-16"}, + {"nutraĵo", "nutra\xbco", "ISO-8859-3"}, + {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"}, + {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"}, + {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"}, + {"Kağan", "Ka\xf0an", "ISO-8859-9"}, + {"Résumé", "R\x8esum\x8e", "macintosh"}, + {"Gdańsk", "Gda\xf1sk", "windows-1250"}, + {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"}, + {"Résumé", "R\xe9sum\xe9", "windows-1252"}, + {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"}, + {"Kağan", "Ka\xf0an", "windows-1254"}, + {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"}, + {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"}, + {"latviešu", "latvie\xf0u", "windows-1257"}, + {"Việt", "Vi\xea\xf2t", "windows-1258"}, + {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"}, + {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"}, + {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"}, + {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"}, + {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"}, + {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"}, + {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"}, + {"㧯", "\x82\x31\x89\x38", "gb18030"}, + {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"}, + {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"}, +} + +func TestDecode(t *testing.T) { + for _, data := range testData { + str := "" + str, err := gcharset.Convert("UTF-8", data.otherEncoding, data.other) + if err != nil { + t.Errorf("Could not create decoder for %v", err) + continue + } + + if str != data.utf8 { + t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8) + } + } +} + + +func TestEncode(t *testing.T) { + for _, data := range testData { + str := "" + str, err := gcharset.Convert(data.otherEncoding, "UTF-8", data.utf8) + if err != nil { + t.Errorf("Could not create decoder for %v", err) + continue + } + + if str != data.other { + t.Errorf("Unexpected value: %#v (expected %#v)", str, data.other) + } + } +} + +func TestConvert(t *testing.T) { + srcCharset := "big5" + src := "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed" + dstCharset := "gbk" + dst := "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed" + + str, err := gcharset.Convert(dstCharset, srcCharset, src) + if err != nil { + t.Errorf("convert error. %v", err) + return + } + + if str != dst { + t.Errorf("unexpected value:%#v (expected %#v)", str, dst) + } +} \ No newline at end of file diff --git a/g/encoding/gxml/gxml.go b/g/encoding/gxml/gxml.go index f36639106..29ac497ab 100644 --- a/g/encoding/gxml/gxml.go +++ b/g/encoding/gxml/gxml.go @@ -11,32 +11,13 @@ import ( "github.com/clbanning/mxj" "encoding/xml" "io" - "strings" - "golang.org/x/text/transform" - "golang.org/x/text/encoding/simplifiedchinese" + "gitee.com/johng/gf/g/encoding/gcharset" + "gitee.com/johng/gf/g/util/gregx" ) // 将XML内容解析为map变量 func Decode(xmlbyte []byte) (map[string]interface{}, error) { - //@author wenzi1 - //@date 20180529 - //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 - //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 - if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 { - charsetReader := func(charset string, input io.Reader) (io.Reader, error) { - reader := input - switch charset { - case "GBK": - reader = transform.NewReader(input,simplifiedchinese.GBK.NewDecoder()) - case "GB18030": - reader = transform.NewReader(input, simplifiedchinese.GB18030.NewDecoder()) - default: - reader = input - } - return reader, nil - } - mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader} - } + Prepare(xmlbyte) return mxj.NewMapXml(xmlbyte) } @@ -51,30 +32,40 @@ func EncodeWithIndent(v map[string]interface{}, rootTag...string) ([]byte, error // XML格式内容直接转换为JSON格式内容 func ToJson(xmlbyte []byte) ([]byte, error) { - //@author wenzi1 - //@date 20180529 - //XML中的encoding如果非UTF-8时mxj包中会自动转换，但是自动转换时需要调用方提供转换的方法,这里提供一个空方法 - //所以如果涉及到字符集转换那么需要用户自行转为utf8时再调用该方法 - if strings.Index(string(xmlbyte), "encoding=\"UTF-8\"") == -1 { - charsetReader := func(charset string, input io.Reader) (io.Reader, error) { - reader := input - switch charset { - case "GBK": - reader = transform.NewReader(input,simplifiedchinese.GBK.NewDecoder()) - case "GB18030": - reader = transform.NewReader(input, simplifiedchinese.GB18030.NewDecoder()) - default: - reader = input - } - return reader, nil - } - mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader} - } + Prepare(xmlbyte) mv, err := mxj.NewMapXml(xmlbyte) - if err == nil { return mv.Json() } else { return nil, err } +} + +//XML字符集预处理 +//@author wenzi1 +//@date 20180604 +func Prepare(xmlbyte []byte) error { + patten := "<\\?xml\\s+.*?\\s+encoding=\"(.*?)\"" + charsetReader := func(charset string, input io.Reader) (io.Reader, error) { + reader, err := gcharset.GetCharset(charset) + if err != nil { + return nil, err + } + return reader.NewDecoder().NewReader(input), nil + } + + matchStr, err := gregx.MatchString(patten, string(xmlbyte)) + if err != nil { + return err + } + + charset, err := gcharset.GetCharset(matchStr[1]) + if err != nil { + return err + } + + if charset.Name != "UTF-8" { + mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader} + } + return nil } \ No newline at end of file From 3c610c4111222f9ad4c674a7e8b2af3e9fbdd8e7 Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Mon, 4 Jun 2018 12:35:20 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E9=9B=86=E8=BD=AC=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gcharset/gcharset.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/g/encoding/gcharset/gcharset.go b/g/encoding/gcharset/gcharset.go index 63e53f644..b67858c8e 100644 --- a/g/encoding/gcharset/gcharset.go +++ b/g/encoding/gcharset/gcharset.go @@ -5,6 +5,9 @@ // You can obtain one at https://gitee.com/johng/gf. //gcharset +//@author wenzi1 +//@date 20180604 + package gcharset import ( From 32e4aa8d8484e8a878acd232b279a5b0c80a1809 Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Wed, 6 Jun 2018 14:12:40 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E5=AE=8C=E5=96=84gxml=E4=B8=AD=E5=8C=B9?= =?UTF-8?q?=E9=85=8Dencoding=E7=9A=84=E5=A4=84=E7=90=86=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0gcharset=E5=8C=85=E7=9A=84=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gcharset/gcharset.go | 5 ++++- g/encoding/gxml/gxml.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/g/encoding/gcharset/gcharset.go b/g/encoding/gcharset/gcharset.go index b67858c8e..4a2490759 100644 --- a/g/encoding/gcharset/gcharset.go +++ b/g/encoding/gcharset/gcharset.go @@ -7,7 +7,8 @@ //gcharset //@author wenzi1 //@date 20180604 - +//字符集转换方法. +//使用mahonia实现的字符集转换方法，支持的字符集包括常见的utf8/UTF-16/UTF-16LE/macintosh/big5/gbk/gb18030,支持的全量字符集可以参考mahonia包 package gcharset import ( @@ -16,6 +17,8 @@ import ( "fmt" ) + + //获取支持的字符集 func GetCharset(name string) (*mahonia.Charset, error) { s := mahonia.GetCharset(name) diff --git a/g/encoding/gxml/gxml.go b/g/encoding/gxml/gxml.go index 29ac497ab..ac16eb4dc 100644 --- a/g/encoding/gxml/gxml.go +++ b/g/encoding/gxml/gxml.go @@ -11,7 +11,7 @@ import ( "github.com/clbanning/mxj" "encoding/xml" "io" - "gitee.com/johng/gf/g/encoding/gcharset" + "gitee.com/wenzi1/gf/g/encoding/gcharset" "gitee.com/johng/gf/g/util/gregx" ) @@ -45,7 +45,7 @@ func ToJson(xmlbyte []byte) ([]byte, error) { //@author wenzi1 //@date 20180604 func Prepare(xmlbyte []byte) error { - patten := "<\\?xml\\s+.*?\\s+encoding=\"(.*?)\"" + patten := "<\\?xml\\s+version\\s*=.*?\\s+encoding\\s*=\\s*[\\'|\"](.*?)[\\'|\"]\\s*\\?\\s*>" charsetReader := func(charset string, input io.Reader) (io.Reader, error) { reader, err := gcharset.GetCharset(charset) if err != nil { From 0d15c9f8388248c28d61aa5d06110e0a5c1ce726 Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Wed, 6 Jun 2018 14:37:38 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dgxml=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E9=9B=86=E8=BD=AC=E6=8D=A2=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gxml/gxml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/g/encoding/gxml/gxml.go b/g/encoding/gxml/gxml.go index ac16eb4dc..04ef17bb7 100644 --- a/g/encoding/gxml/gxml.go +++ b/g/encoding/gxml/gxml.go @@ -11,7 +11,7 @@ import ( "github.com/clbanning/mxj" "encoding/xml" "io" - "gitee.com/wenzi1/gf/g/encoding/gcharset" + "gitee.com/johng/gf/g/encoding/gcharset" "gitee.com/johng/gf/g/util/gregx" ) From 995c10cf2bf071dde2c321ed2905a1d932efee5b Mon Sep 17 00:00:00 2001 From: wenzi1 Date: Wed, 6 Jun 2018 14:58:50 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E5=88=A0=E9=99=A4gcharset=E4=B8=AD?= =?UTF-8?q?=E7=9A=84getcharset=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- g/encoding/gcharset/gcharset.go | 9 --------- g/encoding/gxml/gxml.go | 16 +++++++++------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/g/encoding/gcharset/gcharset.go b/g/encoding/gcharset/gcharset.go index 4a2490759..b2829e25b 100644 --- a/g/encoding/gcharset/gcharset.go +++ b/g/encoding/gcharset/gcharset.go @@ -18,15 +18,6 @@ import ( ) - -//获取支持的字符集 -func GetCharset(name string) (*mahonia.Charset, error) { - s := mahonia.GetCharset(name) - if s == nil { - return nil, errors.New(fmt.Sprintf("not support charset: %s", name)) - } - return s, nil -} //2个字符集之间的转换 func Convert(dstCharset string, srcCharset string, src string) (dst string, err error) { s := mahonia.GetCharset(srcCharset) diff --git a/g/encoding/gxml/gxml.go b/g/encoding/gxml/gxml.go index 04ef17bb7..b155a1045 100644 --- a/g/encoding/gxml/gxml.go +++ b/g/encoding/gxml/gxml.go @@ -11,8 +11,10 @@ import ( "github.com/clbanning/mxj" "encoding/xml" "io" - "gitee.com/johng/gf/g/encoding/gcharset" "gitee.com/johng/gf/g/util/gregx" + "github.com/axgle/mahonia" + "errors" + "fmt" ) // 将XML内容解析为map变量 @@ -47,9 +49,9 @@ func ToJson(xmlbyte []byte) ([]byte, error) { func Prepare(xmlbyte []byte) error { patten := "<\\?xml\\s+version\\s*=.*?\\s+encoding\\s*=\\s*[\\'|\"](.*?)[\\'|\"]\\s*\\?\\s*>" charsetReader := func(charset string, input io.Reader) (io.Reader, error) { - reader, err := gcharset.GetCharset(charset) - if err != nil { - return nil, err + reader := mahonia.GetCharset(charset) + if reader == nil { + return nil, errors.New(fmt.Sprintf("not support charset:%s", charset)) } return reader.NewDecoder().NewReader(input), nil } @@ -59,9 +61,9 @@ func Prepare(xmlbyte []byte) error { return err } - charset, err := gcharset.GetCharset(matchStr[1]) - if err != nil { - return err + charset := mahonia.GetCharset(matchStr[1]) + if charset == nil { + return errors.New(fmt.Sprintf("not support charset:%s", matchStr[1])) } if charset.Name != "UTF-8" {