!12 完善gxml中匹配encoding的处理，增加gcharset包的说明

Merge pull request !12 from 蚊子/master
2026-06-07 10:22:11 +08:00 · 2018-06-07 08:27:41 +08:00
parent a553c288d6 995c10cf2b
commit 3af59aff40
4 changed files with 219 additions and 1 deletions
--- a/g/encoding/gcharset/gcharset.go
+++ b/g/encoding/gcharset/gcharset.go
@ -0,0 +1,54 @@
+// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
+//
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT was not distributed with this file,
+// You can obtain one at https://gitee.com/johng/gf.
+
+//gcharset
+//@author wenzi1
+//@date   20180604
+//字符集转换方法.
+//使用mahonia实现的字符集转换方法，支持的字符集包括常见的utf8/UTF-16/UTF-16LE/macintosh/big5/gbk/gb18030,支持的全量字符集可以参考mahonia包
+package gcharset
+
+import (
+	"github.com/axgle/mahonia"
+	"errors"
+	"fmt"
+)
+
+
+//2个字符集之间的转换
+func Convert(dstCharset string, srcCharset string, src string) (dst string, err error) {
+	s := mahonia.GetCharset(srcCharset)
+	if 	s == nil {
+		return "", errors.New(fmt.Sprintf("not support charset:%s", srcCharset))
+	}
+
+	d := mahonia.GetCharset(dstCharset)
+	if d == nil {
+		return "", errors.New(fmt.Sprintf("not support charset:%s", dstCharset))
+	}
+	
+	srctmp := src
+	if s.Name != "UTF-8" {
+		srctmp = s.NewDecoder().ConvertString(srctmp)
+	}
+	
+	dst = srctmp
+	if d.Name != "UTF-8" {
+		dst = d.NewEncoder().ConvertString(dst)
+	}
+	
+	return dst, nil
+}
+
+//指定字符集转UTF8
+func ToUTF8(charset string, src string) (dst string, err error) {
+	return Convert("UTF-8", charset, src)
+}
+
+//UTF8转指定字符集
+func UTF8To(charset string, src string) (dst string, err error) {
+	return  Convert(charset, "UTF-8", src)
+}
--- a/g/encoding/gcharset/gcharset_test.go
+++ b/g/encoding/gcharset/gcharset_test.go
@ -0,0 +1,99 @@
+package gcharset_test
+
+import (
+	"testing"
+	"gitee.com/wenzi1/gf/g/encoding/gcharset"
+)
+
+
+var testData = []struct {
+	utf8, other, otherEncoding string
+}{
+	{"Résumé", "Résumé", "utf8"},
+	{"Résumé", "R\xe9sum\xe9", "latin-1"},
+	{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
+	{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
+	{"これは漢字です。", "\xfe\xff0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16"},
+	{"𝄢𝄞𝄪𝄫", "\xfe\xff\xd8\x34\xdd\x22\xd8\x34\xdd\x1e\xd8\x34\xdd\x2a\xd8\x34\xdd\x2b", "UTF-16"},
+	{"Hello, world", "Hello, world", "ASCII"},
+	{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
+	{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
+	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
+	{"latviešu", "latvie\xf0u", "ISO-8859-13"},
+	{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
+	{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
+	{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
+	{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
+	{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
+	{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
+	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
+	{"Kağan", "Ka\xf0an", "ISO-8859-9"},
+	{"Résumé", "R\x8esum\x8e", "macintosh"},
+	{"Gdańsk", "Gda\xf1sk", "windows-1250"},
+	{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
+	{"Résumé", "R\xe9sum\xe9", "windows-1252"},
+	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
+	{"Kağan", "Ka\xf0an", "windows-1254"},
+	{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
+	{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
+	{"latviešu", "latvie\xf0u", "windows-1257"},
+	{"Việt", "Vi\xea\xf2t", "windows-1258"},
+	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
+	{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
+	{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
+	{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
+	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
+	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
+	{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
+	{"㧯", "\x82\x31\x89\x38", "gb18030"},
+	{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
+	{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
+}
+
+func TestDecode(t *testing.T) {
+	for _, data := range testData {
+		str := ""
+		str, err := gcharset.Convert("UTF-8", data.otherEncoding, data.other)
+		if err != nil {
+			t.Errorf("Could not create decoder for %v", err)
+			continue
+		}
+		
+		if str != data.utf8 {
+			t.Errorf("Unexpected value: %#v (expected %#v)", str, data.utf8)
+		}
+	}
+}
+
+
+func TestEncode(t *testing.T) {
+	for _, data := range testData {
+		str := ""
+		str, err := gcharset.Convert(data.otherEncoding, "UTF-8", data.utf8)
+		if err != nil {
+			t.Errorf("Could not create decoder for %v", err)
+			continue
+		}
+
+		if str != data.other {
+			t.Errorf("Unexpected value: %#v (expected %#v)", str, data.other)
+		}
+	}
+}
+
+func TestConvert(t *testing.T) {
+	srcCharset := "big5"
+	src := "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed"
+	dstCharset := "gbk"
+	dst := "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed"
+
+	str, err := gcharset.Convert(dstCharset, srcCharset, src)
+	if err != nil {
+		t.Errorf("convert error. %v", err)
+		return
+	}
+
+	if str != dst {
+		t.Errorf("unexpected value:%#v (expected %#v)", str, dst)
+	}
+}
--- a/g/encoding/gcompress/gcompress.go
+++ b/g/encoding/gcompress/gcompress.go
@ -10,6 +10,7 @@ package gcompress
 import (
    "bytes"
    "compress/zlib"
+    "compress/gzip"
    "io"
 )

@ -38,4 +39,30 @@ func UnZlib(data []byte) []byte {
    }
    io.Copy(&out, r)
    return out.Bytes()
+}
+
+//做gzip解压缩
+func UnGzip(data []byte) []byte {
+    var buf bytes.Buffer
+	content := bytes.NewReader(data)
+	zipdata, err := gzip.NewReader(content)
+	if err != nil {
+		return nil
+	}
+	io.Copy(&buf, zipdata)
+	zipdata.Close()
+	return buf.Bytes()
+}
+
+//做gzip压缩
+func Gzip(data []byte) []byte {
+    var buf bytes.Buffer
+	zip := gzip.NewWriter(&buf)
+	_, err := zip.Write(data)
+	if err != nil {
+		return nil
+	}
+	zip.Close()
+
+	return buf.Bytes()
 }
--- a/g/encoding/gxml/gxml.go
+++ b/g/encoding/gxml/gxml.go
@ -9,10 +9,17 @@ package gxml

 import (
    "github.com/clbanning/mxj"
+    "encoding/xml"
+    "io"
+	"gitee.com/johng/gf/g/util/gregx"
+	"github.com/axgle/mahonia"
+	"errors"
+	"fmt"
 )

 // 将XML内容解析为map变量
 func Decode(xmlbyte []byte) (map[string]interface{}, error) {
+    Prepare(xmlbyte)
    return mxj.NewMapXml(xmlbyte)
 }

@ -27,9 +34,40 @@ func EncodeWithIndent(v map[string]interface{}, rootTag...string) ([]byte, error

 // XML格式内容直接转换为JSON格式内容
 func ToJson(xmlbyte []byte) ([]byte, error) {
-    if mv, err := mxj.NewMapXml(xmlbyte); err == nil {
+    Prepare(xmlbyte)
+	mv, err := mxj.NewMapXml(xmlbyte)
+	if err == nil {
        return mv.Json()
    } else {
        return nil, err
    }
+}
+
+//XML字符集预处理
+//@author wenzi1 
+//@date 20180604
+func Prepare(xmlbyte []byte) error {
+	patten := "<\\?xml\\s+version\\s*=.*?\\s+encoding\\s*=\\s*[\\'|\"](.*?)[\\'|\"]\\s*\\?\\s*>"
+	charsetReader := func(charset string, input io.Reader) (io.Reader, error) {
+		reader := mahonia.GetCharset(charset)
+		if reader == nil {
+			return nil, errors.New(fmt.Sprintf("not support charset:%s", charset))
+		}
+		return reader.NewDecoder().NewReader(input), nil
+	}
+
+	matchStr, err := gregx.MatchString(patten, string(xmlbyte))
+	if err != nil {
+		return err
+	}
+
+	charset := mahonia.GetCharset(matchStr[1])
+	if charset == nil {
+		return errors.New(fmt.Sprintf("not support charset:%s", matchStr[1]))
+	}
+
+	if charset.Name != "UTF-8" {
+		mxj.CustomDecoder = &xml.Decoder{Strict:false,CharsetReader:charsetReader}
+	}
+	return nil
 }