From bcc9153991a00e3743433d8f63ce2698d8818cfe Mon Sep 17 00:00:00 2001 From: linx Date: Mon, 22 Jun 2020 21:12:11 +0800 Subject: [PATCH 1/3] add a SetProxy function and a chaining function Proxy for ghttp.Client to do HTTP request via proxy. #285 --- go.mod | 1 + net/ghttp/ghttp_client_chain.go | 14 ++++++++ net/ghttp/ghttp_client_config.go | 62 ++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/go.mod b/go.mod index 1afa28a7d..682fb3f61 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/mattn/go-runewidth v0.0.9 // indirect github.com/olekukonko/tablewriter v0.0.1 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/net v0.0.0-20200602114024-627f9648deb9 golang.org/x/text v0.3.2 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c ) diff --git a/net/ghttp/ghttp_client_chain.go b/net/ghttp/ghttp_client_chain.go index 9191d67bf..1829885be 100644 --- a/net/ghttp/ghttp_client_chain.go +++ b/net/ghttp/ghttp_client_chain.go @@ -135,3 +135,17 @@ func (c *Client) Retry(retryCount int, retryInterval time.Duration) *Client { newClient.SetRetry(retryCount, retryInterval) return c } + +// Proxy is a chaining function, +// which sets proxy for next request. +// Make sure you pass the correct `proxyURL`. +// The correct pattern is like `http://USER:PASSWORD@IP:PORT` or `socks5://USER:PASSWORD@IP:PORT`. +// Only `http` and `socks5` proxies are supported currently. +func (c *Client) Proxy(proxyURL string) *Client { + newClient := c + if c.parent == nil { + newClient = c.Clone() + } + newClient.SetProxy(proxyURL) + return c +} diff --git a/net/ghttp/ghttp_client_config.go b/net/ghttp/ghttp_client_config.go index 7d94ed3bc..16e10c3f2 100644 --- a/net/ghttp/ghttp_client_config.go +++ b/net/ghttp/ghttp_client_config.go @@ -10,7 +10,11 @@ import ( "context" "crypto/tls" "github.com/gogf/gf/text/gstr" + "golang.org/x/net/proxy" + "net" "net/http" + "net/url" + "strings" "time" "github.com/gogf/gf/text/gregex" @@ -158,3 +162,61 @@ func (c *Client) SetRedirectLimit(redirectLimit int) *Client { } return c } + +// SetProxy set proxy for the client. +// This func will do nothing when the parameter `proxyURL` is empty or in wrong pattern. +// The correct pattern is like `http://USER:PASSWORD@IP:PORT` or `socks5://USER:PASSWORD@IP:PORT`. +// Only `http` and `socks5` proxies are supported currently. +func (c *Client) SetProxy(proxyURL string) { + if strings.TrimSpace(proxyURL) == "" { + return + } + _proxy, err := url.Parse(proxyURL) + if err != nil { + return + } + if _proxy.Scheme == "http" { + c.Transport = &http.Transport{ + Proxy: http.ProxyURL(_proxy), + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + } + } else { + var auth = &proxy.Auth{} + user := _proxy.User.Username() + + if user != "" { + auth.User = user + password, hasPassword := _proxy.User.Password() + if hasPassword && password != "" { + auth.Password = password + } + } else { + auth = nil + } + // refer to the source code, error is always nil + dialer, err := proxy.SOCKS5( + "tcp", + _proxy.Host, + auth, + &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }, + ) + if err != nil { + return + } + c.Transport = &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) { + return dialer.Dial(network, addr) + }, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + } + } +} From 977827e4534109de44da81a05567e346600b4385 Mon Sep 17 00:00:00 2001 From: linx Date: Fri, 26 Jun 2020 13:59:33 +0800 Subject: [PATCH 2/3] add examples for SetProxy function and Proxy chaining function --- net/ghttp/ghttp_client_config.go | 23 +++------ net/ghttp/ghttp_z_example_test.go | 77 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/net/ghttp/ghttp_client_config.go b/net/ghttp/ghttp_client_config.go index 16e10c3f2..d7ebcdbdb 100644 --- a/net/ghttp/ghttp_client_config.go +++ b/net/ghttp/ghttp_client_config.go @@ -176,13 +176,7 @@ func (c *Client) SetProxy(proxyURL string) { return } if _proxy.Scheme == "http" { - c.Transport = &http.Transport{ - Proxy: http.ProxyURL(_proxy), - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - } + c.Transport.(*http.Transport).Proxy = http.ProxyURL(_proxy) } else { var auth = &proxy.Auth{} user := _proxy.User.Username() @@ -202,21 +196,16 @@ func (c *Client) SetProxy(proxyURL string) { _proxy.Host, auth, &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, + Timeout: 10 * time.Second, + KeepAlive: 10 * time.Second, }, ) if err != nil { return } - c.Transport = &http.Transport{ - DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) { - return dialer.Dial(network, addr) - }, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, + c.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (conn net.Conn, e error) { + return dialer.Dial(network, addr) } + //c.SetTimeout(10*time.Second) } } diff --git a/net/ghttp/ghttp_z_example_test.go b/net/ghttp/ghttp_z_example_test.go index 0a7b3203e..45e0c7411 100644 --- a/net/ghttp/ghttp_z_example_test.go +++ b/net/ghttp/ghttp_z_example_test.go @@ -7,8 +7,10 @@ package ghttp_test import ( + "fmt" "github.com/gogf/gf/frame/g" "github.com/gogf/gf/net/ghttp" + "time" ) func ExampleGetServer() { @@ -27,3 +29,78 @@ func ExampleClientResponse_RawDump() { } response.RawDump() } + +// ExampleClient_SetProxy a example for `ghttp.Client.SetProxy` method. +// please prepare two proxy server before running this example. +// http proxy server listening on `127.0.0.1:1081` +// socks5 proxy server listening on `127.0.0.1:1080` +func ExampleClient_SetProxy() { + // connect to a http proxy server + client := ghttp.NewClient() + client.SetProxy("http://127.0.0.1:1081") + client.SetTimeout(5 * time.Second) // it's suggested to set http client timeout + response, err := client.Get("https://api.ip.sb/ip") + if err != nil { + // err is not nil when your proxy server is down. + // eg. Get "https://api.ip.sb/ip": proxyconnect tcp: dial tcp 127.0.0.1:1087: connect: connection refused + fmt.Println(err) + } + response.RawDump() + // connect to a http proxy server which needs auth + client.SetProxy("http://user:password:127.0.0.1:1081") + client.SetTimeout(5 * time.Second) // it's suggested to set http client timeout + response, err = client.Get("https://api.ip.sb/ip") + if err != nil { + // err is not nil when your proxy server is down. + // eg. Get "https://api.ip.sb/ip": proxyconnect tcp: dial tcp 127.0.0.1:1087: connect: connection refused + fmt.Println(err) + } + response.RawDump() + + // connect to a socks5 proxy server + client.SetProxy("socks5://127.0.0.1:1080") + client.SetTimeout(5 * time.Second) // it's suggested to set http client timeout + response, err = client.Get("https://api.ip.sb/ip") + if err != nil { + // err is not nil when your proxy server is down. + // eg. Get "https://api.ip.sb/ip": socks connect tcp 127.0.0.1:1087->api.ip.sb:443: dial tcp 127.0.0.1:1087: connect: connection refused + fmt.Println(err) + } + fmt.Println(response.RawResponse()) + + // connect to a socks5 proxy server which needs auth + client.SetProxy("socks5://user:password@127.0.0.1:1080") + client.SetTimeout(5 * time.Second) // it's suggested to set http client timeout + response, err = client.Get("https://api.ip.sb/ip") + if err != nil { + // err is not nil when your proxy server is down. + // eg. Get "https://api.ip.sb/ip": socks connect tcp 127.0.0.1:1087->api.ip.sb:443: dial tcp 127.0.0.1:1087: connect: connection refused + fmt.Println(err) + } + fmt.Println(response.RawResponse()) +} + +// ExampleClientChain_Proxy a chain version of example for `ghttp.Client.Proxy` method. +// please prepare two proxy server before running this example. +// http proxy server listening on `127.0.0.1:1081` +// socks5 proxy server listening on `127.0.0.1:1080` +// for more details, please refer to ExampleClient_SetProxy +func ExampleClientChain_Proxy() { + client := ghttp.NewClient() + response, err := client.Proxy("http://127.0.0.1:1081").Get("https://api.ip.sb/ip") + if err != nil { + // err is not nil when your proxy server is down. + // eg. Get "https://api.ip.sb/ip": proxyconnect tcp: dial tcp 127.0.0.1:1087: connect: connection refused + fmt.Println(err) + } + fmt.Println(response.RawResponse()) + + client2 := ghttp.NewClient() + response, err = client2.Proxy("socks5://127.0.0.1:1080").Get("https://api.ip.sb/ip") + if err != nil { + // err is not nil when your proxy server is down. + // eg. Get "https://api.ip.sb/ip": socks connect tcp 127.0.0.1:1087->api.ip.sb:443: dial tcp 127.0.0.1:1087: connect: connection refused + fmt.Println(err) + } + fmt.Println(response.RawResponse()) +} From f3f6adb03ae005cd1424efee62a7a2d41b0f5c84 Mon Sep 17 00:00:00 2001 From: kirile Date: Sat, 4 Jul 2020 16:43:30 +0800 Subject: [PATCH 3/3] read Client's timeout for net.Dialer make sure Client.Transport is *http.Transport --- net/ghttp/ghttp_client_config.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/net/ghttp/ghttp_client_config.go b/net/ghttp/ghttp_client_config.go index d7ebcdbdb..207cfe868 100644 --- a/net/ghttp/ghttp_client_config.go +++ b/net/ghttp/ghttp_client_config.go @@ -176,7 +176,9 @@ func (c *Client) SetProxy(proxyURL string) { return } if _proxy.Scheme == "http" { - c.Transport.(*http.Transport).Proxy = http.ProxyURL(_proxy) + if _, ok := c.Transport.(*http.Transport); ok { + c.Transport.(*http.Transport).Proxy = http.ProxyURL(_proxy) + } } else { var auth = &proxy.Auth{} user := _proxy.User.Username() @@ -196,15 +198,17 @@ func (c *Client) SetProxy(proxyURL string) { _proxy.Host, auth, &net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 10 * time.Second, + Timeout: c.Client.Timeout, + KeepAlive: c.Client.Timeout, }, ) if err != nil { return } - c.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (conn net.Conn, e error) { - return dialer.Dial(network, addr) + if _, ok := c.Transport.(*http.Transport); ok { + c.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (conn net.Conn, e error) { + return dialer.Dial(network, addr) + } } //c.SetTimeout(10*time.Second) }