A fantastic HTTP request libarary used in Golang.

goz

A fantastic HTTP request library used in golang. Inspired by guzzle

Installation

go get -u github.com/idoubi/goz

Documentation

API documentation can be found here: https://godoc.org/github.com/idoubi/goz

Basic Usage

package main

import (
    "github.com/idoubi/goz"
)

func main() {
    cli := goz.NewClient()

	resp, err := cli.Get("http://127.0.0.1:8091/get")
	if err != nil {
		log.Fatalln(err)
	}

	fmt.Printf("%T", resp)
	// Output: *goz.Response
}

Query Params

  • query map
cli := goz.NewClient()

resp, err := cli.Get("http://127.0.0.1:8091/get-with-query", goz.Options{
    Query: map[string]interface{}{
        "key1": "value1",
        "key2": []string{"value21", "value22"},
        "key3": "333",
    },
})
if err != nil {
    log.Fatalln(err)
}

fmt.Printf("%s", resp.GetRequest().URL.RawQuery)
// Output: key1=value1&key2=value21&key2=value22&key3=333
  • query string
cli := goz.NewClient()

resp, err := cli.Get("http://127.0.0.1:8091/get-with-query?key0=value0", goz.Options{
    Query: "key1=value1&key2=value21&key2=value22&key3=333",
})
if err != nil {
    log.Fatalln(err)
}

fmt.Printf("%s", resp.GetRequest().URL.RawQuery)
// Output: key1=value1&key2=value21&key2=value22&key3=333

Post Data

  • post form
cli := goz.NewClient()

resp, err := cli.Post("http://127.0.0.1:8091/post-with-form-params", goz.Options{
    Headers: map[string]interface{}{
        "Content-Type": "application/x-www-form-urlencoded",
    },
    FormParams: map[string]interface{}{
        "key1": "value1",
        "key2": []string{"value21", "value22"},
        "key3": "333",
    },
})
if err != nil {
    log.Fatalln(err)
}

body, _ := resp.GetBody()
fmt.Println(body)
// Output: form params:{"key1":["value1"],"key2":["value21","value22"],"key3":["333"]}
  • post json
cli := goz.NewClient()

resp, err := cli.Post("http://127.0.0.1:8091/post-with-json", goz.Options{
    Headers: map[string]interface{}{
        "Content-Type": "application/json",
    },
    JSON: struct {
        Key1 string   `json:"key1"`
        Key2 []string `json:"key2"`
        Key3 int      `json:"key3"`
    }{"value1", []string{"value21", "value22"}, 333},
})
if err != nil {
    log.Fatalln(err)
}

body, _ := resp.GetBody()
fmt.Println(body)
// Output: json:{"key1":"value1","key2":["value21","value22"],"key3":333}

Request Headers

cli := goz.NewClient()

resp, err := cli.Post("http://127.0.0.1:8091/post-with-headers", goz.Options{
    Headers: map[string]interface{}{
        "User-Agent": "testing/1.0",
        "Accept":     "application/json",
        "X-Foo":      []string{"Bar", "Baz"},
    },
})
if err != nil {
    log.Fatalln(err)
}

headers := resp.GetRequest().Header["X-Foo"]
fmt.Println(headers)
// Output: [Bar Baz]

Response

cli := goz.NewClient()
resp, err := cli.Get("http://127.0.0.1:8091/get")
if err != nil {
    log.Fatalln(err)
}

body, err := resp.GetBody()
if err != nil {
    log.Fatalln(err)
}
fmt.Printf("%T", body)
// Output: goz.ResponseBody

part := body.Read(30)
fmt.Printf("%T", part)
// Output: []uint8

contents := body.GetContents()
fmt.Printf("%T", contents)
// Output: string

fmt.Println(resp.GetStatusCode())
// Output: 200

fmt.Println(resp.GetReasonPhrase())
// Output: OK

headers := resp.GetHeaders()
fmt.Printf("%T", headers)
// Output: map[string][]string

flag := resp.HasHeader("Content-Type")
fmt.Printf("%T", flag)
// Output: bool

header := resp.GetHeader("content-type")
fmt.Printf("%T", header)
// Output: []string
    
headerLine := resp.GetHeaderLine("content-type")
fmt.Printf("%T", headerLine)
// Output: string

Proxy

cli := goz.NewClient()

resp, err := cli.Get("https://www.fbisb.com/ip.php", goz.Options{
    Timeout: 5.0,
    Proxy:   "http://127.0.0.1:1087",
})
if err != nil {
    log.Fatalln(err)
}

fmt.Println(resp.GetStatusCode())
// Output: 200

Timeout

cli := goz.NewClient(goz.Options{
    Timeout: 0.9,
})
resp, err := cli.Get("http://127.0.0.1:8091/get-timeout")
if err != nil {
    if resp.IsTimeout() {
        fmt.Println("timeout")
        // Output: timeout
        return
    }
}

fmt.Println("not timeout")

License

MIT

Copyright (c) 2017-present, idoubi

Owner
艾逗笔
有逻辑的脑子万里挑一。
艾逗笔
Comments
  • 与现有版本不兼容,请谨慎考虑合并

    与现有版本不兼容,请谨慎考虑合并

    • 相比原版变化:

    1.增加了文件下载功能
    2.GetBody() 返回io.ReaderCloser ,而不是原版本中的文本格式数据。
    3.GetBody() 将专门负责处理流式数据,因此代码逻辑处理完毕,必须使用io.ReaderCloser 接口提供的函数关闭。
    4.原版本的GetBody()被现有版本GetContents()代替,由于文本数据,一次性返回,因此不需要手动关闭,程序会自动释放相关io资源。
    5.删除、简化了原版本中为处理数据类型转换而定义的ResponseBody,本版本中使用系统系统默认的数据类型转换即可,简单快捷。
    6.增强原版本中表单参数只能传递string、[]string的问题,该版本支持数字、文本、[]string等。
    7.增加请求时浏览器自带的默认参数,完全模拟浏览器发送数据。
    8.增加被请求的网站数据编码自动转换功能(采集网站时不需要考虑对方是站点的编码类型,gbk系列、utf8全程自动转换)。

  • post的数据无法获取

    post的数据无法获取

    step3 step4发起的step2请求,都可以正常解析,step1发起的无法解析表单数据

    package main
    
    import (
    	"encoding/json"
    	"fmt"
    	"github.com/gin-gonic/gin"
    	"github.com/mikemintang/go-curl"
    	"github.com/rifflock/lfshook"
    	"github.com/sirupsen/logrus"
    	"io/ioutil"
    	"net/http"
    	"net/url"
    	"os"
    	"strings"
    )
    
    func main() {
    	pathMap := lfshook.PathMap{
    		logrus.InfoLevel:  "./info.log",
    		logrus.ErrorLevel: "./error.log",
    	}
    	Log := logrus.New()
    	Log.Hooks.Add(lfshook.NewHook(pathMap, &logrus.JSONFormatter{}));
    
    	router := gin.New()
    	router.GET("/step1", func(c *gin.Context) {
    		header := map[string]string{
    			"Content-Type": "application/x-www-form-urlencoded",
    			//"X-Requested-With": "XMLHttpRequest",
    		}
    		url := "http://localhost:81/step2";
    		request := curl.NewRequest()
    		postData := map[string]interface{}{
    			"name": "zhangsan",
    			"age":  18,
    		};
    		resp, err := request.SetUrl(url).SetHeaders(header).SetPostData(postData).Post();
    		if err != nil {
    			Log.Error("发生错误 => ", err);
    		}
    
    		Log.Info("请求成功,响应内容为 => ", resp.Body)
    		c.String(http.StatusOK, resp.Body);
    	});
    	router.POST("/step2", func(c *gin.Context) {
    		// 获取请求数据
    		header := c.Request.Header;
    		header_tmp, _ := json.Marshal(header)
    		header_str := string(header_tmp)
    
    		form := c.Request.Form;
    		name := c.PostForm("name");
    		age := c.Request.PostFormValue("age");
    		fmt.Fprintln(os.Stdout, "form name数据 => ", name);
    		fmt.Fprintln(os.Stdout, "form age数据 => ", age);
    		form_tmp, _ := json.Marshal(form);
    		form_str := string(form_tmp)
    		c.String(http.StatusOK, "请求方式 => "+c.Request.Method)
    		c.String(http.StatusOK, "请求头 => "+header_str)
    		c.String(http.StatusOK, "form数据 => "+form_str)
    		c.String(http.StatusOK, "接收到的name数据 => "+name)
    		c.String(http.StatusOK, "接收到的age数据 => "+age)
    		age2, _ := c.GetPostForm("age");
    		c.String(http.StatusOK, "接收到的age数据 => "+age2)
    	})
    
    	router.GET("/step3", func(c *gin.Context) {
    		client := http.Client{}
    		url := "http://localhost:81/step2";
    		postdata_str := "name=zhangsan&age=18";
    		postdata := strings.NewReader(postdata_str);
    		request, _ := http.NewRequest("POST", url, postdata);
    		request.Header.Add("Content-Type", "application/x-www-form-urlencoded;");
    		//request.Header.Add("X-Requested-With", "XMLHttpRequest");
    		resp, err := client.Do(request);
    		if err != nil {
    			Log.Error("发生错误 => ", err);
    		}
    		defer resp.Body.Close();
    		Log.Info("请求成功,响应内容为 => ", resp.Body)
    		body, _ := ioutil.ReadAll(resp.Body)
    		c.String(http.StatusOK, string(body));
    	});
    
    	router.GET("/step4", func(c *gin.Context) {
    		client := &http.Client{}
    
    		postValues := url.Values{}
    		postValues.Add("name", "zhangsan")
    		postValues.Add("age", "18")
    
    		url := "http://localhost:81/step2";
    		resp, err := client.PostForm(url, postValues)
    		defer resp.Body.Close()
    		if err != nil {
    			fmt.Println(err.Error())
    		}
    		if resp.StatusCode == 200 {
    			body, _ := ioutil.ReadAll(resp.Body)
    			c.String(http.StatusOK, string(body));
    		}
    
    	});
    	router.Run(":81");
    }
    
    
  • 简单的get方法一直超时

    简单的get方法一直超时

    只使用了 url := "http://www.baidu.com" resp, err := req.SetUrl(url).Get() 无论url是什么都会显示 Get http://www.baidu.com: read tcp xxx.xxx.xxx.xxx: xxxx ->115.239.211.112:80: i/o timeout

  • 请添加下载文件代码

    请添加下载文件代码

    //  看一下这段代码,我已经调试完毕,下载文件使用的代码,看看怎么融合到你的这个包
    func  Down(resource_url string,sava_path  string) {
    	uri, err := url.ParseRequestURI(resource_url)
    	if err != nil {
    		panic("网址错误")
    	}
    
    	filename := path.Base(uri.Path)
    	client := &http.Client{};
    	client.Timeout = time.Second * 600 //设置超时时间
    	resp, err := client.Get(resource_url)
    	if err != nil {
    		panic(err)
    	}
    	if resp.ContentLength <= 0 {
    		log.Println("[*] Destination server does not support breakpoint download.")
    	}
    	raw := resp.Body
    	defer raw.Close()
    	reader := bufio.NewReaderSize(raw, 1024*50);  //相当于一个临时缓冲区(设置为可以单次存储50M的文件),每次读取以后就把原始数据重新加载一份,等待下一次读取
    
    	//根据目录创建镜像文件,等待数据写入
    	if !strings.HasSuffix(sava_path,"/"){
    		sava_path+="/"
    	}
    	fmt.Println(sava_path+filename)
    	file,err:=os.OpenFile(sava_path+filename,os.O_WRONLY |os.O_CREATE,0777)
    	if err != nil {
    		log.Panic("创建镜像文件失败,无法进行后续的写入操作"+err.Error())
    	}
    	writer := bufio.NewWriter(file)
    	buff := make([]byte, 32*1024)
    	for {
    		curr_read_size, reader_err := reader.Read(buff)
    		if curr_read_size > 0 {
    			write_size, write_err := writer.Write(buff[0:curr_read_size])
    			if write_err != nil {
    				log.Panic("写入发生错误"+write_err.Error(),"写入长度:",write_size)
    				break
    			}
    		}
    		// 读取结束
    		if reader_err == io.EOF {
    			writer.Flush()
    			break
    		}
    	}
    	fmt.Println("下载完毕!")
    }
    
    
    
    
  • 实现有问题

    实现有问题

    http.Request Close 参数没有设置 httpClient 会默认在下一次请求时复用同一个 tcp链接 然而此时对方已经关闭,可能会出现 EOF 错误

    参考:https://stackoverflow.com/questions/28046100/golang-http-concurrent-requests-post-eof

  • 是否 goroutine 安全?

    是否 goroutine 安全?

    同时在多个 goroutine 中使用 goz 下载文件时,会发生段错误

    panic: runtime error: invalid memory address or nil pointer dereference
    [signal SIGSEGV: segmentation violation code=0x1 addr=0x40 pc=0x67d7b9]
    
    goroutine 20255 [running]:
    github.com/idoubi/goz.(*Response).GetBody(0xc012042a60, 0x0, 0x0, 0x0, 0x0, 0x0)
    	/home/gs/go/pkg/mod/github.com/idoubi/[email protected]/response.go:46 +0x59
    main.DownloadFile(0xc011180240, 0x3d, 0xc010990740, 0x34, 0x2)
    	/home/gs/face/douban/douban.go:403 +0x35a
    main.DownloadPhotos.func1(0xc011180240, 0x3d, 0xc0119fe240, 0xc00b104000)
    	/home/gs/face/douban/douban.go:86 +0x12b
    created by main.DownloadPhotos
    	/home/gs/face/douban/douban.go:83 +0x433
    

    douban.go:403 行的代码是

    body, err := response.GetBody()
    

    也就是在调用 GetBody() 的时候会出错

    所以 goz 是 goroutine 安全的吗?如果不是的话还建议在 README 中说明,以便用户自己做并发控制

  • 第二次執行新增單元測試隔離功能

    第二次執行新增單元測試隔離功能

    在寫單元測試時,會把很多元件進行隔離,Goz 沒有隔離的能力,這裡又很喜歡用Goz,但是這個我無法寫單元測試 比如一個程式是有二個主程式A B組成時,這時會發生一個狀況,當A程式更改時,會造成B程式的單元測試瓦解。當B程式更改時,會造成A程式的單元測試故障,所以在單元測試時,隔離的能力對我來說是很重要 我對Goz程式進行最小幅度的更改,並自己寫一個單元測試去證明這個修改方式可以增強 Goz 的單元測試的能力,謝謝 圖片

  • 当响应header有多个set-cookie时会有问题

    当响应header有多个set-cookie时会有问题

    使用fmt.Println(response.Raw)获取,得到如下结果

    &{200 OK 200 HTTP/1.1 1 1 map[Cache-Control:[private, must-revalidate] Connection:[keep-alive] Content-Type:[text/html; charset=UTF-8] Date:[Thu, 30 May 2019 06:35:03 GMT] Expires:[-1] Pragma:[no-cache] Server:[nginx] Set-Cookie:[XSRF-TOKEN=eyJpdiI6ImVZOThBVFlRZ3MzZmNsaVJmczd6Z3c9PSIsInZhbHVlIjoiSUdiSU5MblF0XC83N25iUXliMjNsNUlqTFE0MXpJQ3RLdU1LeGMrR1dmWG9pT25cL2hrd2I2ZG5iQlVnQVNTblJzZENCcm1LcTlcL3RVelZ3RFNmOWtkYlE9PSIsIm1hYyI6IjU2ZWU2MGY5NGE1OWJhNjBiZjM5YTFhNDgwMzJkNjViNDkxYmI2NGEyZmUzZGQ0N2NkNjhmZDQ5NDg2YTcyOGIifQ%3D%3D; expires=Thu, 13-Jun-2019 06:35:03 GMT; Max-Age=1209600; path=/ laravel_session=eyJpdiI6ImpPMTI0MHI2WnM2OGw2RERpN1NOaWc9PSIsInZhbHVlIjoiSjhkUUJ4TFdIN2VGSkFQd0NkcUZGXC90dkFZOG1oSE1wWGwwVnFLTm1qejVqN1VBSWlrNWdlck9ZQnQxbHRHMmVyZGdwUmlcL1hxXC9qaTY5QlFaTTNUNEE9PSIsIm1hYyI6IjI3NzA4OTZiOTcxNzFiZjM0MTgzNGNlYmMyMjc0NmY2MDJhNTUxNzRmM2YwMWNjNDQ0ODBkOTQ0NjJlMTU0Y2MifQ%3D%3D; expires=Thu, 13-Jun-2019 06:35:03 GMT; Max-Age=1209600; path=/; HttpOnly] Vary:[Accept-Encoding] X-Content-Type-Options:[nosniff]] 0xc0002349e0 -1 [chunked] false true map[] 0xc000110000 0xc0000d02c0}
    

    使用fmt.Println(response.Headers) 获取,得到如下结果

    map[Cache-Control:private, must-revalidate Connection:keep-alive Content-Type:text/html; charset=UTF-8 Date:Thu, 30 May 2019 06:35:03 GMT Expires:-1 Pragma:no-cache Server:nginx Set-Cookie:XSRF-TOKEN=eyJpdiI6ImVZOThBVFlRZ3MzZmNsaVJmczd6Z3c9PSIsInZhbHVlIjoiSUdiSU5MblF0XC83N25iUXliMjNsNUlqTFE0MXpJQ3RLdU1LeGMrR1dmWG9pT25cL2hrd2I2ZG5iQlVnQVNTblJzZENCcm1LcTlcL3RVelZ3RFNmOWtkYlE9PSIsIm1hYyI6IjU2ZWU2MGY5NGE1OWJhNjBiZjM5YTFhNDgwMzJkNjViNDkxYmI2NGEyZmUzZGQ0N2NkNjhmZDQ5NDg2YTcyOGIifQ%3D%3D; expires=Thu, 13-Jun-2019 06:35:03 GMT; Max-Age=1209600; path=/ Vary:Accept-Encoding X-Content-Type-Options:nosniff]
    
    

    可以发现,laravel_session消失了。

    应该是下面这段代码的问题:

    for k, v := range this.Raw.Header {
    	headers[k] = v[0]
    }
    
httpreq is an http request library written with Golang to make requests and handle responses easily.

httpreq is an http request library written with Golang to make requests and handle responses easily. Install go get github.com/binalyze/http

Feb 10, 2022
A golang tool which makes http requests and prints the address of the request along with the MD5 hash of the response.

Golang Tool This repository is a golang tool which makes http requests to the external server and prints the address of the request along with the MD5

Oct 17, 2021
Simple web-hook based receiver executing things via HTTP request

Updater is a simple web-hook-based receiver executing things via HTTP requests and invoking remote updates without exposing any sensitive info, like ssh keys, passwords, etc.

Dec 19, 2022
Sample script to request test

Multiple request test script This script running in GO if not have compiler execute this: *** Open url: https://go.dev/dl/ download GO to your OS ***

Nov 23, 2021
Docker-Project - A simplified backend that listens to POST request

This is a simplified backend that listens to POST request. Once it receives such a request it will push it to a PostgreSQL database.

Feb 5, 2022
Speak HTTP like a local. (the simple, intuitive HTTP console, golang version)

http-gonsole This is the Go port of the http-console. Speak HTTP like a local Talking to an HTTP server with curl can be fun, but most of the time it'

Jul 14, 2021
Http client call for golang http api calls

httpclient-call-go This library is used to make http calls to different API services Install Package go get

Oct 7, 2022
fhttp is a fork of net/http that provides an array of features pertaining to the fingerprint of the golang http client.

fhttp The f stands for flex. fhttp is a fork of net/http that provides an array of features pertaining to the fingerprint of the golang http client. T

Jan 1, 2023
NATS HTTP Round Tripper - This is a Golang http.RoundTripper that uses NATS as a transport.

This is a Golang http.RoundTripper that uses NATS as a transport. Included is a http.RoundTripper for clients, a server that uses normal HTTP Handlers and any existing http handler mux and a Caddy Server transport.

Dec 6, 2022
Http-conection - A simple example of how to establish a HTTP connection using Golang

A simple example of how to establish a HTTP connection using Golang

Feb 1, 2022
Fast HTTP package for Go. Tuned for high performance. Zero memory allocations in hot paths. Up to 10x faster than net/http
Fast HTTP package for Go. Tuned for high performance. Zero memory allocations in hot paths. Up to 10x faster than net/http

fasthttp Fast HTTP implementation for Go. Currently fasthttp is successfully used by VertaMedia in a production serving up to 200K rps from more than

Jan 2, 2023
Simple HTTP package that wraps net/http

Simple HTTP package that wraps net/http

Jan 17, 2022
An enhanced http client for Golang
An enhanced http client for Golang

go-http-client An enhanced http client for Golang Documentation on go.dev ?? This package provides you a http client package for your http requests. Y

Dec 23, 2022
Go (golang) http calls with retries and backoff

pester pester wraps Go's standard lib http client to provide several options to increase resiliency in your request. If you experience poor network co

Dec 28, 2022
http client for golang
http client for golang

Request HTTP client for golang, Inspired by Javascript-axios Python-request. If you have experience about axios or requests, you will love it. No 3rd

Dec 18, 2022
A nicer interface for golang stdlib HTTP client

rq A nicer interface for golang stdlib HTTP client Documents rq: here client: here jar: here Why? Because golang HTTP client is a pain in the a... Fea

Dec 12, 2022
HTTP mocking for Golang

httpmock Easy mocking of http responses from external resources. Install Currently supports Go 1.7 - 1.15. v1 branch has to be used instead of master.

Dec 28, 2022
An HTTP performance testing tool written in GoLang

Gonce A HTTP API performance testing tool written in GoLang Description Installation Usage Description A performance testing tool written in GoLang. S

Jan 28, 2022
Declarative golang HTTP client

go-req Declarative golang HTTP client package req_test

Dec 20, 2022