Golang binary decoder for mapping data into the structure

Go Report Card Build Status CodeCov GoDoc License: MIT

binstruct

Golang binary decoder to structure

Install

go get -u github.com/ghostiam/binstruct

Examples

ZIP decoder
PNG decoder

Use

For struct

From file or other io.ReadSeeker:

package main

import (
	"encoding/binary"
	"fmt"
	"log"
	"os"

	"github.com/ghostiam/binstruct"
)

func main() {
	file, err := os.Open("testdata/file.bin")
	if err != nil {
		log.Fatal(err)
	}

	type dataStruct struct {
		Arr []int16 `bin:"len:4"`
	}

	var actual dataStruct
	decoder := binstruct.NewDecoder(file, binary.BigEndian)
	// decoder.SetDebug(true) // you can enable the output of bytes read for debugging
	err = decoder.Decode(&actual)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%+v", actual)

	// Output:
	// {Arr:[1 2 3 4]}
}

From bytes

package main

import (
	"fmt"
	"log"
	
	"github.com/ghostiam/binstruct"
)

func main() {
	data := []byte{
		0x00, 0x01,
		0x00, 0x02,
		0x00, 0x03,
		0x00, 0x04,
	}

	type dataStruct struct {
		Arr []int16 `bin:"len:4"`
	}

	var actual dataStruct
	err := binstruct.UnmarshalBE(data, &actual) // UnmarshalLE() or Unmarshal()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%+v", actual)

	// Output: {Arr:[1 2 3 4]}
}

or just use reader without mapping data into the structure

You can not use the functionality for mapping data into the structure, you can use the interface to get data from the stream (io.ReadSeeker)

reader.go

type Reader interface {
	io.ReadSeeker

	// Peek returns the next n bytes without advancing the reader.
	Peek(n int) ([]byte, error)

	// ReadBytes reads up to n bytes. It returns the number of bytes
	// read, bytes and any error encountered.
	ReadBytes(n int) (an int, b []byte, err error)
	// ReadAll reads until an error or EOF and returns the data it read.
	ReadAll() ([]byte, error)

	// ReadByte read and return one byte
	ReadByte() (byte, error)
	// ReadBool read one byte and return boolean value
	ReadBool() (bool, error)

	// ReadUint8 read one byte and return uint8 value
	ReadUint8() (uint8, error)
	// ReadUint16 read two bytes and return uint16 value
	ReadUint16() (uint16, error)
	// ReadUint32 read four bytes and return uint32 value
	ReadUint32() (uint32, error)
	// ReadUint64 read eight bytes and return uint64 value
	ReadUint64() (uint64, error)

	// ReadInt8 read one byte and return int8 value
	ReadInt8() (int8, error)
	// ReadInt16 read two bytes and return int16 value
	ReadInt16() (int16, error)
	// ReadInt32 read four bytes and return int32 value
	ReadInt32() (int32, error)
	// ReadInt64 read eight bytes and return int64 value
	ReadInt64() (int64, error)

	// ReadFloat32 read four bytes and return float32 value
	ReadFloat32() (float32, error)
	// ReadFloat64 read eight bytes and return float64 value
	ReadFloat64() (float64, error)

	// Unmarshal parses the binary data and stores the result
	// in the value pointed to by v.
	Unmarshal(v interface{}) error
}

Example:

package main

import (
	"encoding/binary"
	"fmt"
	"log"
	
	"github.com/ghostiam/binstruct"
)

func main() {
	data := []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F}

	reader := binstruct.NewReaderFromBytes(data, binary.BigEndian, false)

	i16, err := reader.ReadInt16()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(i16)

	i32, err := reader.ReadInt32()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(i32)

	b, err := reader.Peek(4)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Peek bytes: %#v\n", b)

	an, b, err := reader.ReadBytes(4)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Read %d bytes: %#v\n", an, b)

	other, err := reader.ReadAll()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Read all: %#v\n", other)

	// Output:
	// 258
	// 50595078
	// Peek bytes: []byte{0x7, 0x8, 0x9, 0xa}
	// Read 4 bytes: []byte{0x7, 0x8, 0x9, 0xa}
	// Read all: []byte{0xb, 0xc, 0xd, 0xe, 0xf}
}

Decode to fields

type test struct {
	// Read 1 byte
	Field bool
	Field byte
	Field [1]byte
	Field int8
	Field uint8

	// Read 2 bytes
	Field int16
	Field uint16
	Field [2]byte

	// Read 4 bytes
	Field int32
	Field uint32
	Field [4]byte

	// Read 8 bytes
	Field int64
	Field uint64
	Field [8]byte

	// You can override length
	Field int64 `bin:"len:2"`

	// Fields of type int, uint and string are not read automatically 
	// because the size is not known, you need to set it manually
	Field int    `bin:"len:2"`
	Field uint   `bin:"len:4"`
	Field string `bin:"len:42"`
	
	// Can read arrays and slices
	Array [2]int32              // read 8 bytes (4+4byte for 2 int32)
	Slice []int32 `bin:"len:2"` // read 8 bytes (4+4byte for 2 int32)
	
	// Also two-dimensional slices work (binstruct_test.go:307 Test_SliceOfSlice)
	Slice2D [][]int32 `bin:"len:2,[len:2]"`
	// and even three-dimensional slices (binstruct_test.go:329 Test_SliceOfSliceOfSlice)
	Slice3D [][][]int32 `bin:"len:2,[len:2,[len:2]]"`
	
	// Structures and embedding are also supported.
	Struct struct {
		...
	}
	OtherStruct Other
	Other // embedding
}

type Other struct {
	...
}

Tags

type test struct {
	IgnoredField []byte `bin:"-"`          // ignore field
	CallMethod   []byte `bin:"MethodName"` // Call method "MethodName"
	ReadLength   []byte `bin:"len:42"`     // read 42 bytes
	
	// Offsets test binstruct_test.go:9
	Offset      byte `bin:"offset:42"`      // move to 42 bytes from current position and read byte
	OffsetStart byte `bin:"offsetStart:42"` // move to 42 bytes from start position and read byte
	OffsetEnd   byte `bin:"offsetEnd:-42"`  // move to -42 bytes from end position and read byte
	OffsetStart byte `bin:"offsetStart:42, offset:10"` // also worked and equally `offsetStart:52`

	// Calculations supported +,-,/,* and are performed from left to right that is 2+2*2=8 not 6!!!
	CalcTagValue []byte `bin:"len:10+5+2+3"` // equally len:20
	
	// You can refer to another field to get the value.
	DataLength              int    // actual length
	ValueFromOtherField     string `bin:"len:DataLength"`
	CalcValueFromOtherField string `bin:"len:DataLength+10"` // also work calculations
} 

// Method can be:
func (*test) MethodName(r binstruct.Reader) (error) {}
// or
func (*test) MethodName(r binstruct.Reader) (FieldType, error) {}

See the tests and examples for more information.

License

MIT License

Owner
Similar Resources

Go package for dealing with maps, slices, JSON and other data.

Objx Objx - Go package for dealing with maps, slices, JSON and other data. Get started: Install Objx with one line of code, or update it with another

Dec 27, 2022

Asn.1 BER and DER encoding library for golang.

WARNING This repo has been archived! NO further developement will be made in the foreseen future. asn1 -- import "github.com/PromonLogicalis/asn1" Pac

Nov 14, 2022

auto-generate capnproto schema from your golang source files. Depends on go-capnproto-1.0 at https://github.com/glycerine/go-capnproto

bambam: auto-generate capnproto schema from your golang source files. Adding capnproto serialization to an existing Go project used to mean writing a

Sep 27, 2022

CBOR RFC 7049 (Go/Golang) - safe & fast with standard API + toarray & keyasint, CBOR tags, float64/32/16, fuzz tested.

CBOR RFC 7049 (Go/Golang) - safe & fast with standard API + toarray & keyasint, CBOR tags, float64/32/16, fuzz tested.

CBOR library in Go fxamacker/cbor is a CBOR encoder & decoder in Go. It has a standard API, CBOR tags, options for duplicate map keys, float64→32→16,

Jan 6, 2023

Fast implementation of base58 encoding on golang.

Fast Implementation of Base58 encoding Fast implementation of base58 encoding in Go. Base algorithm is adapted from https://github.com/trezor/trezor-c

Dec 9, 2022

msgpack.org[Go] MessagePack encoding for Golang

MessagePack encoding for Golang ❤️ Uptrace.dev - All-in-one tool to optimize performance and monitor errors & logs Join Discord to ask questions. Docu

Dec 28, 2022

Encode and decode Go (golang) struct types via protocol buffers.

protostructure protostructure is a Go library for encoding and decoding a struct type over the wire. This library is useful when you want to send arbi

Nov 15, 2022

golang struct 或其他对象向 []byte 的序列化或反序列化

bytecodec 字节流编解码 这个库实现 struct 或其他对象向 []byte 的序列化或反序列化 可以帮助你在编写 tcp 服务,或者需要操作字节流时,简化数据的组包、解包 这个库的组织逻辑 copy 借鉴了标准库 encoding/json 🙏 安装 使用 go get 安装最新版本

Jun 30, 2022

Dynamically Generates Ysoserial's Payload by Golang

Dynamically Generates Ysoserial's Payload by Golang

Gososerial 介绍 ysoserial是java反序列化安全方面著名的工具 无需java环境,无需下载ysoserial.jar文件 输入命令直接获得payload,方便编写安全工具 目前已支持CC1-CC7,K1-K4和CB1链 Introduce Ysoserial is a well-

Jul 10, 2022
Comments
  • function tag on nested struct

    function tag on nested struct

    type Bin struct {
    	...
    	Pin           struct {
    		Pin1         []byte `bin:"len:4"` // LE
    		Pin1Unknown  []byte `bin:"len:4"`
    		Pin1Checksum uint16 `bin:"Uint16l,len:2"`
    		Pin2         []byte `bin:"len:4"` // LE
    		Pin2Unknown  []byte `bin:"len:4"`
    		Pin2Checksum uint16 `bin:"Uint16l,len:2"`
    	} // 20 bytes
            ...
    }
    
    // Uint16l returns a uint16 read as little endian
    func (*Bin) Uint16l(r binstruct.Reader) (uint16, error) {
    	var out uint16
    	if err := binary.Read(r, binary.LittleEndian, &out); err != nil {
    		return 0, err
    	}
    	return out, nil
    }
    
    2021/12/25 16:05:41 main.go:21: failed set value to field "Pin": unmarshal struct: failed set value to field "Pin1Checksum": call custom func: 
    failed call method, expected methods:
            func (*) Uint16l(r binstruct.Reader) error {} 
    or
            func (*) Uint16l(r binstruct.Reader) (uint16, error) {}
    exit status 1
    
  • a question

    a question

    package main
    
    import (
    	"fmt"
    	"log"
    
    	"github.com/ghostiam/binstruct"
    )
    
    func main() {
    	data := []byte{
    		0xbf, 0x81, 0xb7, 0xa, 0xd9, 0x5e, 0xd8, 0xb9, 0xff, 0x7f, 0xe9, 0x11, 0x85, 0xbd, 0x90, 0x29, 0xbc, 0x6, 0x80, 0x0, 0xaf,
    	}
    
    	type dataStruct struct {
    
    		First []byte `bin:"len:2"`
    		Mid   []byte `bin:"len:6"`
    		Last  []byte `bin:"len:"`
    	}
    
    	var actual dataStruct
    	err := binstruct.UnmarshalBE(data, &actual) // UnmarshalLE() or Unmarshal()
    	if err != nil {
    		log.Fatal(err)
    	}
    
    	fmt.Printf("%+v", actual)
    
    	// Output: {Arr:[1 2 3 4]}
    }
    
    

    result :

    main.dataStruct{First:[]uint8{0xbf, 0x81}, Mid:[]uint8{0xb7, 0xa, 0xd9, 0x5e, 0x
    d8, 0xb9}, Last:[]uint8(nil)}
    

    how UnmarshalBE above dataStruct with last of []byte with dont set lenght of []byte ?

    i need Last is []byte{0x11, 0x85, 0xbd, 0x90, 0x29, 0xbc, 0x6, 0x80, 0x0, 0xaf}

    Thanks.

  • Allow unordinary lengths for (u)ints

    Allow unordinary lengths for (u)ints

    Currently, only lengths that are a power of two are correctly read from bytes. Other cases fail due to being forced into reading the number of bytes of the target field.

    For example:

    type Foo struct {
        Prefix  int32 `bin:"len:3"`
        Version uint8
    }
    

    This struct should read 3 bytes and interpret them as an "int24" - the 24th bit should be the indicator of negativity - and store that value in an int32. Currently, because len is not a power of two in this example, readers will look to the type of the target field and determine that 4 bytes should be read and interpreted as an int32.

    This pull request addresses this issue by using a variable length reader/interpreter for these cases. When len is provided, exactly that number of bytes are interpreted. First they are ingested by iterative bit shifting and ORing. In the int cases, the value is byte shifted left as a uint, then cast as an int and shifted right to account for negativity via twos-complement shifting rules.

  • Add license scan report and status

    Add license scan report and status

    Your FOSSA integration was successful! Attached in this PR is a badge and license report to track scan status in your README.

    Below are docs for integrating FOSSA license checks into your CI:

csvutil provides fast and idiomatic mapping between CSV and Go (golang) values.
csvutil provides fast and idiomatic mapping between CSV and Go (golang) values.

csvutil Package csvutil provides fast and idiomatic mapping between CSV and Go (golang) values. This package does not provide a CSV parser itself, it

Jan 6, 2023
Fixed width file parser (encoder/decoder) in GO (golang)

Fixed width file parser (encoder/decoder) for GO (golang) This library is using to parse fixed-width table data like: Name Address

Sep 27, 2022
PHP session encoder/decoder written in Go

php_session_decoder PHP session encoder/decoder written in Go Installation Install: The recommended way to install is using gonuts.io: nut get yvasiya

Sep 27, 2022
BPG decoder for Go (Zero Dependencies)

Go语言QQ群: 102319854, 1055927514 凹语言(凹读音“Wa”)(The Wa Programming Language): https://github.com/wa-lang/wa BPG for Go BPG is defined at: http://bellard.o

Sep 7, 2020
Sqlyog-password-decoder - Simple decode passwords from .sycs file (SQLyog export connections file)

Decode password: ./sqlyog-password-decoder -str password -action decode Encode p

Nov 21, 2021
binary serialization format

Colfer Colfer is a binary serialization format optimized for speed and size. The project's compiler colf(1) generates source code from schema definiti

Dec 25, 2022
Simple, specialised, and efficient binary marshaling

?? surge Documentation A library for fast binary (un)marshaling. Designed to be used in Byzantine networks, ?? surge never explicitly panics, protects

Oct 4, 2022
Musgo is a Go code generator for binary MUS format with validation support.

Musgo is a Go code generator for binary MUS format with validation support. Generated code converts data to and from MUS format.

Dec 29, 2022
Encode and decode binary message and file formats in Go

Encode and Decode Binary Formats in Go This module wraps the package encoding/binary of the Go standard library and provides the missing Marshal() and

Dec 22, 2022
Go library for decoding generic map values into native Go structures and vice versa.

mapstructure mapstructure is a Go library for decoding generic map values to structures and vice versa, while providing helpful error handling. This l

Jan 1, 2023