// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a BSD-style license found in the LICENSE file. package codec // Contains code shared by both encode and decode. import ( "encoding/binary" "fmt" "math" "reflect" "sort" "strings" "sync" "time" "unicode" "unicode/utf8" ) const ( structTagName = "codec" // Support // encoding.BinaryMarshaler: MarshalBinary() (data []byte, err error) // encoding.BinaryUnmarshaler: UnmarshalBinary(data []byte) error // This constant flag will enable or disable it. supportBinaryMarshal = true // Each Encoder or Decoder uses a cache of functions based on conditionals, // so that the conditionals are not run every time. // // Either a map or a slice is used to keep track of the functions. // The map is more natural, but has a higher cost than a slice/array. // This flag (useMapForCodecCache) controls which is used. useMapForCodecCache = false // For some common container types, we can short-circuit an elaborate // reflection dance and call encode/decode directly. // The currently supported types are: // - slices of strings, or id's (int64,uint64) or interfaces. // - maps of str->str, str->intf, id(int64,uint64)->intf, intf->intf shortCircuitReflectToFastPath = true // for debugging, set this to false, to catch panic traces. // Note that this will always cause rpc tests to fail, since they need io.EOF sent via panic. recoverPanicToErr = true ) type charEncoding uint8 const ( c_RAW charEncoding = iota c_UTF8 c_UTF16LE c_UTF16BE c_UTF32LE c_UTF32BE ) // valueType is the stream type type valueType uint8 const ( valueTypeUnset valueType = iota valueTypeNil valueTypeInt valueTypeUint valueTypeFloat valueTypeBool valueTypeString valueTypeSymbol valueTypeBytes valueTypeMap valueTypeArray valueTypeTimestamp valueTypeExt valueTypeInvalid = 0xff ) var ( bigen = binary.BigEndian structInfoFieldName = "_struct" cachedTypeInfo = make(map[uintptr]*typeInfo, 4) cachedTypeInfoMutex sync.RWMutex intfSliceTyp = reflect.TypeOf([]interface{}(nil)) intfTyp = intfSliceTyp.Elem() strSliceTyp = reflect.TypeOf([]string(nil)) boolSliceTyp = reflect.TypeOf([]bool(nil)) uintSliceTyp = reflect.TypeOf([]uint(nil)) uint8SliceTyp = reflect.TypeOf([]uint8(nil)) uint16SliceTyp = reflect.TypeOf([]uint16(nil)) uint32SliceTyp = reflect.TypeOf([]uint32(nil)) uint64SliceTyp = reflect.TypeOf([]uint64(nil)) intSliceTyp = reflect.TypeOf([]int(nil)) int8SliceTyp = reflect.TypeOf([]int8(nil)) int16SliceTyp = reflect.TypeOf([]int16(nil)) int32SliceTyp = reflect.TypeOf([]int32(nil)) int64SliceTyp = reflect.TypeOf([]int64(nil)) float32SliceTyp = reflect.TypeOf([]float32(nil)) float64SliceTyp = reflect.TypeOf([]float64(nil)) mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil)) mapStrIntfTyp = reflect.TypeOf(map[string]interface{}(nil)) mapStrStrTyp = reflect.TypeOf(map[string]string(nil)) mapIntIntfTyp = reflect.TypeOf(map[int]interface{}(nil)) mapInt64IntfTyp = reflect.TypeOf(map[int64]interface{}(nil)) mapUintIntfTyp = reflect.TypeOf(map[uint]interface{}(nil)) mapUint64IntfTyp = reflect.TypeOf(map[uint64]interface{}(nil)) stringTyp = reflect.TypeOf("") timeTyp = reflect.TypeOf(time.Time{}) rawExtTyp = reflect.TypeOf(RawExt{}) mapBySliceTyp = reflect.TypeOf((*MapBySlice)(nil)).Elem() binaryMarshalerTyp = reflect.TypeOf((*binaryMarshaler)(nil)).Elem() binaryUnmarshalerTyp = reflect.TypeOf((*binaryUnmarshaler)(nil)).Elem() rawExtTypId = reflect.ValueOf(rawExtTyp).Pointer() intfTypId = reflect.ValueOf(intfTyp).Pointer() timeTypId = reflect.ValueOf(timeTyp).Pointer() intfSliceTypId = reflect.ValueOf(intfSliceTyp).Pointer() strSliceTypId = reflect.ValueOf(strSliceTyp).Pointer() boolSliceTypId = reflect.ValueOf(boolSliceTyp).Pointer() uintSliceTypId = reflect.ValueOf(uintSliceTyp).Pointer() uint8SliceTypId = reflect.ValueOf(uint8SliceTyp).Pointer() uint16SliceTypId = reflect.ValueOf(uint16SliceTyp).Pointer() uint32SliceTypId = reflect.ValueOf(uint32SliceTyp).Pointer() uint64SliceTypId = reflect.ValueOf(uint64SliceTyp).Pointer() intSliceTypId = reflect.ValueOf(intSliceTyp).Pointer() int8SliceTypId = reflect.ValueOf(int8SliceTyp).Pointer() int16SliceTypId = reflect.ValueOf(int16SliceTyp).Pointer() int32SliceTypId = reflect.ValueOf(int32SliceTyp).Pointer() int64SliceTypId = reflect.ValueOf(int64SliceTyp).Pointer() float32SliceTypId = reflect.ValueOf(float32SliceTyp).Pointer() float64SliceTypId = reflect.ValueOf(float64SliceTyp).Pointer() mapStrStrTypId = reflect.ValueOf(mapStrStrTyp).Pointer() mapIntfIntfTypId = reflect.ValueOf(mapIntfIntfTyp).Pointer() mapStrIntfTypId = reflect.ValueOf(mapStrIntfTyp).Pointer() mapIntIntfTypId = reflect.ValueOf(mapIntIntfTyp).Pointer() mapInt64IntfTypId = reflect.ValueOf(mapInt64IntfTyp).Pointer() mapUintIntfTypId = reflect.ValueOf(mapUintIntfTyp).Pointer() mapUint64IntfTypId = reflect.ValueOf(mapUint64IntfTyp).Pointer() // Id = reflect.ValueOf().Pointer() // mapBySliceTypId = reflect.ValueOf(mapBySliceTyp).Pointer() binaryMarshalerTypId = reflect.ValueOf(binaryMarshalerTyp).Pointer() binaryUnmarshalerTypId = reflect.ValueOf(binaryUnmarshalerTyp).Pointer() intBitsize uint8 = uint8(reflect.TypeOf(int(0)).Bits()) uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits()) bsAll0x00 = []byte{0, 0, 0, 0, 0, 0, 0, 0} bsAll0xff = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} ) type binaryUnmarshaler interface { UnmarshalBinary(data []byte) error } type binaryMarshaler interface { MarshalBinary() (data []byte, err error) } // MapBySlice represents a slice which should be encoded as a map in the stream. // The slice contains a sequence of key-value pairs. type MapBySlice interface { MapBySlice() } // WARNING: DO NOT USE DIRECTLY. EXPORTED FOR GODOC BENEFIT. WILL BE REMOVED. // // BasicHandle encapsulates the common options and extension functions. type BasicHandle struct { extHandle EncodeOptions DecodeOptions } // Handle is the interface for a specific encoding format. // // Typically, a Handle is pre-configured before first time use, // and not modified while in use. Such a pre-configured Handle // is safe for concurrent access. type Handle interface { writeExt() bool getBasicHandle() *BasicHandle newEncDriver(w encWriter) encDriver newDecDriver(r decReader) decDriver } // RawExt represents raw unprocessed extension data. type RawExt struct { Tag byte Data []byte } type extTypeTagFn struct { rtid uintptr rt reflect.Type tag byte encFn func(reflect.Value) ([]byte, error) decFn func(reflect.Value, []byte) error } type extHandle []*extTypeTagFn // AddExt registers an encode and decode function for a reflect.Type. // Note that the type must be a named type, and specifically not // a pointer or Interface. An error is returned if that is not honored. // // To Deregister an ext, call AddExt with 0 tag, nil encfn and nil decfn. func (o *extHandle) AddExt( rt reflect.Type, tag byte, encfn func(reflect.Value) ([]byte, error), decfn func(reflect.Value, []byte) error, ) (err error) { // o is a pointer, because we may need to initialize it if rt.PkgPath() == "" || rt.Kind() == reflect.Interface { err = fmt.Errorf("codec.Handle.AddExt: Takes named type, especially not a pointer or interface: %T", reflect.Zero(rt).Interface()) return } // o cannot be nil, since it is always embedded in a Handle. // if nil, let it panic. // if o == nil { // err = errors.New("codec.Handle.AddExt: extHandle cannot be a nil pointer.") // return // } rtid := reflect.ValueOf(rt).Pointer() for _, v := range *o { if v.rtid == rtid { v.tag, v.encFn, v.decFn = tag, encfn, decfn return } } *o = append(*o, &extTypeTagFn{rtid, rt, tag, encfn, decfn}) return } func (o extHandle) getExt(rtid uintptr) *extTypeTagFn { for _, v := range o { if v.rtid == rtid { return v } } return nil } func (o extHandle) getExtForTag(tag byte) *extTypeTagFn { for _, v := range o { if v.tag == tag { return v } } return nil } func (o extHandle) getDecodeExtForTag(tag byte) ( rv reflect.Value, fn func(reflect.Value, []byte) error) { if x := o.getExtForTag(tag); x != nil { // ext is only registered for base rv = reflect.New(x.rt).Elem() fn = x.decFn } return } func (o extHandle) getDecodeExt(rtid uintptr) (tag byte, fn func(reflect.Value, []byte) error) { if x := o.getExt(rtid); x != nil { tag = x.tag fn = x.decFn } return } func (o extHandle) getEncodeExt(rtid uintptr) (tag byte, fn func(reflect.Value) ([]byte, error)) { if x := o.getExt(rtid); x != nil { tag = x.tag fn = x.encFn } return } type structFieldInfo struct { encName string // encode name // only one of 'i' or 'is' can be set. If 'i' is -1, then 'is' has been set. is []int // (recursive/embedded) field index in struct i int16 // field index in struct omitEmpty bool toArray bool // if field is _struct, is the toArray set? // tag string // tag // name string // field name // encNameBs []byte // encoded name as byte stream // ikind int // kind of the field as an int i.e. int(reflect.Kind) } func parseStructFieldInfo(fname string, stag string) *structFieldInfo { if fname == "" { panic("parseStructFieldInfo: No Field Name") } si := structFieldInfo{ // name: fname, encName: fname, // tag: stag, } if stag != "" { for i, s := range strings.Split(stag, ",") { if i == 0 { if s != "" { si.encName = s } } else { switch s { case "omitempty": si.omitEmpty = true case "toarray": si.toArray = true } } } } // si.encNameBs = []byte(si.encName) return &si } type sfiSortedByEncName []*structFieldInfo func (p sfiSortedByEncName) Len() int { return len(p) } func (p sfiSortedByEncName) Less(i, j int) bool { return p[i].encName < p[j].encName } func (p sfiSortedByEncName) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // typeInfo keeps information about each type referenced in the encode/decode sequence. // // During an encode/decode sequence, we work as below: // - If base is a built in type, en/decode base value // - If base is registered as an extension, en/decode base value // - If type is binary(M/Unm)arshaler, call Binary(M/Unm)arshal method // - Else decode appropriately based on the reflect.Kind type typeInfo struct { sfi []*structFieldInfo // sorted. Used when enc/dec struct to map. sfip []*structFieldInfo // unsorted. Used when enc/dec struct to array. rt reflect.Type rtid uintptr // baseId gives pointer to the base reflect.Type, after deferencing // the pointers. E.g. base type of ***time.Time is time.Time. base reflect.Type baseId uintptr baseIndir int8 // number of indirections to get to base mbs bool // base type (T or *T) is a MapBySlice m bool // base type (T or *T) is a binaryMarshaler unm bool // base type (T or *T) is a binaryUnmarshaler mIndir int8 // number of indirections to get to binaryMarshaler type unmIndir int8 // number of indirections to get to binaryUnmarshaler type toArray bool // whether this (struct) type should be encoded as an array } func (ti *typeInfo) indexForEncName(name string) int { //tisfi := ti.sfi const binarySearchThreshold = 16 if sfilen := len(ti.sfi); sfilen < binarySearchThreshold { // linear search. faster than binary search in my testing up to 16-field structs. for i, si := range ti.sfi { if si.encName == name { return i } } } else { // binary search. adapted from sort/search.go. h, i, j := 0, 0, sfilen for i < j { h = i + (j-i)/2 if ti.sfi[h].encName < name { i = h + 1 } else { j = h } } if i < sfilen && ti.sfi[i].encName == name { return i } } return -1 } func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) { var ok bool cachedTypeInfoMutex.RLock() pti, ok = cachedTypeInfo[rtid] cachedTypeInfoMutex.RUnlock() if ok { return } cachedTypeInfoMutex.Lock() defer cachedTypeInfoMutex.Unlock() if pti, ok = cachedTypeInfo[rtid]; ok { return } ti := typeInfo{rt: rt, rtid: rtid} pti = &ti var indir int8 if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok { ti.m, ti.mIndir = true, indir } if ok, indir = implementsIntf(rt, binaryUnmarshalerTyp); ok { ti.unm, ti.unmIndir = true, indir } if ok, _ = implementsIntf(rt, mapBySliceTyp); ok { ti.mbs = true } pt := rt var ptIndir int8 // for ; pt.Kind() == reflect.Ptr; pt, ptIndir = pt.Elem(), ptIndir+1 { } for pt.Kind() == reflect.Ptr { pt = pt.Elem() ptIndir++ } if ptIndir == 0 { ti.base = rt ti.baseId = rtid } else { ti.base = pt ti.baseId = reflect.ValueOf(pt).Pointer() ti.baseIndir = ptIndir } if rt.Kind() == reflect.Struct { var siInfo *structFieldInfo if f, ok := rt.FieldByName(structInfoFieldName); ok { siInfo = parseStructFieldInfo(structInfoFieldName, f.Tag.Get(structTagName)) ti.toArray = siInfo.toArray } sfip := make([]*structFieldInfo, 0, rt.NumField()) rgetTypeInfo(rt, nil, make(map[string]bool), &sfip, siInfo) // // try to put all si close together // const tryToPutAllStructFieldInfoTogether = true // if tryToPutAllStructFieldInfoTogether { // sfip2 := make([]structFieldInfo, len(sfip)) // for i, si := range sfip { // sfip2[i] = *si // } // for i := range sfip { // sfip[i] = &sfip2[i] // } // } ti.sfip = make([]*structFieldInfo, len(sfip)) ti.sfi = make([]*structFieldInfo, len(sfip)) copy(ti.sfip, sfip) sort.Sort(sfiSortedByEncName(sfip)) copy(ti.sfi, sfip) } // sfi = sfip cachedTypeInfo[rtid] = pti return } func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool, sfi *[]*structFieldInfo, siInfo *structFieldInfo, ) { // for rt.Kind() == reflect.Ptr { // // indexstack = append(indexstack, 0) // rt = rt.Elem() // } for j := 0; j < rt.NumField(); j++ { f := rt.Field(j) stag := f.Tag.Get(structTagName) if stag == "-" { continue } if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) { continue } // if anonymous and there is no struct tag and its a struct (or pointer to struct), inline it. if f.Anonymous && stag == "" { ft := f.Type for ft.Kind() == reflect.Ptr { ft = ft.Elem() } if ft.Kind() == reflect.Struct { indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j) rgetTypeInfo(ft, indexstack2, fnameToHastag, sfi, siInfo) continue } } // do not let fields with same name in embedded structs override field at higher level. // this must be done after anonymous check, to allow anonymous field // still include their child fields if _, ok := fnameToHastag[f.Name]; ok { continue } si := parseStructFieldInfo(f.Name, stag) // si.ikind = int(f.Type.Kind()) if len(indexstack) == 0 { si.i = int16(j) } else { si.i = -1 si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j) } if siInfo != nil { if siInfo.omitEmpty { si.omitEmpty = true } } *sfi = append(*sfi, si) fnameToHastag[f.Name] = stag != "" } } func panicToErr(err *error) { if recoverPanicToErr { if x := recover(); x != nil { //debug.PrintStack() panicValToErr(x, err) } } } func doPanic(tag string, format string, params ...interface{}) { params2 := make([]interface{}, len(params)+1) params2[0] = tag copy(params2[1:], params) panic(fmt.Errorf("%s: "+format, params2...)) } func checkOverflowFloat32(f float64, doCheck bool) { if !doCheck { return } // check overflow (logic adapted from std pkg reflect/value.go OverflowFloat() f2 := f if f2 < 0 { f2 = -f } if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 { decErr("Overflow float32 value: %v", f2) } } func checkOverflow(ui uint64, i int64, bitsize uint8) { // check overflow (logic adapted from std pkg reflect/value.go OverflowUint() if bitsize == 0 { return } if i != 0 { if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc { decErr("Overflow int value: %v", i) } } if ui != 0 { if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc { decErr("Overflow uint value: %v", ui) } } }