custom high-performance encoder implementation (#52)
Ignacio Hagopian
4 years ago
committed by
James Mills
parent
755b1879b5
commit
fd179b4a86
@ -1,13 +1,3 @@
|
||||
// Package bitcask implements a high-performance key-value store based on a
|
||||
// WAL and LSM.
|
||||
//
|
||||
// By default, the client assumes a default configuration regarding maximum key size,
|
||||
// maximum value size, maximum datafile size, and memory pools to avoid allocations.
|
||||
// Refer to Constants section to know default values.
|
||||
//
|
||||
// For extra performance, configure the memory pool option properly. This option
|
||||
// requires to specify the maximum number of concurrent use of the package. Failing to
|
||||
// set a high-enough value would impact latency and throughput. Likewise, overestimating
|
||||
// would yield in an unnecessary big memory footprint.
|
||||
// The default configuration doesn't use a memory pool.
|
||||
package bitcask
|
||||
|
||||
@ -0,0 +1,113 @@
|
||||
package codec
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prologic/bitcask/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
KeySize = 4
|
||||
ValueSize = 8
|
||||
checksumSize = 4
|
||||
)
|
||||
|
||||
// NewEncoder creates a streaming Entry encoder.
|
||||
func NewEncoder(w io.Writer) *Encoder {
|
||||
return &Encoder{w: bufio.NewWriter(w)}
|
||||
}
|
||||
|
||||
// Encoder wraps an underlying io.Writer and allows you to stream
|
||||
// Entry encodings on it.
|
||||
type Encoder struct {
|
||||
w *bufio.Writer
|
||||
}
|
||||
|
||||
// Encode takes any Entry and streams it to the underlying writer.
|
||||
// Messages are framed with a key-length and value-length prefix.
|
||||
func (e *Encoder) Encode(msg model.Entry) (int64, error) {
|
||||
var bufKeyValue = make([]byte, ValueSize)
|
||||
|
||||
bufKeySize := bufKeyValue[:KeySize]
|
||||
binary.BigEndian.PutUint32(bufKeySize, uint32(len(msg.Key)))
|
||||
if _, err := e.w.Write(bufKeySize); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing key length prefix")
|
||||
}
|
||||
|
||||
bufValueSize := bufKeyValue[:ValueSize]
|
||||
binary.BigEndian.PutUint64(bufValueSize, uint64(len(msg.Value)))
|
||||
if _, err := e.w.Write(bufValueSize); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing value length prefix")
|
||||
}
|
||||
|
||||
if _, err := e.w.Write([]byte(msg.Key)); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing key data")
|
||||
}
|
||||
if _, err := e.w.Write(msg.Value); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing value data")
|
||||
}
|
||||
|
||||
bufChecksumSize := make([]byte, checksumSize)
|
||||
binary.BigEndian.PutUint32(bufChecksumSize, msg.Checksum)
|
||||
if _, err := e.w.Write(bufChecksumSize); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing checksum data")
|
||||
}
|
||||
|
||||
if err := e.w.Flush(); err != nil {
|
||||
return 0, errors.Wrap(err, "failed flushing data")
|
||||
}
|
||||
|
||||
return int64(KeySize + ValueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
|
||||
}
|
||||
|
||||
// NewDecoder creates a streaming Entry decoder.
|
||||
func NewDecoder(r io.Reader) *Decoder {
|
||||
return &Decoder{r: r}
|
||||
}
|
||||
|
||||
// Decoder wraps an underlying io.Reader and allows you to stream
|
||||
// Entry decodings on it.
|
||||
type Decoder struct {
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
func (d *Decoder) Decode(v *model.Entry) (int64, error) {
|
||||
prefixBuf := make([]byte, KeySize+ValueSize)
|
||||
|
||||
_, err := io.ReadFull(d.r, prefixBuf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
actualKeySize, actualValueSize := GetKeyValueSizes(prefixBuf)
|
||||
buf := make([]byte, actualKeySize+actualValueSize+checksumSize)
|
||||
if _, err = io.ReadFull(d.r, buf); err != nil {
|
||||
return 0, errors.Wrap(translateError(err), "failed reading saved data")
|
||||
}
|
||||
|
||||
DecodeWithoutPrefix(buf, actualValueSize, v)
|
||||
return int64(KeySize + ValueSize + actualKeySize + actualValueSize + checksumSize), nil
|
||||
}
|
||||
|
||||
func GetKeyValueSizes(buf []byte) (uint64, uint64) {
|
||||
actualKeySize := binary.BigEndian.Uint32(buf[:KeySize])
|
||||
actualValueSize := binary.BigEndian.Uint64(buf[KeySize:])
|
||||
|
||||
return uint64(actualKeySize), actualValueSize
|
||||
}
|
||||
|
||||
func DecodeWithoutPrefix(buf []byte, valueOffset uint64, v *model.Entry) {
|
||||
v.Key = buf[:valueOffset]
|
||||
v.Value = buf[valueOffset : len(buf)-checksumSize]
|
||||
v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])
|
||||
}
|
||||
|
||||
func translateError(err error) error {
|
||||
if err == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"hash/crc32"
|
||||
|
||||
pb "github.com/prologic/bitcask/internal/proto"
|
||||
)
|
||||
|
||||
func NewEntry(key, value []byte) pb.Entry {
|
||||
checksum := crc32.ChecksumIEEE(value)
|
||||
|
||||
return pb.Entry{
|
||||
Checksum: checksum,
|
||||
Key: key,
|
||||
Value: value,
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"hash/crc32"
|
||||
)
|
||||
// Entry represents a key/value in the database
|
||||
type Entry struct {
|
||||
Checksum uint32
|
||||
Key []byte
|
||||
Offset int64
|
||||
Value []byte
|
||||
}
|
||||
|
||||
|
||||
func NewEntry(key, value []byte) Entry {
|
||||
checksum := crc32.ChecksumIEEE(value)
|
||||
|
||||
return Entry{
|
||||
Checksum: checksum,
|
||||
Key: key,
|
||||
Value: value,
|
||||
}
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
package proto
|
||||
|
||||
//go:generate protoc --go_out=. entry.proto
|
||||
@ -1,97 +0,0 @@
|
||||
package streampb
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
// prefixSize is the number of bytes we preallocate for storing
|
||||
// our big endian lenth prefix buffer.
|
||||
prefixSize = 8
|
||||
)
|
||||
|
||||
// NewEncoder creates a streaming protobuf encoder.
|
||||
func NewEncoder(w io.Writer) *Encoder {
|
||||
return &Encoder{w: bufio.NewWriter(w)}
|
||||
}
|
||||
|
||||
// Encoder wraps an underlying io.Writer and allows you to stream
|
||||
// proto encodings on it.
|
||||
type Encoder struct {
|
||||
w *bufio.Writer
|
||||
}
|
||||
|
||||
// Encode takes any proto.Message and streams it to the underlying writer.
|
||||
// Messages are framed with a length prefix.
|
||||
func (e *Encoder) Encode(msg proto.Message) (int64, error) {
|
||||
prefixBuf := make([]byte, prefixSize)
|
||||
|
||||
buf, err := proto.Marshal(msg)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
binary.BigEndian.PutUint64(prefixBuf, uint64(len(buf)))
|
||||
|
||||
if _, err := e.w.Write(prefixBuf); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing length prefix")
|
||||
}
|
||||
|
||||
n, err := e.w.Write(buf)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing marshaled data")
|
||||
}
|
||||
|
||||
if err = e.w.Flush(); err != nil {
|
||||
return 0, errors.Wrap(err, "failed flushing data")
|
||||
}
|
||||
|
||||
return int64(n + prefixSize), nil
|
||||
}
|
||||
|
||||
// NewDecoder creates a streaming protobuf decoder.
|
||||
func NewDecoder(r io.Reader) *Decoder {
|
||||
return &Decoder{r: r}
|
||||
}
|
||||
|
||||
// Decoder wraps an underlying io.Reader and allows you to stream
|
||||
// proto decodings on it.
|
||||
type Decoder struct {
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
// Decode takes a proto.Message and unmarshals the next payload in the
|
||||
// underlying io.Reader. It returns an EOF when it's done.
|
||||
func (d *Decoder) Decode(v proto.Message) (int64, error) {
|
||||
prefixBuf := make([]byte, prefixSize)
|
||||
|
||||
_, err := io.ReadFull(d.r, prefixBuf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
n := binary.BigEndian.Uint64(prefixBuf)
|
||||
|
||||
buf := make([]byte, n)
|
||||
|
||||
idx := uint64(0)
|
||||
for idx < n {
|
||||
m, err := d.r.Read(buf[idx:n])
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(translateError(err), "failed reading marshaled data")
|
||||
}
|
||||
idx += uint64(m)
|
||||
}
|
||||
return int64(idx + prefixSize), proto.Unmarshal(buf[:n], v)
|
||||
}
|
||||
|
||||
func translateError(err error) error {
|
||||
if err == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
}
|
||||
Loading…
Reference in New Issue