Browse Source

Refactored and stripped down to work with bitcask backend

pull/1/head
James Mills 3 years ago
parent
commit
5a372462d3
Signed by untrusted user who does not match committer: prologic GPG Key ID: AC4C014F1440EBD6
  1. 22
      .drone.yml
  2. 12
      .gitignore
  3. 25
      .goreleaser.yml
  4. 13
      LICENSE
  5. 20
      LICENSE.old
  6. 39
      Makefile
  7. 30
      README.md
  8. 68
      cmd/kvnode-server/main.go
  9. 60
      glide.lock
  10. 11
      glide.yaml
  11. 23
      go.mod
  12. 125
      go.sum
  13. 101
      main.go
  14. 386
      server.go
  15. 25
      tools/release.sh
  16. 18
      version.go
  17. 15
      version_test.go

22
.drone.yml

@ -0,0 +1,22 @@
kind: pipeline
name: default
steps:
- name: build
image: golang:latest
commands:
- go test -v -short -cover -coverprofile=coverage.txt -coverpkg=$(go list) ./...
- name: coverage
image: plugins/codecov
settings:
token:
from_secret: codecov-token
- name: notify
image: plugins/webhook
urls: https://msgbus.mills.io/ci.mills.io
when:
status:
- success
- failure

12
.gitignore

@ -1,2 +1,10 @@
kvnode-server*
data*
*~*
*.bak
/coverage.txt
/bitraft
/tmp
/dist
/data
/data1
/data2

25
.goreleaser.yml

@ -0,0 +1,25 @@
builds:
-
flags: -tags "static_build"
ldflags: -w -X .Version={{.Version}} -X .Commit={{.Commit}}
env:
- CGO_ENABLED=0
sign:
artifacts: checksum
archive:
replacements:
darwin: Darwin
linux: Linux
windows: Windows
386: i386
amd64: x86_64
checksum:
name_template: 'checksums.txt'
snapshot:
name_template: "{{ .Tag }}-next"
changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'

13
LICENSE

@ -1,4 +1,6 @@
Copyright (c) 2017 Josh Baker
MIT License
Copyright (c) 2019 James Mills
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@ -7,14 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

20
LICENSE.old

@ -0,0 +1,20 @@
Copyright (c) 2017 Josh Baker
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

39
Makefile

@ -1,2 +1,37 @@
all:
go build -o kvnode-server cmd/kvnode-server/main.go
.PHONY: dev build install image release profile bench test clean
CGO_ENABLED=0
VERSION=$(shell git describe --abbrev=0 --tags)
COMMIT=$(shell git rev-parse --short HEAD)
all: dev
dev: build
@./bitraft --version
build: clean
@go build \
-tags "netgo static_build" -installsuffix netgo \
-ldflags "-w -X $(shell go list).Version=$(VERSION) -X $(shell go list).Commit=$(COMMIT)" \
.
install: build
@go install .
image:
@docker build -t prologic/bitraft .
release:
@./tools/release.sh
profile: build
@go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench ./...
bench: build
@go test -v -benchmem -bench=. ./...
test: build
@go test -v -cover -coverprofile=coverage.txt -covermode=atomic -coverpkg=$(shell go list) -race ./...
clean:
@git clean -f -d -X

30
README.md

@ -1,9 +1,14 @@
# kvnode
# bitraft
Minimal Key/Value store with basic Redis support.
A [Bitcask](https://github.com/prologic/bitcask) Distributed Key/Value store
using [Raft](https://github.com/hashicorp/raft) for concensus with a
[Redis](https://redis.org) compatible API written in [Go](https://golang.org).
- Redis API
- LevelDB disk-based storage
Based off of [kvnode](https://github.com/tidwall/kvnode).
(See [LICENSE.old](/LICENSE.old))
- Redis compatible API
- Bitcask disk-based storage
- Raft support with [Finn](https://github.com/tidwall/finn) commands
- Compatible with existing Redis clients
@ -13,10 +18,7 @@ Commands:
SET key value
GET key
DEL key [key ...]
PDEL pattern
KEYS pattern [PIVOT prefix] [LIMIT count] [DESC] [WITHVALUES]
MSET key value [key value ...]
MGET key [key ...]
KEYS [WITHVALUES]
FLUSHDB
SHUTDOWN
```
@ -56,11 +58,11 @@ Ideally you call `RAFTSNAPSHOT` and then store the state.bin on some other serve
To restore:
- Create a new raft cluster
- Download the state.bin snapshot
- Pipe the commands using the `kvnode-server --parse-snapshot` and `redis-cli --pipe` commands
- Pipe the commands using the `bitraft --parse-snapshot` and `redis-cli --pipe` commands
Example:
```
kvnode-server --parse-snapshot state.bin | redis-cli -h 10.0.1.5 -p 4920 --pipe
bitraft --parse-snapshot state.bin | redis-cli -h 10.0.1.5 -p 4920 --pipe
```
This will execute all of the `state.bin` commands on the leader at `10.0.1.5:4920`
@ -68,9 +70,9 @@ This will execute all of the `state.bin` commands on the leader at `10.0.1.5:492
For information on the `redis-cli --pipe` command see [Redis Mass Insert](https://redis.io/topics/mass-insert).
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
## License
kvnode source code is available under the MIT [License](/LICENSE).
bitraft source code is available under the MIT [License](/LICENSE).
Previously based off of [kvnode](https://github.com/tidwall/kvnode).
(See [LICENSE.old](/LICENSE.old))

68
cmd/kvnode-server/main.go

@ -1,68 +0,0 @@
package main
import (
"flag"
"os"
"strings"
"github.com/tidwall/finn"
"github.com/tidwall/kvnode"
"github.com/tidwall/redlog"
)
func main() {
var addr string
var dir string
var logdir string
var join string
var consistency string
var durability string
var fastlog bool
var parseSnapshot string
flag.BoolVar(&fastlog, "fastlog", false, "use FastLog as the raftlog")
flag.StringVar(&addr, "addr", "127.0.0.1:4920", "bind/discoverable ip:port")
flag.StringVar(&dir, "data", "data", "data directory")
flag.StringVar(&logdir, "log-dir", "", "log directory. If blank it will equals --data")
flag.StringVar(&join, "join", "", "Join a cluster by providing an address")
flag.StringVar(&consistency, "consistency", "high", "Consistency (low,medium,high)")
flag.StringVar(&durability, "durability", "high", "Durability (low,medium,high)")
flag.StringVar(&parseSnapshot, "parse-snapshot", "", "Parse and output a snapshot to Redis format")
flag.Parse()
var log = redlog.New(os.Stderr)
if parseSnapshot != "" {
err := kvnode.WriteRedisCommandsFromSnapshot(os.Stdout, parseSnapshot)
if err != nil {
log.Warningf("%v", err)
os.Exit(1)
}
return
}
var lconsistency finn.Level
switch strings.ToLower(consistency) {
default:
log.Warningf("invalid --consistency")
case "low":
lconsistency = finn.Low
case "medium", "med":
lconsistency = finn.Medium
case "high":
lconsistency = finn.High
}
var ldurability finn.Level
switch strings.ToLower(durability) {
default:
log.Warningf("invalid --durability")
case "low":
ldurability = finn.Low
case "medium", "med":
ldurability = finn.Medium
case "high":
ldurability = finn.High
}
if logdir == "" {
logdir = dir
}
if err := kvnode.ListenAndServe(addr, join, dir, logdir, fastlog, lconsistency, ldurability); err != nil {
log.Warningf("%v", err)
}
}

60
glide.lock

@ -1,60 +0,0 @@
hash: b2c61803983deb77d83454ab32ad119dfa22020e411d1080d9c76a9597fe94a2
updated: 2017-02-18T08:42:16.670808667-07:00
imports:
- name: github.com/armon/go-metrics
version: 93f237eba9b0602f3e73710416558854a81d9337
- name: github.com/boltdb/bolt
version: e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd
- name: github.com/garyburd/redigo
version: 8873b2f1995f59d4bcdd2b0dc9858e2cb9bf0c13
subpackages:
- internal
- redis
- name: github.com/golang/snappy
version: 553a641470496b2327abcac10b36396bd98e45c9
- name: github.com/hashicorp/go-msgpack
version: fa3f63826f7c23912c15263591e65d54d080b458
subpackages:
- codec
- name: github.com/hashicorp/raft
version: 5f09c4ffdbcd2a53768e78c47717415de12b6728
- name: github.com/syndtr/goleveldb
version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65
subpackages:
- leveldb
- leveldb/cache
- leveldb/comparer
- leveldb/errors
- leveldb/filter
- leveldb/iterator
- leveldb/journal
- leveldb/memdb
- leveldb/opt
- leveldb/storage
- leveldb/table
- leveldb/util
- name: github.com/tidwall/finn
version: a7509cd2c0eac255748e96cbf11b0343d6323016
- name: github.com/tidwall/match
version: 173748da739a410c5b0b813b956f89ff94730b4c
- name: github.com/tidwall/raft-boltdb
version: 25b87f2c567777bf5d12195b74aea260bae9dc68
- name: github.com/tidwall/raft-fastlog
version: 2f0d0a0ce55888c2572c52faf0e13ce34f3bf388
- name: github.com/tidwall/raft-leveldb
version: ada471496dc9ca9917f1abbf5625bb5dcba37381
- name: github.com/tidwall/raft-redcon
version: 79c5e64fd86b2fa1c9deab1a1204586552516f05
- name: github.com/tidwall/redcon
version: 8b15dea700da4e6a9e28d4ab6c1d2275efa3f2e6
- name: github.com/tidwall/redlog
version: 550629ebbfa9925a73f69cce7cdd2e8dae52c713
- name: golang.org/x/crypto
version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8
subpackages:
- ssh/terminal
- name: golang.org/x/sys
version: 075e574b89e4c2d22f2286a7e2b919519c6f3547
subpackages:
- unix
testImports: []

11
glide.yaml

@ -1,11 +0,0 @@
package: github.com/tidwall/kvnode
import:
- package: github.com/syndtr/goleveldb
subpackages:
- leveldb
- leveldb/filter
- leveldb/opt
- package: github.com/tidwall/finn
- package: github.com/tidwall/match
- package: github.com/tidwall/redcon
- package: github.com/tidwall/redlog

23
go.mod

@ -3,19 +3,36 @@ module github.com/prologic/bitraft
require (
github.com/armon/go-metrics v0.0.0-20170114134737-93f237eba9b0
github.com/boltdb/bolt v0.0.0-20170131192018-e9cf4fae01b5
github.com/coreos/etcd v3.3.12+incompatible // indirect
github.com/garyburd/redigo v1.0.0
github.com/golang/protobuf v1.3.1 // indirect
github.com/golang/snappy v0.0.0-20170215233205-553a64147049
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c
github.com/hashicorp/raft v0.0.0-20160824023112-5f09c4ffdbcd
github.com/kisielk/errcheck v1.2.0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/prologic/bitcask v0.0.0-20190319214626-2d9bfbb408e1
github.com/sirupsen/logrus v1.4.0
github.com/spf13/afero v1.2.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.3.2 // indirect
github.com/stretchr/testify v1.3.0
github.com/syndtr/goleveldb v0.0.0-20161227110519-23851d93a229
github.com/tidwall/finn v0.1.0
github.com/tidwall/kvnode v0.0.0-20170302195729-a9ab8591d78c
github.com/tidwall/match v0.0.0-20160830173930-173748da739a
github.com/tidwall/raft-boltdb v0.0.0-20160909211738-25b87f2c5677
github.com/tidwall/raft-fastlog v0.0.0-20160922202426-2f0d0a0ce558
github.com/tidwall/raft-leveldb v0.0.0-20170127185243-ada471496dc9
github.com/tidwall/raft-redcon v0.1.0
github.com/tidwall/redcon v0.0.0-20170209173215-8b15dea700da
github.com/tidwall/redcon v0.9.0
github.com/tidwall/redlog v0.0.0-20170204190734-550629ebbfa9
golang.org/x/crypto v0.0.0-20170209233901-453249f01cfe
golang.org/x/sys v0.0.0-20170217003442-075e574b89e4
github.com/ugorji/go/codec v0.0.0-20190316192920-e2bddce071ad // indirect
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a
golang.org/x/net v0.0.0-20190318221613-d196dffd7c2b // indirect
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect
golang.org/x/sys v0.0.0-20190318195719-6c81ef8f67ca
golang.org/x/tools v0.0.0-20190318200714-bb1270c20edf // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
)

125
go.sum

@ -0,0 +1,125 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20170114134737-93f237eba9b0 h1:LtjLavjiw0eiPS4NORxiohz+gV/sB7vm1y4guq5lk2Q=
github.com/armon/go-metrics v0.0.0-20170114134737-93f237eba9b0/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/boltdb/bolt v0.0.0-20170131192018-e9cf4fae01b5 h1:CEa4aInusZzqB0d5gjtQFqUQjWBa30RQR8mXuu2RnXw=
github.com/boltdb/bolt v0.0.0-20170131192018-e9cf4fae01b5/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/garyburd/redigo v1.0.0 h1:W6d6zr96WMrMxQws1I4sc7rrJ1dbQK5KrC+NwH0ReTM=
github.com/garyburd/redigo v1.0.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc=
github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/raft v0.0.0-20160824023112-5f09c4ffdbcd h1:gN6xm3iAclW5DKJWYiXO8tZN25Zy7UsB6Wh/85OB8Bg=
github.com/hashicorp/raft v0.0.0-20160824023112-5f09c4ffdbcd/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prologic/bitcask v0.0.0-20190318094708-ebefd0abf42c h1:C3TnIdhjiQ02ueVIUYlUfhQgjORBIhyTYMvdtFw1WaA=
github.com/prologic/bitcask v0.0.0-20190318094708-ebefd0abf42c/go.mod h1:VaC8qqH0Pjnl+HdhH/zA1b6xSDW3I1PAJGVSTDQ/xJ8=
github.com/prologic/bitcask v0.0.0-20190319085503-e117ffd2e969 h1:76sHN6qOX+n+UUKvcDKmWCXaQqfOc/WUHt5EnKZqepk=
github.com/prologic/bitcask v0.0.0-20190319085503-e117ffd2e969/go.mod h1:VaC8qqH0Pjnl+HdhH/zA1b6xSDW3I1PAJGVSTDQ/xJ8=
github.com/prologic/bitcask v0.0.0-20190319214626-2d9bfbb408e1 h1:KqRtI5Un5G77ChLAvcY16JYZSaPSf4IgySwV8C1Tdpk=
github.com/prologic/bitcask v0.0.0-20190319214626-2d9bfbb408e1/go.mod h1:VaC8qqH0Pjnl+HdhH/zA1b6xSDW3I1PAJGVSTDQ/xJ8=
github.com/prologic/trie v0.0.0-20190316011403-395e39dac705 h1:2J+cSlAeECj0lfMKSmM7n5OlIio+yLovaKLZJzwLc6U=
github.com/prologic/trie v0.0.0-20190316011403-395e39dac705/go.mod h1:LFuDmpHJGmciXd8Rl5YMhVlLMps9gz2GtYLzwxrFhzs=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/syndtr/goleveldb v0.0.0-20161227110519-23851d93a229 h1:arXQNTPyszL9q5nmGtSXyGocRDQRxdtoSS25nZgPvCI=
github.com/syndtr/goleveldb v0.0.0-20161227110519-23851d93a229/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/tidwall/finn v0.1.0 h1:hZbWyUG8yL6+TK37imRdOkO4Vi6FnEQkZ1rfbY/QaRk=
github.com/tidwall/finn v0.1.0/go.mod h1:p8J9D7X1/zcfFH9iJVCLH41XhG+Hra3HDomUM9GuXQA=
github.com/tidwall/kvnode v0.0.0-20170302195729-a9ab8591d78c h1:kY+01Vcq2XRc+uhI68CsNo33VkCoNFmB+3+dgxhXHrs=
github.com/tidwall/kvnode v0.0.0-20170302195729-a9ab8591d78c/go.mod h1:cssC4JW9kBUS+xXQ3ztZCJX7Uk8gPMcN9Wt6Fm3wV9o=
github.com/tidwall/match v0.0.0-20160830173930-173748da739a h1:jkSy//MOkpJzPmsdrxnM+wiF/wdmVCFGegxccsSkm2Q=
github.com/tidwall/match v0.0.0-20160830173930-173748da739a/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/raft-boltdb v0.0.0-20160909211738-25b87f2c5677 h1:8FkXr+GCV4wb8WAct/V1vKB/Ivy11Y+fm919EHgdfWA=
github.com/tidwall/raft-boltdb v0.0.0-20160909211738-25b87f2c5677/go.mod h1:O7b2tvwZmC+IFu8djLOZj0jc/tjssDPiJ8xIt+U2jTU=
github.com/tidwall/raft-fastlog v0.0.0-20160922202426-2f0d0a0ce558 h1:hQYEIfMzrH6LRzjz7Jp5Rv8jrty1bAR5M0DjOYSxxks=
github.com/tidwall/raft-fastlog v0.0.0-20160922202426-2f0d0a0ce558/go.mod h1:KNwBhka/a5Ucw5bfEzKHTEKuCO2Do1tKs+kDdu3Sbb4=
github.com/tidwall/raft-leveldb v0.0.0-20170127185243-ada471496dc9 h1:Z5QMqF/MSuvnrTibHqs/xx+ZE5gypLV02YU8Ry4kJ7A=
github.com/tidwall/raft-leveldb v0.0.0-20170127185243-ada471496dc9/go.mod h1:KNAMyK8s/oUOTbIL/T07fTL6/EgJfHhK8XeeEPq35eU=
github.com/tidwall/raft-redcon v0.1.0 h1:qwYaFaAVNFleY2EFm0j7UK4vEpoNa19ohH7U4idbg+s=
github.com/tidwall/raft-redcon v0.1.0/go.mod h1:YhoECfJs8MXbrwak9H7wKYDMZ3rMaB7el7zZ7MRw9Xw=
github.com/tidwall/redcon v0.0.0-20170209173215-8b15dea700da h1:EdqrYLlEqpDGkgb5ELV7pyT5EXw+Rg3eXkiplGRDUyc=
github.com/tidwall/redcon v0.0.0-20170209173215-8b15dea700da/go.mod h1:bdYBm4rlcWpst2XMwKVzWDF9CoUxEbUmM7CQrKeOZas=
github.com/tidwall/redcon v0.9.0 h1:tiT9DLAoohsdNaFg9Si5dRsv9+FjvZYnhMOEtSFwBqA=
github.com/tidwall/redcon v0.9.0/go.mod h1:bdYBm4rlcWpst2XMwKVzWDF9CoUxEbUmM7CQrKeOZas=
github.com/tidwall/redlog v0.0.0-20170204190734-550629ebbfa9 h1:UaKqBNPFP34MjxJarhxUwlCdn60KuRN08uLC+Bxo348=
github.com/tidwall/redlog v0.0.0-20170204190734-550629ebbfa9/go.mod h1:NssoNA+Uwqd5WHKkVwAzO7AT6VuG3wiC8r5nBqds3Ao=
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v0.0.0-20190316192920-e2bddce071ad/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
golang.org/x/crypto v0.0.0-20170209233901-453249f01cfe h1:uSczOUTEw513p4N5SoJC1T8jTNx/T/2oj6pO5qhRgSM=
golang.org/x/crypto v0.0.0-20170209233901-453249f01cfe/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a h1:YX8ljsm6wXlHZO+aRz9Exqr0evNhKRNe5K/gi+zKh4U=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190318221613-d196dffd7c2b/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170217003442-075e574b89e4/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190318195719-6c81ef8f67ca h1:o2TLx1bGN3W+Ei0EMU5fShLupLmTOU95KvJJmfYhAzM=
golang.org/x/sys v0.0.0-20190318195719-6c81ef8f67ca/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190318200714-bb1270c20edf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

101
main.go

@ -0,0 +1,101 @@
package main
import (
"fmt"
"os"
"strings"
log "github.com/sirupsen/logrus"
flag "github.com/spf13/pflag"
"github.com/tidwall/finn"
)
var (
debug bool
version bool
maxDatafileSize int
bind string
dir string
logdir string
join string
consistency string
durability string
parseSnapshot string
)
func init() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [options] <path>\n", os.Args[0])
flag.PrintDefaults()
}
flag.BoolVarP(&version, "version", "V", false, "display version information")
flag.BoolVarP(&debug, "debug", "D", false, "enable debug logging")
flag.IntVar(&maxDatafileSize, "max-datafile-size", 1<<20, "maximum datafile size in bytes")
flag.StringVarP(&bind, "bind", "b", "127.0.0.1:4920", "bind/discoverable ip:port")
flag.StringVarP(&dir, "data", "d", "data", "data directory")
flag.StringVarP(&logdir, "log-dir", "l", "", "log directory. If blank it will equals --data")
flag.StringVarP(&join, "join", "j", "", "Join a cluster by providing an address")
flag.StringVar(&consistency, "consistency", "low", "Consistency (low,medium,high)")
flag.StringVar(&durability, "durability", "low", "Durability (low,medium,high)")
flag.StringVar(&parseSnapshot, "parse-snapshot", "", "Parse and output a snapshot to Redis format")
}
func main() {
flag.Parse()
if debug {
log.SetLevel(log.DebugLevel)
} else {
log.SetLevel(log.InfoLevel)
}
if version {
fmt.Printf("bitraft version %s", FullVersion())
os.Exit(0)
}
if parseSnapshot != "" {
err := WriteRedisCommandsFromSnapshot(os.Stdout, parseSnapshot)
if err != nil {
log.Warningf("%v", err)
os.Exit(1)
}
return
}
var lconsistency finn.Level
switch strings.ToLower(consistency) {
default:
log.Warningf("invalid --consistency")
case "low":
lconsistency = finn.Low
case "medium", "med":
lconsistency = finn.Medium
case "high":
lconsistency = finn.High
}
var ldurability finn.Level
switch strings.ToLower(durability) {
default:
log.Warningf("invalid --durability")
case "low":
ldurability = finn.Low
case "medium", "med":
ldurability = finn.Medium
case "high":
ldurability = finn.High
}
if logdir == "" {
logdir = dir
}
if err := ListenAndServe(bind, join, dir, logdir, lconsistency, ldurability); err != nil {
log.Warningf("%v", err)
}
}

386
server.go

@ -1,8 +1,7 @@
package kvnode
package main
import (
"bufio"
"bytes"
"compress/gzip"
"encoding/binary"
"errors"
@ -15,45 +14,38 @@ import (
"sync"
"time"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/prologic/bitcask"
log "github.com/sirupsen/logrus"
"github.com/tidwall/finn"
"github.com/tidwall/match"
"github.com/tidwall/redcon"
"github.com/tidwall/redlog"
)
const defaultTCPKeepAlive = time.Minute * 5
var (
errSyntaxError = errors.New("syntax error")
log = redlog.New(os.Stderr)
)
func ListenAndServe(addr, join, dir, logdir string, fastlog bool, consistency, durability finn.Level) error {
var opts finn.Options
if fastlog {
opts.Backend = finn.LevelDB
} else {
opts.Backend = finn.FastLog
}
opts.Consistency = consistency
opts.Durability = durability
opts.ConnAccept = func(conn redcon.Conn) bool {
if tcp, ok := conn.NetConn().(*net.TCPConn); ok {
if err := tcp.SetKeepAlive(true); err != nil {
log.Warningf("could not set keepalive: %s",
tcp.RemoteAddr().String())
} else {
err := tcp.SetKeepAlivePeriod(defaultTCPKeepAlive)
if err != nil {
log.Warningf("could not set keepalive period: %s",
func ListenAndServe(addr, join, dir, logdir string, consistency, durability finn.Level) error {
opts := finn.Options{
Backend: finn.FastLog,
Consistency: consistency,
Durability: durability,
ConnAccept: func(conn redcon.Conn) bool {
if tcp, ok := conn.NetConn().(*net.TCPConn); ok {
if err := tcp.SetKeepAlive(true); err != nil {
log.Warningf("could not set keepalive: %s",
tcp.RemoteAddr().String())
} else {
err := tcp.SetKeepAlivePeriod(defaultTCPKeepAlive)
if err != nil {
log.Warningf("could not set keepalive period: %s",
tcp.RemoteAddr().String())
}
}
}
}
return true
return true
},
}
m, err := NewMachine(dir, addr)
if err != nil {
@ -73,8 +65,7 @@ func ListenAndServe(addr, join, dir, logdir string, fastlog bool, consistency, d
type Machine struct {
mu sync.RWMutex
dir string
db *leveldb.DB
opts *opt.Options
db *bitcask.Bitcask
dbPath string
addr string
closed bool
@ -87,11 +78,7 @@ func NewMachine(dir, addr string) (*Machine, error) {
}
var err error
kvm.dbPath = filepath.Join(dir, "node.db")
kvm.opts = &opt.Options{
NoSync: true,
Filter: filter.NewBloomFilter(10),
}
kvm.db, err = leveldb.OpenFile(kvm.dbPath, kvm.opts)
kvm.db, err = bitcask.Open(kvm.dir)
if err != nil {
return nil, err
}
@ -117,18 +104,10 @@ func (kvm *Machine) Command(
return kvm.cmdEcho(m, conn, cmd)
case "set":
return kvm.cmdSet(m, conn, cmd)
case "mset":
return kvm.cmdMset(m, conn, cmd)
case "get":
return kvm.cmdGet(m, conn, cmd)
case "mget":
return kvm.cmdMget(m, conn, cmd)
case "del":
return kvm.cmdDel(m, conn, cmd, false)
case "pdel":
return kvm.cmdPdel(m, conn, cmd, false)
case "delif":
return kvm.cmdDel(m, conn, cmd, true)
return kvm.cmdDel(m, conn, cmd)
case "keys":
return kvm.cmdKeys(m, conn, cmd)
case "flushdb":
@ -153,12 +132,10 @@ func (kvm *Machine) Restore(rd io.Reader) error {
return err
}
kvm.db = nil
kvm.db, err = leveldb.OpenFile(kvm.dbPath, kvm.opts)
kvm.db, err = bitcask.Open(kvm.dir)
if err != nil {
return err
}
var read int
batch := new(leveldb.Batch)
num := make([]byte, 8)
gzr, err := gzip.NewReader(rd)
if err != nil {
@ -166,12 +143,6 @@ func (kvm *Machine) Restore(rd io.Reader) error {
}
r := bufio.NewReader(gzr)
for {
if read > 4*1024*1024 {
if err := kvm.db.Write(batch, nil); err != nil {
return err
}
read = 0
}
if _, err := io.ReadFull(r, num); err != nil {
if err == io.EOF {
break
@ -189,11 +160,7 @@ func (kvm *Machine) Restore(rd io.Reader) error {
if _, err := io.ReadFull(r, value); err != nil {
return err
}
batch.Put(key, value)
read += (len(key) + len(value))
}
if err := kvm.db.Write(batch, nil); err != nil {
return err
kvm.db.Put(string(key), value)
}
return gzr.Close()
}
@ -266,19 +233,15 @@ func (kvm *Machine) Snapshot(wr io.Writer) error {
kvm.mu.RLock()
defer kvm.mu.RUnlock()
gzw := gzip.NewWriter(wr)
ss, err := kvm.db.GetSnapshot()
if err != nil {
return err
}
defer ss.Release()
iter := ss.NewIterator(nil, nil)
defer iter.Release()
var buf []byte
num := make([]byte, 8)
for ok := iter.First(); ok; ok = iter.Next() {
buf = buf[:0]
key := iter.Key()
value := iter.Value()
err := kvm.db.Fold(func(key string) error {
var buf []byte
value, err := kvm.db.Get(key)
if err != nil {
return err
}
num := make([]byte, 8)
binary.LittleEndian.PutUint64(num, uint64(len(key)))
buf = append(buf, num...)
buf = append(buf, key...)
@ -288,12 +251,13 @@ func (kvm *Machine) Snapshot(wr io.Writer) error {
if _, err := gzw.Write(buf); err != nil {
return err
}
}
if err := gzw.Close(); err != nil {
return nil
})
if err != nil {
return err
}
iter.Release()
return iter.Error()
return gzw.Close()
}
func (kvm *Machine) cmdSet(
@ -306,30 +270,7 @@ func (kvm *Machine) cmdSet(
func() (interface{}, error) {
kvm.mu.Lock()
defer kvm.mu.Unlock()
return nil, kvm.db.Put(makeKey('k', cmd.Args[1]), cmd.Args[2], nil)
},
func(v interface{}) (interface{}, error) {
conn.WriteString("OK")
return nil, nil
},
)
}
func (kvm *Machine) cmdMset(
m finn.Applier, conn redcon.Conn, cmd redcon.Command,
) (interface{}, error) {
if len(cmd.Args) < 3 || (len(cmd.Args)-1)%2 == 1 {
return nil, finn.ErrWrongNumberOfArguments
}
return m.Apply(conn, cmd,
func() (interface{}, error) {
kvm.mu.Lock()
defer kvm.mu.Unlock()
var batch leveldb.Batch
for i := 1; i < len(cmd.Args); i += 2 {
batch.Put(makeKey('k', cmd.Args[i]), cmd.Args[i+1])
}
return nil, kvm.db.Write(&batch, nil)
return nil, kvm.db.Put(string(cmd.Args[1]), cmd.Args[2])
},
func(v interface{}) (interface{}, error) {
conn.WriteString("OK")
@ -345,18 +286,19 @@ func (kvm *Machine) cmdEcho(m finn.Applier, conn redcon.Conn, cmd redcon.Command
conn.WriteBulk(cmd.Args[1])
return nil, nil
}
func (kvm *Machine) cmdGet(m finn.Applier, conn redcon.Conn, cmd redcon.Command) (interface{}, error) {
if len(cmd.Args) != 2 {
return nil, finn.ErrWrongNumberOfArguments
}
key := makeKey('k', cmd.Args[1])
key := string(cmd.Args[1])
return m.Apply(conn, cmd, nil,
func(interface{}) (interface{}, error) {
kvm.mu.RLock()
defer kvm.mu.RUnlock()
value, err := kvm.db.Get(key, nil)
value, err := kvm.db.Get(key)
if err != nil {
if err == leveldb.ErrNotFound {
if err == bitcask.ErrKeyNotFound {
conn.WriteNull()
return nil, nil
}
@ -368,78 +310,20 @@ func (kvm *Machine) cmdGet(m finn.Applier, conn redcon.Conn, cmd redcon.Command)
)
}
func (kvm *Machine) cmdMget(m finn.Applier, conn redcon.Conn, cmd redcon.Command) (interface{}, error) {
if len(cmd.Args) < 2 {
return nil, finn.ErrWrongNumberOfArguments
}
return m.Apply(conn, cmd, nil,
func(interface{}) (interface{}, error) {
kvm.mu.RLock()
defer kvm.mu.RUnlock()
var values [][]byte
for i := 1; i < len(cmd.Args); i++ {
key := makeKey('k', cmd.Args[i])
value, err := kvm.db.Get(key, nil)
if err != nil {
if err == leveldb.ErrNotFound {
values = append(values, nil)
} else {
return nil, err
}
} else {
values = append(values, bcopy(value))
}
}
conn.WriteArray(len(values))
for _, v := range values {
if v == nil {
conn.WriteNull()
} else {
conn.WriteBulk(v)
}
}
return nil, nil
},
)
}
func (kvm *Machine) cmdDel(m finn.Applier, conn redcon.Conn, cmd redcon.Command, delif bool) (interface{}, error) {
if (delif && len(cmd.Args) < 3) || len(cmd.Args) < 2 {
return nil, finn.ErrWrongNumberOfArguments
}
var valueif []byte
func (kvm *Machine) cmdDel(m finn.Applier, conn redcon.Conn, cmd redcon.Command) (interface{}, error) {
var startIdx = 1
if delif {
valueif = cmd.Args[1]
startIdx = 2
}
return m.Apply(conn, cmd,
func() (interface{}, error) {
kvm.mu.Lock()
defer kvm.mu.Unlock()
var batch leveldb.Batch
var n int
for i := startIdx; i < len(cmd.Args); i++ {
key := makeKey('k', cmd.Args[i])
var has bool
var err error
var val []byte
if delif {
val, err = kvm.db.Get(key, nil)
if err == nil {
has = bytes.Contains(val, valueif)
}
} else {
has, err = kvm.db.Has(key, nil)
}
if err != nil && err != leveldb.ErrNotFound {
key := string(cmd.Args[i])
err := kvm.db.Delete(key)
if err != nil {
return 0, err
} else if has {
n++
batch.Delete(key)
}
}
if err := kvm.db.Write(&batch, nil); err != nil {
return nil, err
n++
}
return n, nil
},
@ -451,166 +335,37 @@ func (kvm *Machine) cmdDel(m finn.Applier, conn redcon.Conn, cmd redcon.Command,
)
}
func (kvm *Machine) cmdPdel(m finn.Applier, conn redcon.Conn, cmd redcon.Command, delif bool) (interface{}, error) {
if len(cmd.Args) != 2 {
return nil, finn.ErrWrongNumberOfArguments
}
pattern := makeKey('k', cmd.Args[1])
spattern := string(pattern)
min, max := match.Allowable(spattern)
bmin := []byte(min)
bmax := []byte(max)
return m.Apply(conn, cmd,
func() (interface{}, error) {
kvm.mu.Lock()
defer kvm.mu.Unlock()
var keys [][]byte
iter := kvm.db.NewIterator(nil, nil)
for ok := iter.Seek(bmin); ok; ok = iter.Next() {
rkey := iter.Key()
if bytes.Compare(rkey, bmax) >= 0 {
break
}
skey := string(rkey)
if !match.Match(skey, spattern) {
continue
}
keys = append(keys, bcopy(rkey))
}
iter.Release()
err := iter.Error()
if err != nil {
return nil, err
}
var batch leveldb.Batch
for _, key := range keys {
batch.Delete(key)
}
if err := kvm.db.Write(&batch, nil); err != nil {
return nil, err
}
return len(keys), nil
},
func(v interface{}) (interface{}, error) {
n := v.(int)
conn.WriteInt(n)
return nil, nil
},
)
}
func (kvm *Machine) cmdKeys(m finn.Applier, conn redcon.Conn, cmd redcon.Command) (interface{}, error) {
if len(cmd.Args) < 2 {
return nil, finn.ErrWrongNumberOfArguments
}
var withvalues bool
var pivot []byte
var usingPivot bool
var desc bool
limit := 500
for i := 2; i < len(cmd.Args); i++ {
switch strings.ToLower(string(cmd.Args[i])) {
default:
return nil, errSyntaxError
case "withvalues":
withvalues = true
case "desc":
desc = true
case "pivot":
i++
if i == len(cmd.Args) {
return nil, errSyntaxError
}
pivot = makeKey('k', cmd.Args[i])
usingPivot = true
case "limit":
i++
if i == len(cmd.Args) {
return nil, errSyntaxError
}
n, err := strconv.ParseInt(string(cmd.Args[i]), 10, 64)
if err != nil || n < 0 {
return nil, errSyntaxError
}
limit = int(n)
}
}
pattern := makeKey('k', cmd.Args[1])
spattern := string(pattern)
min, max := match.Allowable(spattern)
bmin := []byte(min)
bmax := []byte(max)
return m.Apply(conn, cmd, nil,
func(interface{}) (interface{}, error) {
kvm.mu.RLock()
defer kvm.mu.RUnlock()
var keys [][]byte
var values [][]byte
iter := kvm.db.NewIterator(nil, nil)
var ok bool
if desc {
if usingPivot && bytes.Compare(pivot, bmax) < 0 {
bmax = pivot
}
ok = iter.Seek(bmax)
if !ok {
ok = iter.Last()
}
} else {
if usingPivot && bytes.Compare(pivot, bmin) > 0 {
bmin = pivot
}
ok = iter.Seek(bmin)
}
step := func() bool {
if desc {
return iter.Prev()
} else {
return iter.Next()
}
}
var inRange bool
for ; ok; ok = step() {
if len(keys) == limit {
break
}
rkey := iter.Key()
if desc {
if !inRange {
if bytes.Compare(rkey, bmax) >= 0 {
continue
}
inRange = true
}
if bytes.Compare(rkey, bmin) < 0 {
break
}
} else {
if !inRange {
if usingPivot {
if bytes.Compare(rkey, bmin) <= 0 {
continue
}
}
inRange = true
}
if bytes.Compare(rkey, bmax) >= 0 {
break
}
}
skey := string(rkey)
if !match.Match(skey, spattern) {
continue
}
keys = append(keys, bcopy(rkey[1:]))
err := kvm.db.Fold(func(key string) error {
keys = append(keys, []byte(key))
if withvalues {
values = append(values, bcopy(iter.Value()))
value, err := kvm.db.Get(key)
if err != nil {
return err
}
values = append(values, value)
}
}
iter.Release()
err := iter.Error()
return nil
})
if err != nil {
return nil, err
}
@ -638,15 +393,7 @@ func (kvm *Machine) cmdFlushdb(m finn.Applier, conn redcon.Conn, cmd redcon.Comm
func() (interface{}, error) {
kvm.mu.Lock()
defer kvm.mu.Unlock()
if err := kvm.db.Close(); err != nil {
panic(err.Error())
}
if err := os.RemoveAll(kvm.dbPath); err != nil {
panic(err.Error())
}
var err error
kvm.db, err = leveldb.OpenFile(kvm.dbPath, kvm.opts)
if err != nil {
if err := kvm.db.Sync(); err != nil {
panic(err.Error())
}
return nil, nil
@ -657,16 +404,3 @@ func (kvm *Machine) cmdFlushdb(m finn.Applier, conn redcon.Conn, cmd redcon.Comm
},
)
}
func makeKey(prefix byte, b []byte) []byte {
key := make([]byte, 1+len(b))
key[0] = prefix
copy(key[1:], b)
return key
}
func bcopy(b []byte) []byte {
r := make([]byte, len(b))
copy(r, b)
return r
}

25
tools/release.sh

@ -0,0 +1,25 @@
#!/bin/sh
# Get the highest tag number
VERSION="$(git describe --abbrev=0 --tags)"
VERSION=${VERSION:-'0.0.0'}
# Get number parts
MAJOR="${VERSION%%.*}"; VERSION="${VERSION#*.}"
MINOR="${VERSION%%.*}"; VERSION="${VERSION#*.}"
PATCH="${VERSION%%.*}"; VERSION="${VERSION#*.}"
# Increase version
PATCH=$((PATCH+1))
TAG="${1}"
if [ "${TAG}" = "" ]; then
TAG="${MAJOR}.${MINOR}.${PATCH}"
fi
echo "Releasing ${TAG} ..."
git tag -a -s -m "Relase ${TAG}" "${TAG}"
git push --tags
goreleaser release --rm-dist

18
version.go

@ -0,0 +1,18 @@
package main
import (
"fmt"
)
var (
// Version release version
Version = "0.0.1"
// Commit will be overwritten automatically by the build system
Commit = "HEAD"
)
// FullVersion returns the full version and commit hash
func FullVersion() string {
return fmt.Sprintf("%s@%s", Version, Commit)
}

15
version_test.go

@ -0,0 +1,15 @@
package main
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFullVersion(t *testing.T) {
assert := assert.New(t)
expected := fmt.Sprintf("%s@%s", Version, Commit)
assert.Equal(expected, FullVersion())
}
Loading…
Cancel
Save