update mod

vendor/github.com/siddontang/ledisdb/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 siddontang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/siddontang/ledisdb/config/config-docker.toml

@@ -0,0 +1,170 @@
+# LedisDB configuration
+
+# Server listen address
+addr = "0.0.0.0:6380"
+
+# Unix socket permissions, 755 by default.
+# Ignored for tcp socket.
+addr_unixsocketperm = "0770"
+
+# Server http listen address, set empty to disable
+http_addr = "0.0.0.0:11181"
+
+# Data store path, all ledisdb's data will be saved here
+data_dir = "/datastore"
+
+# Set the number of databases. You can use `select dbindex` to choose a db.
+# dbindex must be in [0, databases - 1].
+# Default databases is 16, maximum is 10240 now.
+databases = 16
+
+# Log server command, set empty to disable
+access_log = ""
+
+# Set slaveof to enable replication from master, empty, no replication
+# Any write operations except flushall and replication will be disabled in slave mode.
+slaveof = ""
+
+# Readonly mode, slave server is always readonly even readonly = false
+# for readonly mode, only replication and flushall can write
+readonly = false
+
+# Choose which backend storage to use, now support:
+#
+#   leveldb
+#   rocksdb
+#   goleveldb
+#   memory
+#
+db_name = "leveldb"
+
+# If not set, use data_dir/"db_name"_data
+db_path = ""
+
+# Sync commit to disk if possible
+#  0: no sync
+#  1: sync every second
+#  2: sync every commit
+db_sync_commit = 0
+
+# enable replication or not
+use_replication = false
+
+# set connection buffer, you can increase them appropriately
+# more size, more memory used
+conn_read_buffer_size = 10240
+conn_write_buffer_size = 10240
+
+# if connection receives no data after n seconds, it may be dead, close
+# 0 to disable and not check
+conn_keepalive_interval = 0
+
+# checking TTL (time to live) data every n seconds
+# if you set big, the expired data may not be deleted immediately
+ttl_check_interval = 1
+
+[leveldb]
+# for leveldb and goleveldb
+compression = false
+block_size = 32768
+write_buffer_size = 67108864
+cache_size = 524288000
+max_open_files = 1024
+max_file_size = 33554432
+
+[rocksdb]
+# rocksdb has many many configurations,
+# we only list little now, but may add more later.
+# good luck!
+
+# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc
+compression = 0
+block_size = 65536
+write_buffer_size = 134217728
+cache_size = 1073741824
+max_open_files = 1024
+max_write_buffer_num = 6
+min_write_buffer_number_to_merge = 2
+num_levels = 7
+level0_file_num_compaction_trigger = 8
+level0_slowdown_writes_trigger = 16
+level0_stop_writes_trigger = 64
+target_file_size_base = 67108864
+target_file_size_multiplier = 1
+max_bytes_for_level_base = 536870912
+max_bytes_for_level_multiplier = 8
+disable_auto_compactions = false
+disable_data_sync = false
+use_fsync = false
+background_theads = 16
+high_priority_background_threads = 1
+max_background_compactions = 15
+max_background_flushes = 1
+allow_os_buffer = true
+enable_statistics = false
+stats_dump_period_sec = 3600
+# dangerous to set true, write may got lost after a crash
+# you can set true if replication opened, we may recover from replication log,
+# but it is still not a easy work.
+disable_wal = false
+max_manifest_file_size = 20971520
+
+[lmdb]
+map_size = 524288000
+nosync = true
+
+[replication]
+# Path to store replication information(write ahead log, commit log, etc.)
+# if not set, use data_dir/rpl
+path = ""
+
+# If sync is true, the new log must be sent to some slaves, and then commit.
+# It will reduce performance but have better high availability.
+sync = false
+
+# If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log
+wait_sync_time = 500
+
+# If sync is true, wait at most min(wait_max_slave_acks, (n + 1) / 2) to promise syncing ok.
+# n is slave number
+# If 0, wait (n + 1) / 2 acks.
+wait_max_slave_acks = 2
+
+# store name: file, goleveldb
+# change in runtime is very dangerous
+store_name = "file"
+
+# Expire write ahead logs after the given days
+expired_log_days = 7
+
+# for file store, if 0, use default 256MB, max is 1G
+max_log_file_size = 0
+
+# for file store, if 0, use default 50
+max_log_file_num = 0
+
+# for file store, use mmap for file read and write
+use_mmap = true
+
+# Sync log to disk if possible
+#  0: no sync
+#  1: sync every second
+#  2: sync every commit
+sync_log = 0
+
+# Compress the log or not
+compression = false
+
+[snapshot]
+# Path to store snapshot dump file
+# if not set, use data_dir/snapshot
+# snapshot file name format is dmp-2006-01-02T15:04:05.999999999
+path = ""
+
+# Reserve newest max_num snapshot dump files
+max_num = 1
+
+[tls]
+enabled = false
+certificate = "test.crt"
+key = "test.key"

vendor/github.com/siddontang/ledisdb/config/config.go

@@ -0,0 +1,316 @@
+package config
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"sync"
+
+	"fmt"
+
+	"github.com/pelletier/go-toml"
+	"github.com/siddontang/go/ioutil2"
+)
+
+var (
+	ErrNoConfigFile = errors.New("Running without a config file")
+)
+
+const (
+	DefaultAddr string = "127.0.0.1:6380"
+
+	DefaultDBName string = "goleveldb"
+
+	DefaultDataDir string = "./var"
+
+	KB int = 1024
+	MB int = KB * 1024
+	GB int = MB * 1024
+)
+
+type LevelDBConfig struct {
+	Compression     bool `toml:"compression"`
+	BlockSize       int  `toml:"block_size"`
+	WriteBufferSize int  `toml:"write_buffer_size"`
+	CacheSize       int  `toml:"cache_size"`
+	MaxOpenFiles    int  `toml:"max_open_files"`
+	MaxFileSize     int  `toml:"max_file_size"`
+}
+
+type RocksDBConfig struct {
+	Compression                    int  `toml:"compression"`
+	BlockSize                      int  `toml:"block_size"`
+	WriteBufferSize                int  `toml:"write_buffer_size"`
+	CacheSize                      int  `toml:"cache_size"`
+	MaxOpenFiles                   int  `toml:"max_open_files"`
+	MaxWriteBufferNum              int  `toml:"max_write_buffer_num"`
+	MinWriteBufferNumberToMerge    int  `toml:"min_write_buffer_number_to_merge"`
+	NumLevels                      int  `toml:"num_levels"`
+	Level0FileNumCompactionTrigger int  `toml:"level0_file_num_compaction_trigger"`
+	Level0SlowdownWritesTrigger    int  `toml:"level0_slowdown_writes_trigger"`
+	Level0StopWritesTrigger        int  `toml:"level0_stop_writes_trigger"`
+	TargetFileSizeBase             int  `toml:"target_file_size_base"`
+	TargetFileSizeMultiplier       int  `toml:"target_file_size_multiplier"`
+	MaxBytesForLevelBase           int  `toml:"max_bytes_for_level_base"`
+	MaxBytesForLevelMultiplier     int  `toml:"max_bytes_for_level_multiplier"`
+	DisableAutoCompactions         bool `toml:"disable_auto_compactions"`
+	UseFsync                       bool `toml:"use_fsync"`
+	MaxBackgroundCompactions       int  `toml:"max_background_compactions"`
+	MaxBackgroundFlushes           int  `toml:"max_background_flushes"`
+	EnableStatistics               bool `toml:"enable_statistics"`
+	StatsDumpPeriodSec             int  `toml:"stats_dump_period_sec"`
+	BackgroundThreads              int  `toml:"background_theads"`
+	HighPriorityBackgroundThreads  int  `toml:"high_priority_background_threads"`
+	DisableWAL                     bool `toml:"disable_wal"`
+	MaxManifestFileSize            int  `toml:"max_manifest_file_size"`
+}
+
+type LMDBConfig struct {
+	MapSize int  `toml:"map_size"`
+	NoSync  bool `toml:"nosync"`
+}
+
+type ReplicationConfig struct {
+	Path             string `toml:"path"`
+	Sync             bool   `toml:"sync"`
+	WaitSyncTime     int    `toml:"wait_sync_time"`
+	WaitMaxSlaveAcks int    `toml:"wait_max_slave_acks"`
+	ExpiredLogDays   int    `toml:"expired_log_days"`
+	StoreName        string `toml:"store_name"`
+	MaxLogFileSize   int64  `toml:"max_log_file_size"`
+	MaxLogFileNum    int    `toml:"max_log_file_num"`
+	SyncLog          int    `toml:"sync_log"`
+	Compression      bool   `toml:"compression"`
+	UseMmap          bool   `toml:"use_mmap"`
+	MasterPassword   string `toml:"master_password"`
+}
+
+type SnapshotConfig struct {
+	Path   string `toml:"path"`
+	MaxNum int    `toml:"max_num"`
+}
+
+type TLS struct {
+	Enabled     bool   `toml:"enabled"`
+	Certificate string `toml:"certificate"`
+	Key         string `toml:"key"`
+}
+
+type AuthMethod func(c *Config, password string) bool
+
+type Config struct {
+	m sync.RWMutex `toml:"-"`
+
+	AuthPassword string `toml:"auth_password"`
+
+	//AuthMethod custom authentication method
+	AuthMethod AuthMethod `toml:"-"`
+
+	FileName string `toml:"-"`
+
+	// Addr can be empty to assign a local address dynamically
+	Addr string `toml:"addr"`
+
+	AddrUnixSocketPerm string `toml:"addr_unixsocketperm"`
+
+	HttpAddr string `toml:"http_addr"`
+
+	SlaveOf string `toml:"slaveof"`
+
+	Readonly bool `toml:readonly`
+
+	DataDir string `toml:"data_dir"`
+
+	Databases int `toml:"databases"`
+
+	DBName       string `toml:"db_name"`
+	DBPath       string `toml:"db_path"`
+	DBSyncCommit int    `toml:"db_sync_commit"`
+
+	LevelDB LevelDBConfig `toml:"leveldb"`
+	RocksDB RocksDBConfig `toml:"rocksdb"`
+
+	LMDB LMDBConfig `toml:"lmdb"`
+
+	AccessLog string `toml:"access_log"`
+
+	UseReplication bool              `toml:"use_replication"`
+	Replication    ReplicationConfig `toml:"replication"`
+
+	Snapshot SnapshotConfig `toml:"snapshot"`
+
+	ConnReadBufferSize    int `toml:"conn_read_buffer_size"`
+	ConnWriteBufferSize   int `toml:"conn_write_buffer_size"`
+	ConnKeepaliveInterval int `toml:"conn_keepalive_interval"`
+
+	TTLCheckInterval int `toml:"ttl_check_interval"`
+
+	//tls config
+	TLS TLS `toml:"tls"`
+}
+
+func NewConfigWithFile(fileName string) (*Config, error) {
+	data, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg, err := NewConfigWithData(data)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.FileName = fileName
+	return cfg, nil
+}
+
+func NewConfigWithData(data []byte) (*Config, error) {
+	cfg := NewConfigDefault()
+
+	if err := toml.Unmarshal(data, cfg); err != nil {
+		return nil, fmt.Errorf("newConfigwithData: unmarashal: %s", err)
+	}
+
+	cfg.adjust()
+
+	return cfg, nil
+}
+
+func NewConfigDefault() *Config {
+	cfg := new(Config)
+
+	cfg.Addr = DefaultAddr
+	cfg.HttpAddr = ""
+
+	cfg.DataDir = DefaultDataDir
+
+	cfg.DBName = DefaultDBName
+
+	cfg.SlaveOf = ""
+	cfg.Readonly = false
+
+	// Disable Auth by default, by setting password to blank
+	cfg.AuthPassword = ""
+
+	// default databases number
+	cfg.Databases = 16
+
+	// disable access log
+	cfg.AccessLog = ""
+
+	cfg.LMDB.MapSize = 20 * MB
+	cfg.LMDB.NoSync = true
+
+	cfg.UseReplication = false
+	cfg.Replication.WaitSyncTime = 500
+	cfg.Replication.Compression = true
+	cfg.Replication.WaitMaxSlaveAcks = 2
+	cfg.Replication.SyncLog = 0
+	cfg.Replication.UseMmap = true
+	cfg.Snapshot.MaxNum = 1
+
+	cfg.RocksDB.EnableStatistics = false
+	cfg.RocksDB.UseFsync = false
+	cfg.RocksDB.DisableAutoCompactions = false
+	cfg.RocksDB.DisableWAL = false
+
+	cfg.adjust()
+
+	return cfg
+}
+
+func getDefault(d int, s int) int {
+	if s <= 0 {
+		return d
+	}
+
+	return s
+}
+
+func (cfg *Config) adjust() {
+	cfg.LevelDB.adjust()
+
+	cfg.RocksDB.adjust()
+
+	cfg.Replication.ExpiredLogDays = getDefault(7, cfg.Replication.ExpiredLogDays)
+	cfg.Replication.MaxLogFileNum = getDefault(50, cfg.Replication.MaxLogFileNum)
+	cfg.ConnReadBufferSize = getDefault(4*KB, cfg.ConnReadBufferSize)
+	cfg.ConnWriteBufferSize = getDefault(4*KB, cfg.ConnWriteBufferSize)
+	cfg.TTLCheckInterval = getDefault(1, cfg.TTLCheckInterval)
+	cfg.Databases = getDefault(16, cfg.Databases)
+}
+
+func (cfg *LevelDBConfig) adjust() {
+	cfg.CacheSize = getDefault(4*MB, cfg.CacheSize)
+	cfg.BlockSize = getDefault(4*KB, cfg.BlockSize)
+	cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize)
+	cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles)
+	cfg.MaxFileSize = getDefault(32*MB, cfg.MaxFileSize)
+}
+
+func (cfg *RocksDBConfig) adjust() {
+	cfg.CacheSize = getDefault(4*MB, cfg.CacheSize)
+	cfg.BlockSize = getDefault(4*KB, cfg.BlockSize)
+	cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize)
+	cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles)
+	cfg.MaxWriteBufferNum = getDefault(2, cfg.MaxWriteBufferNum)
+	cfg.MinWriteBufferNumberToMerge = getDefault(1, cfg.MinWriteBufferNumberToMerge)
+	cfg.NumLevels = getDefault(7, cfg.NumLevels)
+	cfg.Level0FileNumCompactionTrigger = getDefault(4, cfg.Level0FileNumCompactionTrigger)
+	cfg.Level0SlowdownWritesTrigger = getDefault(16, cfg.Level0SlowdownWritesTrigger)
+	cfg.Level0StopWritesTrigger = getDefault(64, cfg.Level0StopWritesTrigger)
+	cfg.TargetFileSizeBase = getDefault(32*MB, cfg.TargetFileSizeBase)
+	cfg.TargetFileSizeMultiplier = getDefault(1, cfg.TargetFileSizeMultiplier)
+	cfg.MaxBytesForLevelBase = getDefault(32*MB, cfg.MaxBytesForLevelBase)
+	cfg.MaxBytesForLevelMultiplier = getDefault(1, cfg.MaxBytesForLevelMultiplier)
+	cfg.MaxBackgroundCompactions = getDefault(1, cfg.MaxBackgroundCompactions)
+	cfg.MaxBackgroundFlushes = getDefault(1, cfg.MaxBackgroundFlushes)
+	cfg.StatsDumpPeriodSec = getDefault(3600, cfg.StatsDumpPeriodSec)
+	cfg.BackgroundThreads = getDefault(2, cfg.BackgroundThreads)
+	cfg.HighPriorityBackgroundThreads = getDefault(1, cfg.HighPriorityBackgroundThreads)
+	cfg.MaxManifestFileSize = getDefault(20*MB, cfg.MaxManifestFileSize)
+}
+
+func (cfg *Config) Dump(w io.Writer) error {
+	data, err := toml.Marshal(*cfg)
+	if err != nil {
+		return err
+	}
+	if _, err := w.Write(data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (cfg *Config) DumpFile(fileName string) error {
+	var b bytes.Buffer
+
+	if err := cfg.Dump(&b); err != nil {
+		return err
+	}
+
+	return ioutil2.WriteFileAtomic(fileName, b.Bytes(), 0644)
+}
+
+func (cfg *Config) Rewrite() error {
+	if len(cfg.FileName) == 0 {
+		return ErrNoConfigFile
+	}
+
+	return cfg.DumpFile(cfg.FileName)
+}
+
+func (cfg *Config) GetReadonly() bool {
+	cfg.m.RLock()
+	b := cfg.Readonly
+	cfg.m.RUnlock()
+	return b
+}
+
+func (cfg *Config) SetReadonly(b bool) {
+	cfg.m.Lock()
+	cfg.Readonly = b
+	cfg.m.Unlock()
+}

vendor/github.com/siddontang/ledisdb/config/config.toml

@@ -0,0 +1,170 @@
+# LedisDB configuration
+
+# Server listen address
+addr = "127.0.0.1:6380"
+
+# Unix socket permissions, 755 by default.
+# Ignored for tcp socket.
+addr_unixsocketperm = "0770"
+
+# Server http listen address, set empty to disable
+http_addr = "127.0.0.1:11181"
+
+# Data store path, all ledisdb's data will be saved here
+data_dir = "/tmp/ledis_server"
+
+# Set the number of databases. You can use `select dbindex` to choose a db.
+# dbindex must be in [0, databases - 1].
+# Default databases is 16, maximum is 10240 now.
+databases = 16
+
+# Log server command, set empty to disable
+access_log = ""
+
+# Set slaveof to enable replication from master, empty, no replication
+# Any write operations except flushall and replication will be disabled in slave mode.
+slaveof = ""
+
+# Readonly mode, slave server is always readonly even readonly = false
+# for readonly mode, only replication and flushall can write
+readonly = false
+
+# Choose which backend storage to use, now support:
+#
+#   leveldb
+#   rocksdb
+#   goleveldb
+#   memory
+#
+db_name = "leveldb"
+
+# If not set, use data_dir/"db_name"_data
+db_path = ""
+
+# Sync commit to disk if possible
+#  0: no sync
+#  1: sync every second
+#  2: sync every commit
+db_sync_commit = 0
+
+# enable replication or not
+use_replication = false
+
+# set connection buffer, you can increase them appropriately
+# more size, more memory used
+conn_read_buffer_size = 10240
+conn_write_buffer_size = 10240
+
+# if connection receives no data after n seconds, it may be dead, close
+# 0 to disable and not check
+conn_keepalive_interval = 0
+
+# checking TTL (time to live) data every n seconds
+# if you set big, the expired data may not be deleted immediately
+ttl_check_interval = 1
+
+[leveldb]
+# for leveldb and goleveldb
+compression = false
+block_size = 32768
+write_buffer_size = 67108864
+cache_size = 524288000
+max_open_files = 1024
+max_file_size = 33554432
+
+[rocksdb]
+# rocksdb has many many configurations,
+# we only list little now, but may add more later.
+# good luck!
+
+# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc
+compression = 0
+block_size = 65536
+write_buffer_size = 134217728
+cache_size = 1073741824
+max_open_files = 1024
+max_write_buffer_num = 6
+min_write_buffer_number_to_merge = 2
+num_levels = 7
+level0_file_num_compaction_trigger = 8
+level0_slowdown_writes_trigger = 16
+level0_stop_writes_trigger = 64
+target_file_size_base = 67108864
+target_file_size_multiplier = 1
+max_bytes_for_level_base = 536870912
+max_bytes_for_level_multiplier = 8
+disable_auto_compactions = false
+disable_data_sync = false
+use_fsync = false
+background_theads = 16
+high_priority_background_threads = 1
+max_background_compactions = 15
+max_background_flushes = 1
+allow_os_buffer = true
+enable_statistics = false
+stats_dump_period_sec = 3600
+# dangerous to set true, write may got lost after a crash
+# you can set true if replication opened, we may recover from replication log,
+# but it is still not a easy work.
+disable_wal = false
+max_manifest_file_size = 20971520
+
+[lmdb]
+map_size = 524288000
+nosync = true
+
+[replication]
+# Path to store replication information(write ahead log, commit log, etc.)
+# if not set, use data_dir/rpl
+path = ""
+
+# If sync is true, the new log must be sent to some slaves, and then commit.
+# It will reduce performance but have better high availability.
+sync = false
+
+# If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log
+wait_sync_time = 500
+
+# If sync is true, wait at most min(wait_max_slave_acks, (n + 1) / 2) to promise syncing ok.
+# n is slave number
+# If 0, wait (n + 1) / 2 acks.
+wait_max_slave_acks = 2
+
+# store name: file, goleveldb
+# change in runtime is very dangerous
+store_name = "file"
+
+# Expire write ahead logs after the given days
+expired_log_days = 7
+
+# for file store, if 0, use default 256MB, max is 1G
+max_log_file_size = 0
+
+# for file store, if 0, use default 50
+max_log_file_num = 0
+
+# for file store, use mmap for file read and write
+use_mmap = true
+
+# Sync log to disk if possible
+#  0: no sync
+#  1: sync every second
+#  2: sync every commit
+sync_log = 0
+
+# Compress the log or not
+compression = false
+
+[snapshot]
+# Path to store snapshot dump file
+# if not set, use data_dir/snapshot
+# snapshot file name format is dmp-2006-01-02T15:04:05.999999999
+path = ""
+
+# Reserve newest max_num snapshot dump files
+max_num = 1
+
+[tls]
+enabled = true
+certificate = "test.crt"
+key = "test.key"

vendor/github.com/siddontang/ledisdb/ledis/batch.go

@@ -0,0 +1,139 @@
+package ledis
+
+import (
+	"sync"
+
+	"github.com/siddontang/go/log"
+	"github.com/siddontang/ledisdb/rpl"
+	"github.com/siddontang/ledisdb/store"
+)
+
+type batch struct {
+	l *Ledis
+
+	*store.WriteBatch
+
+	sync.Locker
+
+	//	tx *Tx
+}
+
+func (b *batch) Commit() error {
+	if b.l.cfg.GetReadonly() {
+		return ErrWriteInROnly
+	}
+
+	return b.l.handleCommit(b.WriteBatch, b.WriteBatch)
+
+	// if b.tx == nil {
+	// 	return b.l.handleCommit(b.WriteBatch, b.WriteBatch)
+	// } else {
+	// 	if b.l.r != nil {
+	// 		if err := b.tx.data.Append(b.WriteBatch.BatchData()); err != nil {
+	// 			return err
+	// 		}
+	// 	}
+	// 	return b.WriteBatch.Commit()
+	// }
+}
+
+func (b *batch) Lock() {
+	b.Locker.Lock()
+}
+
+func (b *batch) Unlock() {
+	b.WriteBatch.Rollback()
+	b.Locker.Unlock()
+}
+
+func (b *batch) Put(key []byte, value []byte) {
+	b.WriteBatch.Put(key, value)
+}
+
+func (b *batch) Delete(key []byte) {
+	b.WriteBatch.Delete(key)
+}
+
+type dbBatchLocker struct {
+	l      *sync.Mutex
+	wrLock *sync.RWMutex
+}
+
+func (l *dbBatchLocker) Lock() {
+	l.wrLock.RLock()
+	l.l.Lock()
+}
+
+func (l *dbBatchLocker) Unlock() {
+	l.l.Unlock()
+	l.wrLock.RUnlock()
+}
+
+// type txBatchLocker struct {
+// }
+
+// func (l *txBatchLocker) Lock()   {}
+// func (l *txBatchLocker) Unlock() {}
+
+// type multiBatchLocker struct {
+// }
+
+// func (l *multiBatchLocker) Lock()   {}
+// func (l *multiBatchLocker) Unlock() {}
+
+func (l *Ledis) newBatch(wb *store.WriteBatch, locker sync.Locker) *batch {
+	b := new(batch)
+	b.l = l
+	b.WriteBatch = wb
+
+	b.Locker = locker
+
+	return b
+}
+
+type commiter interface {
+	Commit() error
+}
+
+type commitDataGetter interface {
+	Data() []byte
+}
+
+func (l *Ledis) handleCommit(g commitDataGetter, c commiter) error {
+	l.commitLock.Lock()
+
+	var err error
+	if l.r != nil {
+		var rl *rpl.Log
+		if rl, err = l.r.Log(g.Data()); err != nil {
+			l.commitLock.Unlock()
+
+			log.Fatalf("write wal error %s", err.Error())
+			return err
+		}
+
+		l.propagate(rl)
+
+		if err = c.Commit(); err != nil {
+			l.commitLock.Unlock()
+
+			log.Fatalf("commit error %s", err.Error())
+			l.noticeReplication()
+			return err
+		}
+
+		if err = l.r.UpdateCommitID(rl.ID); err != nil {
+			l.commitLock.Unlock()
+
+			log.Fatalf("update commit id error %s", err.Error())
+			l.noticeReplication()
+			return err
+		}
+	} else {
+		err = c.Commit()
+	}
+
+	l.commitLock.Unlock()
+
+	return err
+}

vendor/github.com/siddontang/ledisdb/ledis/const.go

@@ -0,0 +1,150 @@
+package ledis
+
+import (
+	"errors"
+)
+
+// Version is for version
+const Version = "0.5"
+
+// DataType is defined for the different types
+type DataType byte
+
+// for out use
+const (
+	KV DataType = iota
+	LIST
+	HASH
+	SET
+	ZSET
+)
+
+func (d DataType) String() string {
+	switch d {
+	case KV:
+		return KVName
+	case LIST:
+		return ListName
+	case HASH:
+		return HashName
+	case SET:
+		return SetName
+	case ZSET:
+		return ZSetName
+	default:
+		return "unknown"
+	}
+}
+
+// For different type name
+const (
+	KVName   = "KV"
+	ListName = "LIST"
+	HashName = "HASH"
+	SetName  = "SET"
+	ZSetName = "ZSET"
+)
+
+// for backend store
+const (
+	NoneType   byte = 0
+	KVType     byte = 1
+	HashType   byte = 2
+	HSizeType  byte = 3
+	ListType   byte = 4
+	LMetaType  byte = 5
+	ZSetType   byte = 6
+	ZSizeType  byte = 7
+	ZScoreType byte = 8
+	// BitType     byte = 9
+	// BitMetaType byte = 10
+	SetType   byte = 11
+	SSizeType byte = 12
+
+	maxDataType byte = 100
+
+	/*
+		I make a big mistake about TTL time key format and have to use a new one (change 101 to 103).
+		You must run the ledis-upgrade-ttl to upgrade db.
+	*/
+	ObsoleteExpTimeType byte = 101
+	ExpMetaType         byte = 102
+	ExpTimeType         byte = 103
+
+	MetaType byte = 201
+)
+
+// TypeName is the map of type -> name
+var TypeName = map[byte]string{
+	KVType:     "kv",
+	HashType:   "hash",
+	HSizeType:  "hsize",
+	ListType:   "list",
+	LMetaType:  "lmeta",
+	ZSetType:   "zset",
+	ZSizeType:  "zsize",
+	ZScoreType: "zscore",
+	// BitType:     "bit",
+	// BitMetaType: "bitmeta",
+	SetType:     "set",
+	SSizeType:   "ssize",
+	ExpTimeType: "exptime",
+	ExpMetaType: "expmeta",
+}
+
+const (
+	defaultScanCount int = 10
+)
+
+var (
+	errKeySize        = errors.New("invalid key size")
+	errValueSize      = errors.New("invalid value size")
+	errHashFieldSize  = errors.New("invalid hash field size")
+	errSetMemberSize  = errors.New("invalid set member size")
+	errZSetMemberSize = errors.New("invalid zset member size")
+	errExpireValue    = errors.New("invalid expire value")
+	errListIndex      = errors.New("invalid list index")
+)
+
+// For different const size configuration
+const (
+	// max allowed databases
+	MaxDatabases int = 10240
+
+	// max key size
+	MaxKeySize int = 1024
+
+	// max hash field size
+	MaxHashFieldSize int = 1024
+
+	// max zset member size
+	MaxZSetMemberSize int = 1024
+
+	// max set member size
+	MaxSetMemberSize int = 1024
+
+	// max value size
+	MaxValueSize int = 1024 * 1024 * 1024
+)
+
+// For different common errors
+var (
+	ErrScoreMiss     = errors.New("zset score miss")
+	ErrWriteInROnly  = errors.New("write not support in readonly mode")
+	ErrRplInRDWR     = errors.New("replication not support in read write mode")
+	ErrRplNotSupport = errors.New("replication not support")
+)
+
+// const (
+// 	DBAutoCommit    uint8 = 0x0
+// 	DBInTransaction uint8 = 0x1
+// 	DBInMulti       uint8 = 0x2
+// )
+
+// For bit operation
+const (
+	BitAND = "and"
+	BitOR  = "or"
+	BitXOR = "xor"
+	BitNot = "not"
+)

vendor/github.com/siddontang/ledisdb/ledis/doc.go

@@ -0,0 +1,58 @@
+// Package ledis is a high performance embedded NoSQL.
+//
+// Ledis supports various data structure like kv, list, hash and zset like redis.
+//
+// Other features include replication, data with a limited time-to-live.
+//
+// Usage
+//
+// First create a ledis instance before use:
+//
+//  l := ledis.Open(cfg)
+//
+// cfg is a Config instance which contains configuration for ledis use,
+// like DataDir (root directory for ledis working to store data).
+//
+// After you create a ledis instance, you can select a DB to store you data:
+//
+//  db, _ := l.Select(0)
+//
+// DB must be selected by a index, ledis supports only 16 databases, so the index range is [0-15].
+//
+// KV
+//
+// KV is the most basic ledis type like any other key-value database.
+//
+//  err := db.Set(key, value)
+//  value, err := db.Get(key)
+//
+// List
+//
+// List is simply lists of values, sorted by insertion order.
+// You can push or pop value on the list head (left) or tail (right).
+//
+//  err := db.LPush(key, value1)
+//  err := db.RPush(key, value2)
+//  value1, err := db.LPop(key)
+//  value2, err := db.RPop(key)
+//
+// Hash
+//
+// Hash is a map between fields and values.
+//
+//  n, err := db.HSet(key, field1, value1)
+//  n, err := db.HSet(key, field2, value2)
+//  value1, err := db.HGet(key, field1)
+//  value2, err := db.HGet(key, field2)
+//
+// ZSet
+//
+// ZSet is a sorted collections of values.
+// Every member of zset is associated with score, a int64 value which used to sort, from smallest to greatest score.
+// Members are unique, but score may be same.
+//
+//  n, err := db.ZAdd(key, ScorePair{score1, member1}, ScorePair{score2, member2})
+//  ay, err := db.ZRangeByScore(key, minScore, maxScore, 0, -1)
+//
+//
+package ledis

vendor/github.com/siddontang/ledisdb/ledis/dump.go

@@ -0,0 +1,220 @@
+package ledis
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"io"
+	"os"
+
+	"github.com/siddontang/go/snappy"
+	"github.com/siddontang/ledisdb/store"
+)
+
+// DumpHead is the head of a dump.
+type DumpHead struct {
+	CommitID uint64
+}
+
+// Read reads meta from the Reader.
+func (h *DumpHead) Read(r io.Reader) error {
+	return binary.Read(r, binary.BigEndian, &h.CommitID)
+}
+
+// Write writes meta to the Writer
+func (h *DumpHead) Write(w io.Writer) error {
+	return binary.Write(w, binary.BigEndian, h.CommitID)
+}
+
+// DumpFile dumps data to the file
+func (l *Ledis) DumpFile(path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return l.Dump(f)
+}
+
+// Dump dumps data to the Writer.
+func (l *Ledis) Dump(w io.Writer) error {
+	var err error
+
+	var commitID uint64
+	var snap *store.Snapshot
+
+	l.wLock.Lock()
+
+	if l.r != nil {
+		if commitID, err = l.r.LastCommitID(); err != nil {
+			l.wLock.Unlock()
+			return err
+		}
+	}
+
+	if snap, err = l.ldb.NewSnapshot(); err != nil {
+		l.wLock.Unlock()
+		return err
+	}
+	defer snap.Close()
+
+	l.wLock.Unlock()
+
+	wb := bufio.NewWriterSize(w, 4096)
+
+	h := &DumpHead{commitID}
+
+	if err = h.Write(wb); err != nil {
+		return err
+	}
+
+	it := snap.NewIterator()
+	defer it.Close()
+	it.SeekToFirst()
+
+	compressBuf := make([]byte, 4096)
+
+	var key []byte
+	var value []byte
+	for ; it.Valid(); it.Next() {
+		key = it.RawKey()
+		value = it.RawValue()
+
+		if key, err = snappy.Encode(compressBuf, key); err != nil {
+			return err
+		}
+
+		if err = binary.Write(wb, binary.BigEndian, uint16(len(key))); err != nil {
+			return err
+		}
+
+		if _, err = wb.Write(key); err != nil {
+			return err
+		}
+
+		if value, err = snappy.Encode(compressBuf, value); err != nil {
+			return err
+		}
+
+		if err = binary.Write(wb, binary.BigEndian, uint32(len(value))); err != nil {
+			return err
+		}
+
+		if _, err = wb.Write(value); err != nil {
+			return err
+		}
+	}
+
+	if err = wb.Flush(); err != nil {
+		return err
+	}
+
+	compressBuf = nil
+
+	return nil
+}
+
+// LoadDumpFile clears all data and loads dump file to db
+func (l *Ledis) LoadDumpFile(path string) (*DumpHead, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	return l.LoadDump(f)
+}
+
+// LoadDump clears all data and loads dump file to db
+func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) {
+	l.wLock.Lock()
+	defer l.wLock.Unlock()
+
+	var err error
+	if err = l.flushAll(); err != nil {
+		return nil, err
+	}
+
+	rb := bufio.NewReaderSize(r, 4096)
+
+	h := new(DumpHead)
+
+	if err = h.Read(rb); err != nil {
+		return nil, err
+	}
+
+	var keyLen uint16
+	var valueLen uint32
+
+	var keyBuf bytes.Buffer
+	var valueBuf bytes.Buffer
+
+	deKeyBuf := make([]byte, 4096)
+	deValueBuf := make([]byte, 4096)
+
+	var key, value []byte
+
+	wb := l.ldb.NewWriteBatch()
+	defer wb.Close()
+
+	n := 0
+
+	for {
+		if err = binary.Read(rb, binary.BigEndian, &keyLen); err != nil && err != io.EOF {
+			return nil, err
+		} else if err == io.EOF {
+			break
+		}
+
+		if _, err = io.CopyN(&keyBuf, rb, int64(keyLen)); err != nil {
+			return nil, err
+		}
+
+		if key, err = snappy.Decode(deKeyBuf, keyBuf.Bytes()); err != nil {
+			return nil, err
+		}
+
+		if err = binary.Read(rb, binary.BigEndian, &valueLen); err != nil {
+			return nil, err
+		}
+
+		if _, err = io.CopyN(&valueBuf, rb, int64(valueLen)); err != nil {
+			return nil, err
+		}
+
+		if value, err = snappy.Decode(deValueBuf, valueBuf.Bytes()); err != nil {
+			return nil, err
+		}
+
+		wb.Put(key, value)
+		n++
+		if n%1024 == 0 {
+			if err = wb.Commit(); err != nil {
+				return nil, err
+			}
+		}
+
+		// if err = l.ldb.Put(key, value); err != nil {
+		// 	return nil, err
+		// }
+
+		keyBuf.Reset()
+		valueBuf.Reset()
+	}
+
+	if err = wb.Commit(); err != nil {
+		return nil, err
+	}
+
+	deKeyBuf = nil
+	deValueBuf = nil
+
+	if l.r != nil {
+		if err := l.r.UpdateCommitID(h.CommitID); err != nil {
+			return nil, err
+		}
+	}
+
+	return h, nil
+}

vendor/github.com/siddontang/ledisdb/ledis/event.go

@@ -0,0 +1,136 @@
+package ledis
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+
+	"github.com/siddontang/go/hack"
+)
+
+var errInvalidEvent = errors.New("invalid event")
+
+func formatEventKey(buf []byte, k []byte) ([]byte, error) {
+	if len(k) < 2 {
+		return nil, errInvalidEvent
+	}
+
+	buf = append(buf, fmt.Sprintf("DB:%2d ", k[0])...)
+	buf = append(buf, fmt.Sprintf("%s ", TypeName[k[1]])...)
+
+	db := new(DB)
+	index, _, err := decodeDBIndex(k)
+	if err != nil {
+		return nil, err
+	}
+	db.setIndex(index)
+
+	//to do format at respective place
+
+	switch k[1] {
+	case KVType:
+		key, err := db.decodeKVKey(k)
+		if err != nil {
+			return nil, err
+		}
+		buf = strconv.AppendQuote(buf, hack.String(key))
+	case HashType:
+		key, field, err := db.hDecodeHashKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+		buf = append(buf, ' ')
+		buf = strconv.AppendQuote(buf, hack.String(field))
+	case HSizeType:
+		key, err := db.hDecodeSizeKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+	case ListType:
+		key, seq, err := db.lDecodeListKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+		buf = append(buf, ' ')
+		buf = strconv.AppendInt(buf, int64(seq), 10)
+	case LMetaType:
+		key, err := db.lDecodeMetaKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+	case ZSetType:
+		key, m, err := db.zDecodeSetKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+		buf = append(buf, ' ')
+		buf = strconv.AppendQuote(buf, hack.String(m))
+	case ZSizeType:
+		key, err := db.zDecodeSizeKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+	case ZScoreType:
+		key, m, score, err := db.zDecodeScoreKey(k)
+		if err != nil {
+			return nil, err
+		}
+		buf = strconv.AppendQuote(buf, hack.String(key))
+		buf = append(buf, ' ')
+		buf = strconv.AppendQuote(buf, hack.String(m))
+		buf = append(buf, ' ')
+		buf = strconv.AppendInt(buf, score, 10)
+	case SetType:
+		key, member, err := db.sDecodeSetKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+		buf = append(buf, ' ')
+		buf = strconv.AppendQuote(buf, hack.String(member))
+	case SSizeType:
+		key, err := db.sDecodeSizeKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = strconv.AppendQuote(buf, hack.String(key))
+	case ExpTimeType:
+		tp, key, t, err := db.expDecodeTimeKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = append(buf, TypeName[tp]...)
+		buf = append(buf, ' ')
+		buf = strconv.AppendQuote(buf, hack.String(key))
+		buf = append(buf, ' ')
+		buf = strconv.AppendInt(buf, t, 10)
+	case ExpMetaType:
+		tp, key, err := db.expDecodeMetaKey(k)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = append(buf, TypeName[tp]...)
+		buf = append(buf, ' ')
+		buf = strconv.AppendQuote(buf, hack.String(key))
+	default:
+		return nil, errInvalidEvent
+	}
+
+	return buf, nil
+}

vendor/github.com/siddontang/ledisdb/ledis/ledis.go

@@ -0,0 +1,248 @@
+package ledis
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/siddontang/go/filelock"
+	"github.com/siddontang/go/log"
+	"github.com/siddontang/ledisdb/config"
+	"github.com/siddontang/ledisdb/rpl"
+	"github.com/siddontang/ledisdb/store"
+)
+
+// Ledis is the core structure to handle the database.
+type Ledis struct {
+	cfg *config.Config
+
+	ldb *store.DB
+
+	dbLock sync.Mutex
+	dbs    map[int]*DB
+
+	quit chan struct{}
+	wg   sync.WaitGroup
+
+	//for replication
+	r       *rpl.Replication
+	rc      chan struct{}
+	rbatch  *store.WriteBatch
+	rDoneCh chan struct{}
+	rhs     []NewLogEventHandler
+
+	wLock      sync.RWMutex //allow one write at same time
+	commitLock sync.Mutex   //allow one write commit at same time
+
+	lock io.Closer
+
+	ttlCheckers  []*ttlChecker
+	ttlCheckerCh chan *ttlChecker
+}
+
+// Open opens the Ledis with a config.
+func Open(cfg *config.Config) (*Ledis, error) {
+	if len(cfg.DataDir) == 0 {
+		cfg.DataDir = config.DefaultDataDir
+	}
+
+	if cfg.Databases == 0 {
+		cfg.Databases = 16
+	} else if cfg.Databases > MaxDatabases {
+		cfg.Databases = MaxDatabases
+	}
+
+	os.MkdirAll(cfg.DataDir, 0755)
+
+	var err error
+
+	l := new(Ledis)
+	l.cfg = cfg
+
+	if l.lock, err = filelock.Lock(path.Join(cfg.DataDir, "LOCK")); err != nil {
+		return nil, err
+	}
+
+	l.quit = make(chan struct{})
+
+	if l.ldb, err = store.Open(cfg); err != nil {
+		return nil, err
+	}
+
+	if cfg.UseReplication {
+		if l.r, err = rpl.NewReplication(cfg); err != nil {
+			return nil, err
+		}
+
+		l.rc = make(chan struct{}, 1)
+		l.rbatch = l.ldb.NewWriteBatch()
+		l.rDoneCh = make(chan struct{}, 1)
+
+		l.wg.Add(1)
+		go l.onReplication()
+
+		//first we must try wait all replication ok
+		//maybe some logs are not committed
+		l.WaitReplication()
+	} else {
+		l.r = nil
+	}
+
+	l.dbs = make(map[int]*DB, 16)
+
+	l.checkTTL()
+
+	return l, nil
+}
+
+// Close closes the Ledis.
+func (l *Ledis) Close() {
+	close(l.quit)
+	l.wg.Wait()
+
+	l.ldb.Close()
+
+	if l.r != nil {
+		l.r.Close()
+		//l.r = nil
+	}
+
+	if l.lock != nil {
+		l.lock.Close()
+		//l.lock = nil
+	}
+}
+
+// Select chooses a database.
+func (l *Ledis) Select(index int) (*DB, error) {
+	if index < 0 || index >= l.cfg.Databases {
+		return nil, fmt.Errorf("invalid db index %d, must in [0, %d]", index, l.cfg.Databases-1)
+	}
+
+	l.dbLock.Lock()
+	defer l.dbLock.Unlock()
+
+	db, ok := l.dbs[index]
+	if ok {
+		return db, nil
+	}
+
+	db = l.newDB(index)
+	l.dbs[index] = db
+
+	go func(db *DB) {
+		l.ttlCheckerCh <- db.ttlChecker
+	}(db)
+
+	return db, nil
+}
+
+// FlushAll will clear all data and replication logs
+func (l *Ledis) FlushAll() error {
+	l.wLock.Lock()
+	defer l.wLock.Unlock()
+
+	return l.flushAll()
+}
+
+func (l *Ledis) flushAll() error {
+	it := l.ldb.NewIterator()
+	defer it.Close()
+
+	it.SeekToFirst()
+
+	w := l.ldb.NewWriteBatch()
+	defer w.Rollback()
+
+	n := 0
+	for ; it.Valid(); it.Next() {
+		n++
+		if n == 10000 {
+			if err := w.Commit(); err != nil {
+				log.Fatalf("flush all commit error: %s", err.Error())
+				return err
+			}
+			n = 0
+		}
+		w.Delete(it.RawKey())
+	}
+
+	if err := w.Commit(); err != nil {
+		log.Fatalf("flush all commit error: %s", err.Error())
+		return err
+	}
+
+	if l.r != nil {
+		if err := l.r.Clear(); err != nil {
+			log.Fatalf("flush all replication clear error: %s", err.Error())
+			return err
+		}
+	}
+
+	return nil
+}
+
+// IsReadOnly returns whether Ledis is read only or not.
+func (l *Ledis) IsReadOnly() bool {
+	if l.cfg.GetReadonly() {
+		return true
+	} else if l.r != nil {
+		if b, _ := l.r.CommitIDBehind(); b {
+			return true
+		}
+	}
+	return false
+}
+
+func (l *Ledis) checkTTL() {
+	l.ttlCheckers = make([]*ttlChecker, 0, 16)
+	l.ttlCheckerCh = make(chan *ttlChecker, 16)
+
+	if l.cfg.TTLCheckInterval == 0 {
+		l.cfg.TTLCheckInterval = 1
+	}
+
+	l.wg.Add(1)
+	go func() {
+		defer l.wg.Done()
+
+		tick := time.NewTicker(time.Duration(l.cfg.TTLCheckInterval) * time.Second)
+		defer tick.Stop()
+
+		for {
+			select {
+			case <-tick.C:
+				if l.IsReadOnly() {
+					break
+				}
+
+				for _, c := range l.ttlCheckers {
+					c.check()
+				}
+			case c := <-l.ttlCheckerCh:
+				l.ttlCheckers = append(l.ttlCheckers, c)
+				c.check()
+			case <-l.quit:
+				return
+			}
+		}
+
+	}()
+
+}
+
+// StoreStat returns the statistics.
+func (l *Ledis) StoreStat() *store.Stat {
+	return l.ldb.Stat()
+}
+
+// CompactStore compacts the backend storage.
+func (l *Ledis) CompactStore() error {
+	l.wLock.Lock()
+	defer l.wLock.Unlock()
+
+	return l.ldb.Compact()
+}

vendor/github.com/siddontang/ledisdb/ledis/ledis_db.go

@@ -0,0 +1,208 @@
+package ledis
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"sync"
+
+	"github.com/siddontang/ledisdb/store"
+)
+
+type ibucket interface {
+	Get(key []byte) ([]byte, error)
+	GetSlice(key []byte) (store.Slice, error)
+
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+
+	NewIterator() *store.Iterator
+
+	NewWriteBatch() *store.WriteBatch
+
+	RangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator
+	RevRangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator
+	RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator
+	RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator
+}
+
+// DB is the database.
+type DB struct {
+	l *Ledis
+
+	sdb *store.DB
+
+	bucket ibucket
+
+	index int
+
+	// buffer to store index varint
+	indexVarBuf []byte
+
+	kvBatch   *batch
+	listBatch *batch
+	hashBatch *batch
+	zsetBatch *batch
+	//	binBatch  *batch
+	setBatch *batch
+
+	// status uint8
+
+	ttlChecker *ttlChecker
+
+	lbkeys *lBlockKeys
+}
+
+func (l *Ledis) newDB(index int) *DB {
+	d := new(DB)
+
+	d.l = l
+
+	d.sdb = l.ldb
+
+	d.bucket = d.sdb
+
+	//	d.status = DBAutoCommit
+	d.setIndex(index)
+
+	d.kvBatch = d.newBatch()
+	d.listBatch = d.newBatch()
+	d.hashBatch = d.newBatch()
+	d.zsetBatch = d.newBatch()
+	// d.binBatch = d.newBatch()
+	d.setBatch = d.newBatch()
+
+	d.lbkeys = newLBlockKeys()
+
+	d.ttlChecker = d.newTTLChecker()
+
+	return d
+}
+
+func decodeDBIndex(buf []byte) (int, int, error) {
+	index, n := binary.Uvarint(buf)
+	if n == 0 {
+		return 0, 0, fmt.Errorf("buf is too small to save index")
+	} else if n < 0 {
+		return 0, 0, fmt.Errorf("value larger than 64 bits")
+	} else if index > uint64(MaxDatabases) {
+		return 0, 0, fmt.Errorf("value %d is larger than max databases %d", index, MaxDatabases)
+	}
+	return int(index), n, nil
+}
+
+func (db *DB) setIndex(index int) {
+	db.index = index
+	// the most size for varint is 10 bytes
+	buf := make([]byte, 10)
+	n := binary.PutUvarint(buf, uint64(index))
+
+	db.indexVarBuf = buf[0:n]
+}
+
+func (db *DB) checkKeyIndex(buf []byte) (int, error) {
+	if len(buf) < len(db.indexVarBuf) {
+		return 0, fmt.Errorf("key is too small")
+	} else if !bytes.Equal(db.indexVarBuf, buf[0:len(db.indexVarBuf)]) {
+		return 0, fmt.Errorf("invalid db index")
+	}
+
+	return len(db.indexVarBuf), nil
+}
+
+func (db *DB) newTTLChecker() *ttlChecker {
+	c := new(ttlChecker)
+	c.db = db
+	c.txs = make([]*batch, maxDataType)
+	c.cbs = make([]onExpired, maxDataType)
+	c.nc = 0
+
+	c.register(KVType, db.kvBatch, db.delete)
+	c.register(ListType, db.listBatch, db.lDelete)
+	c.register(HashType, db.hashBatch, db.hDelete)
+	c.register(ZSetType, db.zsetBatch, db.zDelete)
+	//		c.register(BitType, db.binBatch, db.bDelete)
+	c.register(SetType, db.setBatch, db.sDelete)
+
+	return c
+}
+
+func (db *DB) newBatch() *batch {
+	return db.l.newBatch(db.bucket.NewWriteBatch(), &dbBatchLocker{l: &sync.Mutex{}, wrLock: &db.l.wLock})
+}
+
+// Index gets the index of database.
+func (db *DB) Index() int {
+	return int(db.index)
+}
+
+// func (db *DB) IsAutoCommit() bool {
+// 	return db.status == DBAutoCommit
+// }
+
+// FlushAll flushes the data.
+func (db *DB) FlushAll() (drop int64, err error) {
+	all := [...](func() (int64, error)){
+		db.flush,
+		db.lFlush,
+		db.hFlush,
+		db.zFlush,
+		db.sFlush}
+
+	for _, flush := range all {
+		n, e := flush()
+		if e != nil {
+			err = e
+			return
+		}
+
+		drop += n
+	}
+
+	return
+}
+
+func (db *DB) flushType(t *batch, dataType byte) (drop int64, err error) {
+	var deleteFunc func(t *batch, key []byte) int64
+	var metaDataType byte
+	switch dataType {
+	case KVType:
+		deleteFunc = db.delete
+		metaDataType = KVType
+	case ListType:
+		deleteFunc = db.lDelete
+		metaDataType = LMetaType
+	case HashType:
+		deleteFunc = db.hDelete
+		metaDataType = HSizeType
+	case ZSetType:
+		deleteFunc = db.zDelete
+		metaDataType = ZSizeType
+	// case BitType:
+	// 	deleteFunc = db.bDelete
+	// 	metaDataType = BitMetaType
+	case SetType:
+		deleteFunc = db.sDelete
+		metaDataType = SSizeType
+	default:
+		return 0, fmt.Errorf("invalid data type: %s", TypeName[dataType])
+	}
+
+	var keys [][]byte
+	keys, err = db.scanGeneric(metaDataType, nil, 1024, false, "", false)
+	for len(keys) != 0 || err != nil {
+		for _, key := range keys {
+			deleteFunc(t, key)
+			db.rmExpire(t, dataType, key)
+
+		}
+
+		if err = t.Commit(); err != nil {
+			return
+		}
+
+		drop += int64(len(keys))
+		keys, err = db.scanGeneric(metaDataType, nil, 1024, false, "", false)
+	}
+	return
+}

vendor/github.com/siddontang/ledisdb/ledis/migrate.go

@@ -0,0 +1,195 @@
+package ledis
+
+import (
+	"fmt"
+
+	"github.com/siddontang/rdb"
+)
+
+/*
+   To support redis <-> ledisdb, the dump value format is the same as redis.
+   We will not support bitmap, and may add bit operations for kv later.
+
+   But you must know that we use int64 for zset score, not double.
+   Only support rdb version 6.
+*/
+
+// Dump dumps the KV value of key
+func (db *DB) Dump(key []byte) ([]byte, error) {
+	v, err := db.Get(key)
+	if err != nil {
+		return nil, err
+	} else if v == nil {
+		return nil, err
+	}
+
+	return rdb.Dump(rdb.String(v))
+}
+
+// LDump dumps the list value of key
+func (db *DB) LDump(key []byte) ([]byte, error) {
+	v, err := db.LRange(key, 0, -1)
+	if err != nil {
+		return nil, err
+	} else if len(v) == 0 {
+		return nil, err
+	}
+
+	return rdb.Dump(rdb.List(v))
+}
+
+// HDump dumps the hash value of key
+func (db *DB) HDump(key []byte) ([]byte, error) {
+	v, err := db.HGetAll(key)
+	if err != nil {
+		return nil, err
+	} else if len(v) == 0 {
+		return nil, err
+	}
+
+	o := make(rdb.Hash, len(v))
+	for i := 0; i < len(v); i++ {
+		o[i].Field = v[i].Field
+		o[i].Value = v[i].Value
+	}
+
+	return rdb.Dump(o)
+}
+
+// SDump dumps the set value of key
+func (db *DB) SDump(key []byte) ([]byte, error) {
+	v, err := db.SMembers(key)
+	if err != nil {
+		return nil, err
+	} else if len(v) == 0 {
+		return nil, err
+	}
+
+	return rdb.Dump(rdb.Set(v))
+}
+
+// ZDump dumps the zset value of key
+func (db *DB) ZDump(key []byte) ([]byte, error) {
+	v, err := db.ZRangeByScore(key, MinScore, MaxScore, 0, -1)
+	if err != nil {
+		return nil, err
+	} else if len(v) == 0 {
+		return nil, err
+	}
+
+	o := make(rdb.ZSet, len(v))
+	for i := 0; i < len(v); i++ {
+		o[i].Member = v[i].Member
+		o[i].Score = float64(v[i].Score)
+	}
+
+	return rdb.Dump(o)
+}
+
+// Restore restores a key into database.
+func (db *DB) Restore(key []byte, ttl int64, data []byte) error {
+	d, err := rdb.DecodeDump(data)
+	if err != nil {
+		return err
+	}
+
+	//ttl is milliseconds, but we only support seconds
+	//later may support milliseconds
+	if ttl > 0 {
+		ttl = ttl / 1e3
+		if ttl == 0 {
+			ttl = 1
+		}
+	}
+
+	switch value := d.(type) {
+	case rdb.String:
+		if _, err = db.Del(key); err != nil {
+			return err
+		}
+
+		if err = db.Set(key, value); err != nil {
+			return err
+		}
+
+		if ttl > 0 {
+			if _, err = db.Expire(key, ttl); err != nil {
+				return err
+			}
+		}
+	case rdb.Hash:
+		//first clear old key
+		if _, err = db.HClear(key); err != nil {
+			return err
+		}
+
+		fv := make([]FVPair, len(value))
+		for i := 0; i < len(value); i++ {
+			fv[i] = FVPair{Field: value[i].Field, Value: value[i].Value}
+		}
+
+		if err = db.HMset(key, fv...); err != nil {
+			return err
+		}
+
+		if ttl > 0 {
+			if _, err = db.HExpire(key, ttl); err != nil {
+				return err
+			}
+		}
+	case rdb.List:
+		//first clear old key
+		if _, err = db.LClear(key); err != nil {
+			return err
+		}
+
+		if _, err = db.RPush(key, value...); err != nil {
+			return err
+		}
+
+		if ttl > 0 {
+			if _, err = db.LExpire(key, ttl); err != nil {
+				return err
+			}
+		}
+	case rdb.ZSet:
+		//first clear old key
+		if _, err = db.ZClear(key); err != nil {
+			return err
+		}
+
+		sp := make([]ScorePair, len(value))
+		for i := 0; i < len(value); i++ {
+			sp[i] = ScorePair{int64(value[i].Score), value[i].Member}
+		}
+
+		if _, err = db.ZAdd(key, sp...); err != nil {
+			return err
+		}
+
+		if ttl > 0 {
+			if _, err = db.ZExpire(key, ttl); err != nil {
+				return err
+			}
+		}
+	case rdb.Set:
+		//first clear old key
+		if _, err = db.SClear(key); err != nil {
+			return err
+		}
+
+		if _, err = db.SAdd(key, value...); err != nil {
+			return err
+		}
+
+		if ttl > 0 {
+			if _, err = db.SExpire(key, ttl); err != nil {
+				return err
+			}
+		}
+	default:
+		return fmt.Errorf("invalid data type %T", d)
+	}
+
+	return nil
+}

vendor/github.com/siddontang/ledisdb/ledis/replication.go

@@ -0,0 +1,259 @@
+package ledis
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"time"
+
+	"github.com/siddontang/go/log"
+	"github.com/siddontang/go/snappy"
+	"github.com/siddontang/ledisdb/rpl"
+	"github.com/siddontang/ledisdb/store"
+)
+
+const (
+	maxReplLogSize = 1 * 1024 * 1024
+)
+
+// For replication error.
+var (
+	ErrLogMissed = errors.New("log is pured in server")
+)
+
+// ReplicationUsed returns whether replication is used or not.
+func (l *Ledis) ReplicationUsed() bool {
+	return l.r != nil
+}
+
+func (l *Ledis) handleReplication() error {
+	l.wLock.Lock()
+	defer l.wLock.Unlock()
+
+	defer AsyncNotify(l.rDoneCh)
+
+	rl := &rpl.Log{}
+
+	var err error
+	for {
+		if err = l.r.NextNeedCommitLog(rl); err != nil {
+			if err != rpl.ErrNoBehindLog {
+				log.Errorf("get next commit log err, %s", err.Error())
+				return err
+			}
+
+			return nil
+		}
+
+		l.rbatch.Rollback()
+
+		if rl.Compression == 1 {
+			//todo optimize
+			if rl.Data, err = snappy.Decode(nil, rl.Data); err != nil {
+				log.Errorf("decode log error %s", err.Error())
+				return err
+			}
+		}
+
+		if bd, err := store.NewBatchData(rl.Data); err != nil {
+			log.Errorf("decode batch log error %s", err.Error())
+			return err
+		} else if err = bd.Replay(l.rbatch); err != nil {
+			log.Errorf("replay batch log error %s", err.Error())
+		}
+
+		l.commitLock.Lock()
+		if err = l.rbatch.Commit(); err != nil {
+			log.Errorf("commit log error %s", err.Error())
+		} else if err = l.r.UpdateCommitID(rl.ID); err != nil {
+			log.Errorf("update commit id error %s", err.Error())
+		}
+
+		l.commitLock.Unlock()
+		if err != nil {
+			return err
+		}
+	}
+}
+
+func (l *Ledis) onReplication() {
+	defer l.wg.Done()
+
+	l.noticeReplication()
+
+	for {
+		select {
+		case <-l.rc:
+			l.handleReplication()
+		case <-l.quit:
+			return
+		}
+	}
+}
+
+// WaitReplication waits replication done
+func (l *Ledis) WaitReplication() error {
+	if !l.ReplicationUsed() {
+		return ErrRplNotSupport
+
+	}
+
+	for i := 0; i < 100; i++ {
+		l.noticeReplication()
+
+		select {
+		case <-l.rDoneCh:
+		case <-l.quit:
+			return nil
+		}
+		time.Sleep(100 * time.Millisecond)
+
+		b, err := l.r.CommitIDBehind()
+		if err != nil {
+			return err
+		} else if !b {
+			return nil
+		}
+	}
+
+	return errors.New("wait replication too many times")
+}
+
+// StoreLogsFromReader stores logs from the Reader
+func (l *Ledis) StoreLogsFromReader(rb io.Reader) error {
+	if !l.ReplicationUsed() {
+		return ErrRplNotSupport
+	} else if !l.cfg.Readonly {
+		return ErrRplInRDWR
+	}
+
+	log := &rpl.Log{}
+
+	for {
+		if err := log.Decode(rb); err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return err
+			}
+		}
+
+		if err := l.r.StoreLog(log); err != nil {
+			return err
+		}
+
+	}
+
+	l.noticeReplication()
+
+	return nil
+}
+
+func (l *Ledis) noticeReplication() {
+	AsyncNotify(l.rc)
+}
+
+// StoreLogsFromData stores logs from data.
+func (l *Ledis) StoreLogsFromData(data []byte) error {
+	rb := bytes.NewReader(data)
+
+	return l.StoreLogsFromReader(rb)
+}
+
+// ReadLogsTo reads logs and write to the Writer.
+func (l *Ledis) ReadLogsTo(startLogID uint64, w io.Writer) (n int, nextLogID uint64, err error) {
+	if !l.ReplicationUsed() {
+		// no replication log
+		nextLogID = 0
+		err = ErrRplNotSupport
+		return
+	}
+
+	var firtID, lastID uint64
+
+	firtID, err = l.r.FirstLogID()
+	if err != nil {
+		return
+	}
+
+	if startLogID < firtID {
+		err = ErrLogMissed
+		return
+	}
+
+	lastID, err = l.r.LastLogID()
+	if err != nil {
+		return
+	}
+
+	nextLogID = startLogID
+
+	log := &rpl.Log{}
+	for i := startLogID; i <= lastID; i++ {
+		if err = l.r.GetLog(i, log); err != nil {
+			return
+		}
+
+		if err = log.Encode(w); err != nil {
+			return
+		}
+
+		nextLogID = i + 1
+
+		n += log.Size()
+
+		if n > maxReplLogSize {
+			break
+		}
+	}
+
+	return
+}
+
+// ReadLogsToTimeout tries to read events, if no events read,
+// tres to wait the new event singal until timeout seconds
+func (l *Ledis) ReadLogsToTimeout(startLogID uint64, w io.Writer, timeout int, quitCh chan struct{}) (n int, nextLogID uint64, err error) {
+	n, nextLogID, err = l.ReadLogsTo(startLogID, w)
+	if err != nil {
+		return
+	} else if n != 0 {
+		return
+	}
+	//no events read
+	select {
+	case <-l.r.WaitLog():
+	case <-time.After(time.Duration(timeout) * time.Second):
+	case <-quitCh:
+		return
+	}
+	return l.ReadLogsTo(startLogID, w)
+}
+
+func (l *Ledis) propagate(rl *rpl.Log) {
+	for _, h := range l.rhs {
+		h(rl)
+	}
+}
+
+// NewLogEventHandler is the handler to handle new log event.
+type NewLogEventHandler func(rl *rpl.Log)
+
+// AddNewLogEventHandler adds the handler for the new log event
+func (l *Ledis) AddNewLogEventHandler(h NewLogEventHandler) error {
+	if !l.ReplicationUsed() {
+		return ErrRplNotSupport
+	}
+
+	l.rhs = append(l.rhs, h)
+
+	return nil
+}
+
+// ReplicationStat returns the statistics of repliaciton.
+func (l *Ledis) ReplicationStat() (*rpl.Stat, error) {
+	if !l.ReplicationUsed() {
+		return nil, ErrRplNotSupport
+	}
+
+	return l.r.Stat()
+}

vendor/github.com/siddontang/ledisdb/ledis/scan.go

@@ -0,0 +1,402 @@
+package ledis
+
+import (
+	"errors"
+	"regexp"
+
+	"github.com/siddontang/ledisdb/store"
+)
+
+var errDataType = errors.New("error data type")
+var errMetaKey = errors.New("error meta key")
+
+//Scan scans the data. If inclusive is true, scan range [cursor, inf) else (cursor, inf)
+func (db *DB) Scan(dataType DataType, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	storeDataType, err := getDataStoreType(dataType)
+	if err != nil {
+		return nil, err
+	}
+
+	return db.scanGeneric(storeDataType, cursor, count, inclusive, match, false)
+}
+
+// RevScan scans the data reversed. if inclusive is true, revscan range (-inf, cursor] else (inf, cursor)
+func (db *DB) RevScan(dataType DataType, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	storeDataType, err := getDataStoreType(dataType)
+	if err != nil {
+		return nil, err
+	}
+
+	return db.scanGeneric(storeDataType, cursor, count, inclusive, match, true)
+}
+
+func getDataStoreType(dataType DataType) (byte, error) {
+	var storeDataType byte
+	switch dataType {
+	case KV:
+		storeDataType = KVType
+	case LIST:
+		storeDataType = LMetaType
+	case HASH:
+		storeDataType = HSizeType
+	case SET:
+		storeDataType = SSizeType
+	case ZSET:
+		storeDataType = ZSizeType
+	default:
+		return 0, errDataType
+	}
+	return storeDataType, nil
+}
+
+func buildMatchRegexp(match string) (*regexp.Regexp, error) {
+	var err error
+	var r *regexp.Regexp
+
+	if len(match) > 0 {
+		if r, err = regexp.Compile(match); err != nil {
+			return nil, err
+		}
+	}
+
+	return r, nil
+}
+
+func (db *DB) buildScanIterator(minKey []byte, maxKey []byte, inclusive bool, reverse bool) *store.RangeLimitIterator {
+	tp := store.RangeOpen
+
+	if !reverse {
+		if inclusive {
+			tp = store.RangeROpen
+		}
+	} else {
+		if inclusive {
+			tp = store.RangeLOpen
+		}
+	}
+
+	var it *store.RangeLimitIterator
+	if !reverse {
+		it = db.bucket.RangeIterator(minKey, maxKey, tp)
+	} else {
+		it = db.bucket.RevRangeIterator(minKey, maxKey, tp)
+	}
+
+	return it
+}
+
+func (db *DB) buildScanKeyRange(storeDataType byte, key []byte, reverse bool) (minKey []byte, maxKey []byte, err error) {
+	if !reverse {
+		if minKey, err = db.encodeScanMinKey(storeDataType, key); err != nil {
+			return
+		}
+		if maxKey, err = db.encodeScanMaxKey(storeDataType, nil); err != nil {
+			return
+		}
+	} else {
+		if minKey, err = db.encodeScanMinKey(storeDataType, nil); err != nil {
+			return
+		}
+		if maxKey, err = db.encodeScanMaxKey(storeDataType, key); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func checkScanCount(count int) int {
+	if count <= 0 {
+		count = defaultScanCount
+	}
+
+	return count
+}
+
+func (db *DB) scanGeneric(storeDataType byte, key []byte, count int,
+	inclusive bool, match string, reverse bool) ([][]byte, error) {
+
+	r, err := buildMatchRegexp(match)
+	if err != nil {
+		return nil, err
+	}
+
+	minKey, maxKey, err := db.buildScanKeyRange(storeDataType, key, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	count = checkScanCount(count)
+
+	it := db.buildScanIterator(minKey, maxKey, inclusive, reverse)
+
+	v := make([][]byte, 0, count)
+
+	for i := 0; it.Valid() && i < count; it.Next() {
+		if k, err := db.decodeScanKey(storeDataType, it.Key()); err != nil {
+			continue
+		} else if r != nil && !r.Match(k) {
+			continue
+		} else {
+			v = append(v, k)
+			i++
+		}
+	}
+	it.Close()
+	return v, nil
+}
+
+func (db *DB) encodeScanMinKey(storeDataType byte, key []byte) ([]byte, error) {
+	return db.encodeScanKey(storeDataType, key)
+}
+
+func (db *DB) encodeScanMaxKey(storeDataType byte, key []byte) ([]byte, error) {
+	if len(key) > 0 {
+		return db.encodeScanKey(storeDataType, key)
+	}
+
+	k, err := db.encodeScanKey(storeDataType, nil)
+	if err != nil {
+		return nil, err
+	}
+	k[len(k)-1] = storeDataType + 1
+	return k, nil
+}
+
+func (db *DB) encodeScanKey(storeDataType byte, key []byte) ([]byte, error) {
+	switch storeDataType {
+	case KVType:
+		return db.encodeKVKey(key), nil
+	case LMetaType:
+		return db.lEncodeMetaKey(key), nil
+	case HSizeType:
+		return db.hEncodeSizeKey(key), nil
+	case ZSizeType:
+		return db.zEncodeSizeKey(key), nil
+	case SSizeType:
+		return db.sEncodeSizeKey(key), nil
+	default:
+		return nil, errDataType
+	}
+}
+
+func (db *DB) decodeScanKey(storeDataType byte, ek []byte) (key []byte, err error) {
+	switch storeDataType {
+	case KVType:
+		key, err = db.decodeKVKey(ek)
+	case LMetaType:
+		key, err = db.lDecodeMetaKey(ek)
+	case HSizeType:
+		key, err = db.hDecodeSizeKey(ek)
+	case ZSizeType:
+		key, err = db.zDecodeSizeKey(ek)
+	case SSizeType:
+		key, err = db.sDecodeSizeKey(ek)
+	default:
+		err = errDataType
+	}
+	return
+}
+
+// for specail data scan
+
+func (db *DB) buildDataScanKeyRange(storeDataType byte, key []byte, cursor []byte, reverse bool) (minKey []byte, maxKey []byte, err error) {
+	if !reverse {
+		if minKey, err = db.encodeDataScanMinKey(storeDataType, key, cursor); err != nil {
+			return
+		}
+		if maxKey, err = db.encodeDataScanMaxKey(storeDataType, key, nil); err != nil {
+			return
+		}
+	} else {
+		if minKey, err = db.encodeDataScanMinKey(storeDataType, key, nil); err != nil {
+			return
+		}
+		if maxKey, err = db.encodeDataScanMaxKey(storeDataType, key, cursor); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (db *DB) encodeDataScanMinKey(storeDataType byte, key []byte, cursor []byte) ([]byte, error) {
+	return db.encodeDataScanKey(storeDataType, key, cursor)
+}
+
+func (db *DB) encodeDataScanMaxKey(storeDataType byte, key []byte, cursor []byte) ([]byte, error) {
+	if len(cursor) > 0 {
+		return db.encodeDataScanKey(storeDataType, key, cursor)
+	}
+
+	k, err := db.encodeDataScanKey(storeDataType, key, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// here, the last byte is the start seperator, set it to stop seperator
+	k[len(k)-1] = k[len(k)-1] + 1
+	return k, nil
+}
+
+func (db *DB) encodeDataScanKey(storeDataType byte, key []byte, cursor []byte) ([]byte, error) {
+	switch storeDataType {
+	case HashType:
+		return db.hEncodeHashKey(key, cursor), nil
+	case ZSetType:
+		return db.zEncodeSetKey(key, cursor), nil
+	case SetType:
+		return db.sEncodeSetKey(key, cursor), nil
+	default:
+		return nil, errDataType
+	}
+}
+
+func (db *DB) buildDataScanIterator(storeDataType byte, key []byte, cursor []byte, count int,
+	inclusive bool, reverse bool) (*store.RangeLimitIterator, error) {
+
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	minKey, maxKey, err := db.buildDataScanKeyRange(storeDataType, key, cursor, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	it := db.buildScanIterator(minKey, maxKey, inclusive, reverse)
+
+	return it, nil
+}
+
+func (db *DB) hScanGeneric(key []byte, cursor []byte, count int, inclusive bool, match string, reverse bool) ([]FVPair, error) {
+	count = checkScanCount(count)
+
+	r, err := buildMatchRegexp(match)
+	if err != nil {
+		return nil, err
+	}
+
+	v := make([]FVPair, 0, count)
+
+	it, err := db.buildDataScanIterator(HashType, key, cursor, count, inclusive, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	defer it.Close()
+
+	for i := 0; it.Valid() && i < count; it.Next() {
+		_, f, err := db.hDecodeHashKey(it.Key())
+		if err != nil {
+			return nil, err
+		} else if r != nil && !r.Match(f) {
+			continue
+		}
+
+		v = append(v, FVPair{Field: f, Value: it.Value()})
+
+		i++
+	}
+
+	return v, nil
+}
+
+// HScan scans data for hash.
+func (db *DB) HScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]FVPair, error) {
+	return db.hScanGeneric(key, cursor, count, inclusive, match, false)
+}
+
+// HRevScan reversed scans data for hash.
+func (db *DB) HRevScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]FVPair, error) {
+	return db.hScanGeneric(key, cursor, count, inclusive, match, true)
+}
+
+func (db *DB) sScanGeneric(key []byte, cursor []byte, count int, inclusive bool, match string, reverse bool) ([][]byte, error) {
+	count = checkScanCount(count)
+
+	r, err := buildMatchRegexp(match)
+	if err != nil {
+		return nil, err
+	}
+
+	v := make([][]byte, 0, count)
+
+	it, err := db.buildDataScanIterator(SetType, key, cursor, count, inclusive, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	defer it.Close()
+
+	for i := 0; it.Valid() && i < count; it.Next() {
+		_, m, err := db.sDecodeSetKey(it.Key())
+		if err != nil {
+			return nil, err
+		} else if r != nil && !r.Match(m) {
+			continue
+		}
+
+		v = append(v, m)
+
+		i++
+	}
+
+	return v, nil
+}
+
+// SScan scans data for set.
+func (db *DB) SScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.sScanGeneric(key, cursor, count, inclusive, match, false)
+}
+
+// SRevScan scans data reversed for set.
+func (db *DB) SRevScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.sScanGeneric(key, cursor, count, inclusive, match, true)
+}
+
+func (db *DB) zScanGeneric(key []byte, cursor []byte, count int, inclusive bool, match string, reverse bool) ([]ScorePair, error) {
+	count = checkScanCount(count)
+
+	r, err := buildMatchRegexp(match)
+	if err != nil {
+		return nil, err
+	}
+
+	v := make([]ScorePair, 0, count)
+
+	it, err := db.buildDataScanIterator(ZSetType, key, cursor, count, inclusive, reverse)
+	if err != nil {
+		return nil, err
+	}
+
+	defer it.Close()
+
+	for i := 0; it.Valid() && i < count; it.Next() {
+		_, m, err := db.zDecodeSetKey(it.Key())
+		if err != nil {
+			return nil, err
+		} else if r != nil && !r.Match(m) {
+			continue
+		}
+
+		score, err := Int64(it.Value(), nil)
+		if err != nil {
+			return nil, err
+		}
+
+		v = append(v, ScorePair{Score: score, Member: m})
+
+		i++
+	}
+
+	return v, nil
+}
+
+// ZScan scans data for zset.
+func (db *DB) ZScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]ScorePair, error) {
+	return db.zScanGeneric(key, cursor, count, inclusive, match, false)
+}
+
+// ZRevScan scans data reversed for zset.
+func (db *DB) ZRevScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]ScorePair, error) {
+	return db.zScanGeneric(key, cursor, count, inclusive, match, true)
+}

vendor/github.com/siddontang/ledisdb/ledis/sort.go

@@ -0,0 +1,235 @@
+package ledis
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+	"strconv"
+
+	"github.com/siddontang/ledisdb/store"
+)
+
+// Limit is for sort.
+type Limit struct {
+	Offset int
+	Size   int
+}
+
+func getSortRange(values [][]byte, offset int, size int) (int, int) {
+	var start = 0
+	if offset > 0 {
+		start = offset
+	}
+
+	valueLen := len(values)
+	var end = valueLen - 1
+	if size > 0 {
+		end = start + size - 1
+	}
+
+	if start >= valueLen {
+		start = valueLen - 1
+		end = valueLen - 2
+	}
+
+	if end >= valueLen {
+		end = valueLen - 1
+	}
+
+	return start, end
+}
+
+var hashPattern = []byte("*->")
+
+func (db *DB) lookupKeyByPattern(pattern []byte, subKey []byte) []byte {
+	// If the pattern is #, return the substitution key itself
+	if bytes.Equal(pattern, []byte{'#'}) {
+		return subKey
+	}
+
+	// If we can't find '*' in the pattern, return nil
+	if !bytes.Contains(pattern, []byte{'*'}) {
+		return nil
+	}
+
+	key := pattern
+	var field []byte
+
+	// Find out if we're dealing with a hash dereference
+	if n := bytes.Index(pattern, hashPattern); n > 0 && n+3 < len(pattern) {
+		key = pattern[0 : n+1]
+		field = pattern[n+3:]
+	}
+
+	// Perform the '*' substitution
+	key = bytes.Replace(key, []byte{'*'}, subKey, 1)
+
+	var value []byte
+	if field == nil {
+		value, _ = db.Get(key)
+	} else {
+		value, _ = db.HGet(key, field)
+	}
+
+	return value
+}
+
+type sortItem struct {
+	value    []byte
+	cmpValue []byte
+	score    float64
+}
+
+type sortItemSlice struct {
+	alpha         bool
+	sortByPattern bool
+	items         []sortItem
+}
+
+func (s *sortItemSlice) Len() int {
+	return len(s.items)
+}
+
+func (s *sortItemSlice) Swap(i, j int) {
+	s.items[i], s.items[j] = s.items[j], s.items[i]
+}
+
+func (s *sortItemSlice) Less(i, j int) bool {
+	s1 := s.items[i]
+	s2 := s.items[j]
+	if !s.alpha {
+		if s1.score < s2.score {
+			return true
+		} else if s1.score > s2.score {
+			return false
+		} else {
+			return bytes.Compare(s1.value, s2.value) < 0
+		}
+	} else {
+		if s.sortByPattern {
+			if s1.cmpValue == nil || s2.cmpValue == nil {
+				if s1.cmpValue == nil {
+					return true
+				}
+				return false
+			}
+			// Unlike redis, we only use bytes compare
+			return bytes.Compare(s1.cmpValue, s2.cmpValue) < 0
+		}
+
+		// Unlike redis, we only use bytes compare
+		return bytes.Compare(s1.value, s2.value) < 0
+	}
+}
+
+func (db *DB) xsort(values [][]byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) {
+	if len(values) == 0 {
+		return [][]byte{}, nil
+	}
+
+	start, end := getSortRange(values, offset, size)
+
+	dontsort := 0
+
+	if sortBy != nil {
+		if !bytes.Contains(sortBy, []byte{'*'}) {
+			dontsort = 1
+		}
+	}
+
+	items := &sortItemSlice{
+		alpha:         alpha,
+		sortByPattern: sortBy != nil,
+		items:         make([]sortItem, len(values)),
+	}
+
+	for i, value := range values {
+		items.items[i].value = value
+		items.items[i].score = 0
+		items.items[i].cmpValue = nil
+
+		if dontsort == 0 {
+			var cmpValue []byte
+			if sortBy != nil {
+				cmpValue = db.lookupKeyByPattern(sortBy, value)
+			} else {
+				// use value iteself to sort by
+				cmpValue = value
+			}
+
+			if cmpValue == nil {
+				continue
+			}
+
+			if alpha {
+				if sortBy != nil {
+					items.items[i].cmpValue = cmpValue
+				}
+			} else {
+				score, err := strconv.ParseFloat(string(cmpValue), 64)
+				if err != nil {
+					return nil, fmt.Errorf("%s scores can't be converted into double", cmpValue)
+				}
+				items.items[i].score = score
+			}
+		}
+	}
+
+	if dontsort == 0 {
+		if !desc {
+			sort.Sort(items)
+		} else {
+			sort.Sort(sort.Reverse(items))
+		}
+	}
+
+	resLen := end - start + 1
+	if len(sortGet) > 0 {
+		resLen = len(sortGet) * (end - start + 1)
+	}
+
+	res := make([][]byte, 0, resLen)
+	for i := start; i <= end; i++ {
+		if len(sortGet) == 0 {
+			res = append(res, items.items[i].value)
+		} else {
+			for _, getPattern := range sortGet {
+				v := db.lookupKeyByPattern(getPattern, items.items[i].value)
+				res = append(res, v)
+			}
+		}
+	}
+
+	return res, nil
+}
+
+// XLSort sorts list.
+func (db *DB) XLSort(key []byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) {
+	values, err := db.LRange(key, 0, -1)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return db.xsort(values, offset, size, alpha, desc, sortBy, sortGet)
+}
+
+// XSSort sorts set.
+func (db *DB) XSSort(key []byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) {
+	values, err := db.SMembers(key)
+	if err != nil {
+		return nil, err
+	}
+
+	return db.xsort(values, offset, size, alpha, desc, sortBy, sortGet)
+}
+
+// XZSort sorts zset.
+func (db *DB) XZSort(key []byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) {
+	values, err := db.ZRangeByLex(key, nil, nil, store.RangeClose, 0, -1)
+	if err != nil {
+		return nil, err
+	}
+
+	return db.xsort(values, offset, size, alpha, desc, sortBy, sortGet)
+}

vendor/github.com/siddontang/ledisdb/ledis/t_hash.go

@@ -0,0 +1,556 @@
+package ledis
+
+import (
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/siddontang/go/num"
+	"github.com/siddontang/ledisdb/store"
+)
+
+// FVPair is the pair of field and value.
+type FVPair struct {
+	Field []byte
+	Value []byte
+}
+
+var errHashKey = errors.New("invalid hash key")
+var errHSizeKey = errors.New("invalid hsize key")
+
+const (
+	hashStartSep byte = ':'
+	hashStopSep  byte = hashStartSep + 1
+)
+
+func checkHashKFSize(key []byte, field []byte) error {
+	if len(key) > MaxKeySize || len(key) == 0 {
+		return errKeySize
+	} else if len(field) > MaxHashFieldSize || len(field) == 0 {
+		return errHashFieldSize
+	}
+	return nil
+}
+
+func (db *DB) hEncodeSizeKey(key []byte) []byte {
+	buf := make([]byte, len(key)+1+len(db.indexVarBuf))
+
+	pos := 0
+	n := copy(buf, db.indexVarBuf)
+
+	pos += n
+	buf[pos] = HSizeType
+
+	pos++
+	copy(buf[pos:], key)
+
+	return buf
+}
+
+func (db *DB) hDecodeSizeKey(ek []byte) ([]byte, error) {
+	pos, err := db.checkKeyIndex(ek)
+	if err != nil {
+		return nil, err
+	}
+
+	if pos+1 > len(ek) || ek[pos] != HSizeType {
+		return nil, errHSizeKey
+	}
+	pos++
+
+	return ek[pos:], nil
+}
+
+func (db *DB) hEncodeHashKey(key []byte, field []byte) []byte {
+	buf := make([]byte, len(key)+len(field)+1+1+2+len(db.indexVarBuf))
+
+	pos := 0
+	n := copy(buf, db.indexVarBuf)
+	pos += n
+
+	buf[pos] = HashType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	buf[pos] = hashStartSep
+	pos++
+	copy(buf[pos:], field)
+
+	return buf
+}
+
+func (db *DB) hDecodeHashKey(ek []byte) ([]byte, []byte, error) {
+	pos, err := db.checkKeyIndex(ek)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if pos+1 > len(ek) || ek[pos] != HashType {
+		return nil, nil, errHashKey
+	}
+	pos++
+
+	if pos+2 > len(ek) {
+		return nil, nil, errHashKey
+	}
+
+	keyLen := int(binary.BigEndian.Uint16(ek[pos:]))
+	pos += 2
+
+	if keyLen+pos > len(ek) {
+		return nil, nil, errHashKey
+	}
+
+	key := ek[pos : pos+keyLen]
+	pos += keyLen
+
+	if ek[pos] != hashStartSep {
+		return nil, nil, errHashKey
+	}
+
+	pos++
+	field := ek[pos:]
+	return key, field, nil
+}
+
+func (db *DB) hEncodeStartKey(key []byte) []byte {
+	return db.hEncodeHashKey(key, nil)
+}
+
+func (db *DB) hEncodeStopKey(key []byte) []byte {
+	k := db.hEncodeHashKey(key, nil)
+
+	k[len(k)-1] = hashStopSep
+
+	return k
+}
+
+func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) {
+	t := db.hashBatch
+
+	ek := db.hEncodeHashKey(key, field)
+
+	var n int64 = 1
+	if v, _ := db.bucket.Get(ek); v != nil {
+		n = 0
+	} else {
+		if _, err := db.hIncrSize(key, 1); err != nil {
+			return 0, err
+		}
+	}
+
+	t.Put(ek, value)
+	return n, nil
+}
+
+//	ps : here just focus on deleting the hash data,
+//		 any other likes expire is ignore.
+func (db *DB) hDelete(t *batch, key []byte) int64 {
+	sk := db.hEncodeSizeKey(key)
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	var num int64
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		t.Delete(it.Key())
+		num++
+	}
+	it.Close()
+
+	t.Delete(sk)
+	return num
+}
+
+func (db *DB) hExpireAt(key []byte, when int64) (int64, error) {
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if hlen, err := db.HLen(key); err != nil || hlen == 0 {
+		return 0, err
+	}
+
+	db.expireAt(t, HashType, key, when)
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+// HLen returns the lengh of hash.
+func (db *DB) HLen(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	return Int64(db.bucket.Get(db.hEncodeSizeKey(key)))
+}
+
+// HSet sets the field with value of key.
+func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) {
+	if err := checkHashKFSize(key, field); err != nil {
+		return 0, err
+	} else if err := checkValueSize(value); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.hSetItem(key, field, value)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+// HGet gets the value of the field.
+func (db *DB) HGet(key []byte, field []byte) ([]byte, error) {
+	if err := checkHashKFSize(key, field); err != nil {
+		return nil, err
+	}
+
+	return db.bucket.Get(db.hEncodeHashKey(key, field))
+}
+
+// HMset sets multi field-values.
+func (db *DB) HMset(key []byte, args ...FVPair) error {
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var err error
+	var ek []byte
+	var num int64
+	for i := 0; i < len(args); i++ {
+		if err := checkHashKFSize(key, args[i].Field); err != nil {
+			return err
+		} else if err := checkValueSize(args[i].Value); err != nil {
+			return err
+		}
+
+		ek = db.hEncodeHashKey(key, args[i].Field)
+
+		if v, err := db.bucket.Get(ek); err != nil {
+			return err
+		} else if v == nil {
+			num++
+		}
+
+		t.Put(ek, args[i].Value)
+	}
+
+	if _, err = db.hIncrSize(key, num); err != nil {
+		return err
+	}
+
+	//todo add binglog
+	err = t.Commit()
+	return err
+}
+
+// HMget gets multi values of fields
+func (db *DB) HMget(key []byte, args ...[]byte) ([][]byte, error) {
+	var ek []byte
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	r := make([][]byte, len(args))
+	for i := 0; i < len(args); i++ {
+		if err := checkHashKFSize(key, args[i]); err != nil {
+			return nil, err
+		}
+
+		ek = db.hEncodeHashKey(key, args[i])
+
+		r[i] = it.Find(ek)
+	}
+
+	return r, nil
+}
+
+// HDel deletes the fields.
+func (db *DB) HDel(key []byte, args ...[]byte) (int64, error) {
+	t := db.hashBatch
+
+	var ek []byte
+	var v []byte
+	var err error
+
+	t.Lock()
+	defer t.Unlock()
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	var num int64
+	for i := 0; i < len(args); i++ {
+		if err := checkHashKFSize(key, args[i]); err != nil {
+			return 0, err
+		}
+
+		ek = db.hEncodeHashKey(key, args[i])
+
+		v = it.RawFind(ek)
+		if v == nil {
+			continue
+		} else {
+			num++
+			t.Delete(ek)
+		}
+	}
+
+	if _, err = db.hIncrSize(key, -num); err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+
+	return num, err
+}
+
+func (db *DB) hIncrSize(key []byte, delta int64) (int64, error) {
+	t := db.hashBatch
+	sk := db.hEncodeSizeKey(key)
+
+	var err error
+	var size int64
+	if size, err = Int64(db.bucket.Get(sk)); err != nil {
+		return 0, err
+	}
+
+	size += delta
+	if size <= 0 {
+		size = 0
+		t.Delete(sk)
+		db.rmExpire(t, HashType, key)
+	} else {
+		t.Put(sk, PutInt64(size))
+	}
+
+	return size, nil
+}
+
+// HIncrBy increases the value of field by delta.
+func (db *DB) HIncrBy(key []byte, field []byte, delta int64) (int64, error) {
+	if err := checkHashKFSize(key, field); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	var ek []byte
+	var err error
+
+	t.Lock()
+	defer t.Unlock()
+
+	ek = db.hEncodeHashKey(key, field)
+
+	var n int64
+	if n, err = StrInt64(db.bucket.Get(ek)); err != nil {
+		return 0, err
+	}
+
+	n += delta
+
+	_, err = db.hSetItem(key, field, num.FormatInt64ToSlice(n))
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+
+	return n, err
+}
+
+// HGetAll returns all field-values.
+func (db *DB) HGetAll(key []byte) ([]FVPair, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	v := make([]FVPair, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	defer it.Close()
+
+	for ; it.Valid(); it.Next() {
+		_, f, err := db.hDecodeHashKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		v = append(v, FVPair{Field: f, Value: it.Value()})
+	}
+
+	return v, nil
+}
+
+// HKeys returns the all fields.
+func (db *DB) HKeys(key []byte) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	v := make([][]byte, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	defer it.Close()
+
+	for ; it.Valid(); it.Next() {
+		_, f, err := db.hDecodeHashKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+		v = append(v, f)
+	}
+
+	return v, nil
+}
+
+// HValues returns all values
+func (db *DB) HValues(key []byte) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	v := make([][]byte, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	defer it.Close()
+
+	for ; it.Valid(); it.Next() {
+		_, _, err := db.hDecodeHashKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		v = append(v, it.Value())
+	}
+
+	return v, nil
+}
+
+// HClear clears the data.
+func (db *DB) HClear(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	num := db.hDelete(t, key)
+	db.rmExpire(t, HashType, key)
+
+	err := t.Commit()
+	return num, err
+}
+
+// HMclear cleans multi data.
+func (db *DB) HMclear(keys ...[]byte) (int64, error) {
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return 0, err
+		}
+
+		db.hDelete(t, key)
+		db.rmExpire(t, HashType, key)
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+func (db *DB) hFlush() (drop int64, err error) {
+	t := db.hashBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	return db.flushType(t, HashType)
+}
+
+// HExpire expires the data with duration.
+func (db *DB) HExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.hExpireAt(key, time.Now().Unix()+duration)
+}
+
+// HExpireAt expires the data at time when.
+func (db *DB) HExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.hExpireAt(key, when)
+}
+
+// HTTL gets the TTL of data.
+func (db *DB) HTTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(HashType, key)
+}
+
+// HPersist removes the TTL of data.
+func (db *DB) HPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, HashType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+// HKeyExists checks whether data exists or not.
+func (db *DB) HKeyExists(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+	sk := db.hEncodeSizeKey(key)
+	v, err := db.bucket.Get(sk)
+	if v != nil && err == nil {
+		return 1, nil
+	}
+	return 0, err
+}

vendor/github.com/siddontang/ledisdb/ledis/t_kv.go

@@ -0,0 +1,794 @@
+package ledis
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/siddontang/go/num"
+	"github.com/siddontang/ledisdb/store"
+)
+
+// KVPair is the pair of key-value.
+type KVPair struct {
+	Key   []byte
+	Value []byte
+}
+
+var errKVKey = errors.New("invalid encode kv key")
+
+func checkKeySize(key []byte) error {
+	if len(key) > MaxKeySize || len(key) == 0 {
+		return errKeySize
+	}
+	return nil
+}
+
+func checkValueSize(value []byte) error {
+	if len(value) > MaxValueSize {
+		return errValueSize
+	}
+
+	return nil
+}
+
+func (db *DB) encodeKVKey(key []byte) []byte {
+	ek := make([]byte, len(key)+1+len(db.indexVarBuf))
+	pos := copy(ek, db.indexVarBuf)
+	ek[pos] = KVType
+	pos++
+	copy(ek[pos:], key)
+	return ek
+}
+
+func (db *DB) decodeKVKey(ek []byte) ([]byte, error) {
+	pos, err := db.checkKeyIndex(ek)
+	if err != nil {
+		return nil, err
+	}
+	if pos+1 > len(ek) || ek[pos] != KVType {
+		return nil, errKVKey
+	}
+
+	pos++
+
+	return ek[pos:], nil
+}
+
+func (db *DB) encodeKVMinKey() []byte {
+	ek := db.encodeKVKey(nil)
+	return ek
+}
+
+func (db *DB) encodeKVMaxKey() []byte {
+	ek := db.encodeKVKey(nil)
+	ek[len(ek)-1] = KVType + 1
+	return ek
+}
+
+func (db *DB) incr(key []byte, delta int64) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	var n int64
+	n, err = StrInt64(db.bucket.Get(key))
+	if err != nil {
+		return 0, err
+	}
+
+	n += delta
+
+	t.Put(key, num.FormatInt64ToSlice(n))
+
+	err = t.Commit()
+	return n, err
+}
+
+//	ps : here just focus on deleting the key-value data,
+//		 any other likes expire is ignore.
+func (db *DB) delete(t *batch, key []byte) int64 {
+	key = db.encodeKVKey(key)
+	t.Delete(key)
+	return 1
+}
+
+func (db *DB) setExpireAt(key []byte, when int64) (int64, error) {
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if exist, err := db.Exists(key); err != nil || exist == 0 {
+		return 0, err
+	}
+
+	db.expireAt(t, KVType, key, when)
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+// Decr decreases the data.
+func (db *DB) Decr(key []byte) (int64, error) {
+	return db.incr(key, -1)
+}
+
+// DecrBy decreases the data by decrement.
+func (db *DB) DecrBy(key []byte, decrement int64) (int64, error) {
+	return db.incr(key, -decrement)
+}
+
+// Del deletes the data.
+func (db *DB) Del(keys ...[]byte) (int64, error) {
+	if len(keys) == 0 {
+		return 0, nil
+	}
+
+	codedKeys := make([][]byte, len(keys))
+	for i, k := range keys {
+		codedKeys[i] = db.encodeKVKey(k)
+	}
+
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for i, k := range keys {
+		t.Delete(codedKeys[i])
+		db.rmExpire(t, KVType, k)
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+// Exists check data exists or not.
+func (db *DB) Exists(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	var v []byte
+	v, err = db.bucket.Get(key)
+	if v != nil && err == nil {
+		return 1, nil
+	}
+
+	return 0, err
+}
+
+// Get gets the value.
+func (db *DB) Get(key []byte) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	key = db.encodeKVKey(key)
+
+	return db.bucket.Get(key)
+}
+
+// GetSlice gets the slice of the data.
+func (db *DB) GetSlice(key []byte) (store.Slice, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	key = db.encodeKVKey(key)
+
+	return db.bucket.GetSlice(key)
+}
+
+// GetSet gets the value and sets new value.
+func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	} else if err := checkValueSize(value); err != nil {
+		return nil, err
+	}
+
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	oldValue, err := db.bucket.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	t.Put(key, value)
+
+	err = t.Commit()
+
+	return oldValue, err
+}
+
+// Incr increases the data.
+func (db *DB) Incr(key []byte) (int64, error) {
+	return db.incr(key, 1)
+}
+
+// IncrBy increases the data by increment.
+func (db *DB) IncrBy(key []byte, increment int64) (int64, error) {
+	return db.incr(key, increment)
+}
+
+// MGet gets multi data.
+func (db *DB) MGet(keys ...[]byte) ([][]byte, error) {
+	values := make([][]byte, len(keys))
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	for i := range keys {
+		if err := checkKeySize(keys[i]); err != nil {
+			return nil, err
+		}
+
+		values[i] = it.Find(db.encodeKVKey(keys[i]))
+	}
+
+	return values, nil
+}
+
+// MSet sets multi data.
+func (db *DB) MSet(args ...KVPair) error {
+	if len(args) == 0 {
+		return nil
+	}
+
+	t := db.kvBatch
+
+	var err error
+	var key []byte
+	var value []byte
+
+	t.Lock()
+	defer t.Unlock()
+
+	for i := 0; i < len(args); i++ {
+		if err := checkKeySize(args[i].Key); err != nil {
+			return err
+		} else if err := checkValueSize(args[i].Value); err != nil {
+			return err
+		}
+
+		key = db.encodeKVKey(args[i].Key)
+
+		value = args[i].Value
+
+		t.Put(key, value)
+
+	}
+
+	err = t.Commit()
+	return err
+}
+
+// Set sets the data.
+func (db *DB) Set(key []byte, value []byte) error {
+	if err := checkKeySize(key); err != nil {
+		return err
+	} else if err := checkValueSize(value); err != nil {
+		return err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	t.Put(key, value)
+
+	err = t.Commit()
+
+	return err
+}
+
+// SetNX sets the data if not existed.
+func (db *DB) SetNX(key []byte, value []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	} else if err := checkValueSize(value); err != nil {
+		return 0, err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	var n int64 = 1
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	if v, err := db.bucket.Get(key); err != nil {
+		return 0, err
+	} else if v != nil {
+		n = 0
+	} else {
+		t.Put(key, value)
+
+		err = t.Commit()
+	}
+
+	return n, err
+}
+
+// SetEX sets the data with a TTL.
+func (db *DB) SetEX(key []byte, duration int64, value []byte) error {
+	if err := checkKeySize(key); err != nil {
+		return err
+	} else if err := checkValueSize(value); err != nil {
+		return err
+	} else if duration <= 0 {
+		return errExpireValue
+	}
+
+	ek := db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	t.Put(ek, value)
+	db.expireAt(t, KVType, key, time.Now().Unix()+duration)
+
+	return t.Commit()
+}
+
+func (db *DB) flush() (drop int64, err error) {
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+	return db.flushType(t, KVType)
+}
+
+// Expire expires the data.
+func (db *DB) Expire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.setExpireAt(key, time.Now().Unix()+duration)
+}
+
+// ExpireAt expires the data at when.
+func (db *DB) ExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.setExpireAt(key, when)
+}
+
+// TTL returns the TTL of the data.
+func (db *DB) TTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(KVType, key)
+}
+
+// Persist removes the TTL of the data.
+func (db *DB) Persist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+	n, err := db.rmExpire(t, KVType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+// SetRange sets the data with new value from offset.
+func (db *DB) SetRange(key []byte, offset int, value []byte) (int64, error) {
+	if len(value) == 0 {
+		return 0, nil
+	}
+
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	} else if len(value)+offset > MaxValueSize {
+		return 0, errValueSize
+	}
+
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	oldValue, err := db.bucket.Get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	extra := offset + len(value) - len(oldValue)
+	if extra > 0 {
+		oldValue = append(oldValue, make([]byte, extra)...)
+	}
+
+	copy(oldValue[offset:], value)
+
+	t.Put(key, oldValue)
+
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+
+	return int64(len(oldValue)), nil
+}
+
+func getRange(start int, end int, valLen int) (int, int) {
+	if start < 0 {
+		start = valLen + start
+	}
+
+	if end < 0 {
+		end = valLen + end
+	}
+
+	if start < 0 {
+		start = 0
+	}
+
+	if end < 0 {
+		end = 0
+	}
+
+	if end >= valLen {
+		end = valLen - 1
+	}
+	return start, end
+}
+
+// GetRange gets the range of the data.
+func (db *DB) GetRange(key []byte, start int, end int) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+	key = db.encodeKVKey(key)
+
+	value, err := db.bucket.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	valLen := len(value)
+
+	start, end = getRange(start, end, valLen)
+
+	if start > end {
+		return nil, nil
+	}
+
+	return value[start : end+1], nil
+}
+
+// StrLen returns the length of the data.
+func (db *DB) StrLen(key []byte) (int64, error) {
+	s, err := db.GetSlice(key)
+	if err != nil {
+		return 0, err
+	}
+
+	n := s.Size()
+	s.Free()
+	return int64(n), nil
+}
+
+// Append appends the value to the data.
+func (db *DB) Append(key []byte, value []byte) (int64, error) {
+	if len(value) == 0 {
+		return 0, nil
+	}
+
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	oldValue, err := db.bucket.Get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	if len(oldValue)+len(value) > MaxValueSize {
+		return 0, errValueSize
+	}
+
+	oldValue = append(oldValue, value...)
+
+	t.Put(key, oldValue)
+
+	if err := t.Commit(); err != nil {
+		return 0, nil
+	}
+
+	return int64(len(oldValue)), nil
+}
+
+// BitOP does the bit operations in data.
+func (db *DB) BitOP(op string, destKey []byte, srcKeys ...[]byte) (int64, error) {
+	if err := checkKeySize(destKey); err != nil {
+		return 0, err
+	}
+
+	op = strings.ToLower(op)
+	if len(srcKeys) == 0 {
+		return 0, nil
+	} else if op == BitNot && len(srcKeys) > 1 {
+		return 0, fmt.Errorf("BITOP NOT has only one srckey")
+	} else if len(srcKeys) < 2 {
+		return 0, nil
+	}
+
+	key := db.encodeKVKey(srcKeys[0])
+
+	value, err := db.bucket.Get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	if op == BitNot {
+		for i := 0; i < len(value); i++ {
+			value[i] = ^value[i]
+		}
+	} else {
+		for j := 1; j < len(srcKeys); j++ {
+			if err := checkKeySize(srcKeys[j]); err != nil {
+				return 0, err
+			}
+
+			key = db.encodeKVKey(srcKeys[j])
+			ovalue, err := db.bucket.Get(key)
+			if err != nil {
+				return 0, err
+			}
+
+			if len(value) < len(ovalue) {
+				value, ovalue = ovalue, value
+			}
+
+			for i := 0; i < len(ovalue); i++ {
+				switch op {
+				case BitAND:
+					value[i] &= ovalue[i]
+				case BitOR:
+					value[i] |= ovalue[i]
+				case BitXOR:
+					value[i] ^= ovalue[i]
+				default:
+					return 0, fmt.Errorf("invalid op type: %s", op)
+				}
+			}
+
+			for i := len(ovalue); i < len(value); i++ {
+				switch op {
+				case BitAND:
+					value[i] &= 0
+				case BitOR:
+					value[i] |= 0
+				case BitXOR:
+					value[i] ^= 0
+				}
+			}
+		}
+	}
+
+	key = db.encodeKVKey(destKey)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	t.Put(key, value)
+
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+
+	return int64(len(value)), nil
+}
+
+var bitsInByte = [256]int32{0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3,
+	4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3,
+	3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4,
+	5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4,
+	3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4,
+	5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2,
+	2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3,
+	4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+	3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4,
+	5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6,
+	6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5,
+	6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}
+
+func numberBitCount(i uint32) uint32 {
+	i = i - ((i >> 1) & 0x55555555)
+	i = (i & 0x33333333) + ((i >> 2) & 0x33333333)
+	return (((i + (i >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24
+}
+
+// BitCount returns the bit count of data.
+func (db *DB) BitCount(key []byte, start int, end int) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	key = db.encodeKVKey(key)
+	value, err := db.bucket.Get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	start, end = getRange(start, end, len(value))
+	value = value[start : end+1]
+
+	var n int64
+
+	pos := 0
+	for ; pos+4 <= len(value); pos = pos + 4 {
+		n += int64(numberBitCount(binary.BigEndian.Uint32(value[pos : pos+4])))
+	}
+
+	for ; pos < len(value); pos++ {
+		n += int64(bitsInByte[value[pos]])
+	}
+
+	return n, nil
+}
+
+// BitPos returns the pos of the data.
+func (db *DB) BitPos(key []byte, on int, start int, end int) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	if (on & ^1) != 0 {
+		return 0, fmt.Errorf("bit must be 0 or 1, not %d", on)
+	}
+
+	var skipValue uint8
+	if on == 0 {
+		skipValue = 0xFF
+	}
+
+	key = db.encodeKVKey(key)
+	value, err := db.bucket.Get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	start, end = getRange(start, end, len(value))
+	value = value[start : end+1]
+
+	for i, v := range value {
+		if uint8(v) != skipValue {
+			for j := 0; j < 8; j++ {
+				isNull := uint8(v)&(1<<uint8(7-j)) == 0
+
+				if (on == 1 && !isNull) || (on == 0 && isNull) {
+					return int64((start+i)*8 + j), nil
+				}
+			}
+		}
+	}
+
+	return -1, nil
+}
+
+// SetBit sets the bit to the data.
+func (db *DB) SetBit(key []byte, offset int, on int) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	if (on & ^1) != 0 {
+		return 0, fmt.Errorf("bit must be 0 or 1, not %d", on)
+	}
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	key = db.encodeKVKey(key)
+	value, err := db.bucket.Get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	byteOffset := int(uint32(offset) >> 3)
+	extra := byteOffset + 1 - len(value)
+	if extra > 0 {
+		value = append(value, make([]byte, extra)...)
+	}
+
+	byteVal := value[byteOffset]
+	bit := 7 - uint8(uint32(offset)&0x7)
+	bitVal := byteVal & (1 << bit)
+
+	byteVal &= ^(1 << bit)
+	byteVal |= (uint8(on&0x1) << bit)
+
+	value[byteOffset] = byteVal
+
+	t.Put(key, value)
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+
+	if bitVal > 0 {
+		return 1, nil
+	}
+
+	return 0, nil
+}
+
+// GetBit gets the bit of data at offset.
+func (db *DB) GetBit(key []byte, offset int) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	key = db.encodeKVKey(key)
+
+	value, err := db.bucket.Get(key)
+	if err != nil {
+		return 0, err
+	}
+
+	byteOffset := uint32(offset) >> 3
+	bit := 7 - uint8(uint32(offset)&0x7)
+
+	if byteOffset >= uint32(len(value)) {
+		return 0, nil
+	}
+
+	bitVal := value[byteOffset] & (1 << bit)
+	if bitVal > 0 {
+		return 1, nil
+	}
+
+	return 0, nil
+}

vendor/github.com/siddontang/ledisdb/ledis/t_list.go

@@ -0,0 +1,808 @@
+package ledis
+
+import (
+	"container/list"
+	"encoding/binary"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/siddontang/go/hack"
+	"github.com/siddontang/go/log"
+	"github.com/siddontang/go/num"
+	"github.com/siddontang/ledisdb/store"
+	"golang.org/x/net/context"
+)
+
+const (
+	listHeadSeq int32 = 1
+	listTailSeq int32 = 2
+
+	listMinSeq     int32 = 1000
+	listMaxSeq     int32 = 1<<31 - 1000
+	listInitialSeq int32 = listMinSeq + (listMaxSeq-listMinSeq)/2
+)
+
+var errLMetaKey = errors.New("invalid lmeta key")
+var errListKey = errors.New("invalid list key")
+var errListSeq = errors.New("invalid list sequence, overflow")
+
+func (db *DB) lEncodeMetaKey(key []byte) []byte {
+	buf := make([]byte, len(key)+1+len(db.indexVarBuf))
+	pos := copy(buf, db.indexVarBuf)
+	buf[pos] = LMetaType
+	pos++
+
+	copy(buf[pos:], key)
+	return buf
+}
+
+func (db *DB) lDecodeMetaKey(ek []byte) ([]byte, error) {
+	pos, err := db.checkKeyIndex(ek)
+	if err != nil {
+		return nil, err
+	}
+
+	if pos+1 > len(ek) || ek[pos] != LMetaType {
+		return nil, errLMetaKey
+	}
+
+	pos++
+	return ek[pos:], nil
+}
+
+func (db *DB) lEncodeListKey(key []byte, seq int32) []byte {
+	buf := make([]byte, len(key)+7+len(db.indexVarBuf))
+
+	pos := copy(buf, db.indexVarBuf)
+
+	buf[pos] = ListType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	binary.BigEndian.PutUint32(buf[pos:], uint32(seq))
+
+	return buf
+}
+
+func (db *DB) lDecodeListKey(ek []byte) (key []byte, seq int32, err error) {
+	pos := 0
+	pos, err = db.checkKeyIndex(ek)
+	if err != nil {
+		return
+	}
+
+	if pos+1 > len(ek) || ek[pos] != ListType {
+		err = errListKey
+		return
+	}
+
+	pos++
+
+	if pos+2 > len(ek) {
+		err = errListKey
+		return
+	}
+
+	keyLen := int(binary.BigEndian.Uint16(ek[pos:]))
+	pos += 2
+	if keyLen+pos+4 != len(ek) {
+		err = errListKey
+		return
+	}
+
+	key = ek[pos : pos+keyLen]
+	seq = int32(binary.BigEndian.Uint32(ek[pos+keyLen:]))
+	return
+}
+
+func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	var headSeq int32
+	var tailSeq int32
+	var size int32
+	var err error
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	metaKey := db.lEncodeMetaKey(key)
+	headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey)
+	if err != nil {
+		return 0, err
+	}
+
+	pushCnt := len(args)
+	if pushCnt == 0 {
+		return int64(size), nil
+	}
+
+	seq := headSeq
+	var delta int32 = -1
+	if whereSeq == listTailSeq {
+		seq = tailSeq
+		delta = 1
+	}
+
+	//	append elements
+	if size > 0 {
+		seq += delta
+	}
+
+	for i := 0; i < pushCnt; i++ {
+		ek := db.lEncodeListKey(key, seq+int32(i)*delta)
+		t.Put(ek, args[i])
+	}
+
+	seq += int32(pushCnt-1) * delta
+	if seq <= listMinSeq || seq >= listMaxSeq {
+		return 0, errListSeq
+	}
+
+	//	set meta info
+	if whereSeq == listHeadSeq {
+		headSeq = seq
+	} else {
+		tailSeq = seq
+	}
+
+	db.lSetMeta(metaKey, headSeq, tailSeq)
+
+	err = t.Commit()
+
+	if err == nil {
+		db.lSignalAsReady(key)
+	}
+
+	return int64(size) + int64(pushCnt), err
+}
+
+func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var headSeq int32
+	var tailSeq int32
+	var size int32
+	var err error
+
+	metaKey := db.lEncodeMetaKey(key)
+	headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey)
+	if err != nil {
+		return nil, err
+	} else if size == 0 {
+		return nil, nil
+	}
+
+	var value []byte
+
+	seq := headSeq
+	if whereSeq == listTailSeq {
+		seq = tailSeq
+	}
+
+	itemKey := db.lEncodeListKey(key, seq)
+	value, err = db.bucket.Get(itemKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if whereSeq == listHeadSeq {
+		headSeq++
+	} else {
+		tailSeq--
+	}
+
+	t.Delete(itemKey)
+	size = db.lSetMeta(metaKey, headSeq, tailSeq)
+	if size == 0 {
+		db.rmExpire(t, ListType, key)
+	}
+
+	err = t.Commit()
+	return value, err
+}
+
+func (db *DB) ltrim2(key []byte, startP, stopP int64) (err error) {
+	if err := checkKeySize(key); err != nil {
+		return err
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var headSeq int32
+	var llen int32
+	start := int32(startP)
+	stop := int32(stopP)
+
+	ek := db.lEncodeMetaKey(key)
+	if headSeq, _, llen, err = db.lGetMeta(nil, ek); err != nil {
+		return err
+	}
+
+	if start < 0 {
+		start = llen + start
+	}
+	if stop < 0 {
+		stop = llen + stop
+	}
+	if start >= llen || start > stop {
+		db.lDelete(t, key)
+		db.rmExpire(t, ListType, key)
+		return t.Commit()
+	}
+
+	if start < 0 {
+		start = 0
+	}
+	if stop >= llen {
+		stop = llen - 1
+	}
+
+	if start > 0 {
+		for i := int32(0); i < start; i++ {
+			t.Delete(db.lEncodeListKey(key, headSeq+i))
+		}
+	}
+	if stop < int32(llen-1) {
+		for i := int32(stop + 1); i < llen; i++ {
+			t.Delete(db.lEncodeListKey(key, headSeq+i))
+		}
+	}
+
+	db.lSetMeta(ek, headSeq+start, headSeq+stop)
+
+	return t.Commit()
+}
+
+func (db *DB) ltrim(key []byte, trimSize, whereSeq int32) (int32, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	if trimSize == 0 {
+		return 0, nil
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var headSeq int32
+	var tailSeq int32
+	var size int32
+	var err error
+
+	metaKey := db.lEncodeMetaKey(key)
+	headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey)
+	if err != nil {
+		return 0, err
+	} else if size == 0 {
+		return 0, nil
+	}
+
+	var (
+		trimStartSeq int32
+		trimEndSeq   int32
+	)
+
+	if whereSeq == listHeadSeq {
+		trimStartSeq = headSeq
+		trimEndSeq = num.MinInt32(trimStartSeq+trimSize-1, tailSeq)
+		headSeq = trimEndSeq + 1
+	} else {
+		trimEndSeq = tailSeq
+		trimStartSeq = num.MaxInt32(trimEndSeq-trimSize+1, headSeq)
+		tailSeq = trimStartSeq - 1
+	}
+
+	for trimSeq := trimStartSeq; trimSeq <= trimEndSeq; trimSeq++ {
+		itemKey := db.lEncodeListKey(key, trimSeq)
+		t.Delete(itemKey)
+	}
+
+	size = db.lSetMeta(metaKey, headSeq, tailSeq)
+	if size == 0 {
+		db.rmExpire(t, ListType, key)
+	}
+
+	err = t.Commit()
+	return trimEndSeq - trimStartSeq + 1, err
+}
+
+//	ps : here just focus on deleting the list data,
+//		 any other likes expire is ignore.
+func (db *DB) lDelete(t *batch, key []byte) int64 {
+	mk := db.lEncodeMetaKey(key)
+
+	var headSeq int32
+	var tailSeq int32
+	var err error
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	headSeq, tailSeq, _, err = db.lGetMeta(it, mk)
+	if err != nil {
+		return 0
+	}
+
+	var num int64
+	startKey := db.lEncodeListKey(key, headSeq)
+	stopKey := db.lEncodeListKey(key, tailSeq)
+
+	rit := store.NewRangeIterator(it, &store.Range{
+		Min:  startKey,
+		Max:  stopKey,
+		Type: store.RangeClose})
+	for ; rit.Valid(); rit.Next() {
+		t.Delete(rit.RawKey())
+		num++
+	}
+
+	t.Delete(mk)
+
+	return num
+}
+
+func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq int32, size int32, err error) {
+	var v []byte
+	if it != nil {
+		v = it.Find(ek)
+	} else {
+		v, err = db.bucket.Get(ek)
+	}
+	if err != nil {
+		return
+	} else if v == nil {
+		headSeq = listInitialSeq
+		tailSeq = listInitialSeq
+		size = 0
+		return
+	} else {
+		headSeq = int32(binary.LittleEndian.Uint32(v[0:4]))
+		tailSeq = int32(binary.LittleEndian.Uint32(v[4:8]))
+		size = tailSeq - headSeq + 1
+	}
+	return
+}
+
+func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 {
+	t := db.listBatch
+
+	size := tailSeq - headSeq + 1
+	if size < 0 {
+		//	todo : log error + panic
+		log.Fatalf("invalid meta sequence range [%d, %d]", headSeq, tailSeq)
+	} else if size == 0 {
+		t.Delete(ek)
+	} else {
+		buf := make([]byte, 8)
+
+		binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq))
+		binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq))
+
+		t.Put(ek, buf)
+	}
+
+	return size
+}
+
+func (db *DB) lExpireAt(key []byte, when int64) (int64, error) {
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if llen, err := db.LLen(key); err != nil || llen == 0 {
+		return 0, err
+	}
+
+	db.expireAt(t, ListType, key, when)
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+// LIndex returns the value at index.
+func (db *DB) LIndex(key []byte, index int32) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	var seq int32
+	var headSeq int32
+	var tailSeq int32
+	var err error
+
+	metaKey := db.lEncodeMetaKey(key)
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	headSeq, tailSeq, _, err = db.lGetMeta(it, metaKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if index >= 0 {
+		seq = headSeq + index
+	} else {
+		seq = tailSeq + index + 1
+	}
+
+	sk := db.lEncodeListKey(key, seq)
+	v := it.Find(sk)
+
+	return v, nil
+}
+
+// LLen gets the length of the list.
+func (db *DB) LLen(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	ek := db.lEncodeMetaKey(key)
+	_, _, size, err := db.lGetMeta(nil, ek)
+	return int64(size), err
+}
+
+// LPop pops the value.
+func (db *DB) LPop(key []byte) ([]byte, error) {
+	return db.lpop(key, listHeadSeq)
+}
+
+// LTrim trims the value from start to stop.
+func (db *DB) LTrim(key []byte, start, stop int64) error {
+	return db.ltrim2(key, start, stop)
+}
+
+// LTrimFront trims the value from top.
+func (db *DB) LTrimFront(key []byte, trimSize int32) (int32, error) {
+	return db.ltrim(key, trimSize, listHeadSeq)
+}
+
+// LTrimBack trims the value from back.
+func (db *DB) LTrimBack(key []byte, trimSize int32) (int32, error) {
+	return db.ltrim(key, trimSize, listTailSeq)
+}
+
+// LPush push the value to the list.
+func (db *DB) LPush(key []byte, args ...[]byte) (int64, error) {
+	return db.lpush(key, listHeadSeq, args...)
+}
+
+// LSet sets the value at index.
+func (db *DB) LSet(key []byte, index int32, value []byte) error {
+	if err := checkKeySize(key); err != nil {
+		return err
+	}
+
+	var seq int32
+	var headSeq int32
+	var tailSeq int32
+	//var size int32
+	var err error
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+	metaKey := db.lEncodeMetaKey(key)
+
+	headSeq, tailSeq, _, err = db.lGetMeta(nil, metaKey)
+	if err != nil {
+		return err
+	}
+
+	if index >= 0 {
+		seq = headSeq + index
+	} else {
+		seq = tailSeq + index + 1
+	}
+	if seq < headSeq || seq > tailSeq {
+		return errListIndex
+	}
+	sk := db.lEncodeListKey(key, seq)
+	t.Put(sk, value)
+	err = t.Commit()
+	return err
+}
+
+// LRange gets the value of list at range.
+func (db *DB) LRange(key []byte, start int32, stop int32) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	var headSeq int32
+	var llen int32
+	var err error
+
+	metaKey := db.lEncodeMetaKey(key)
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	if headSeq, _, llen, err = db.lGetMeta(it, metaKey); err != nil {
+		return nil, err
+	}
+
+	if start < 0 {
+		start = llen + start
+	}
+	if stop < 0 {
+		stop = llen + stop
+	}
+	if start < 0 {
+		start = 0
+	}
+
+	if start > stop || start >= llen {
+		return [][]byte{}, nil
+	}
+
+	if stop >= llen {
+		stop = llen - 1
+	}
+
+	limit := (stop - start) + 1
+	headSeq += start
+
+	v := make([][]byte, 0, limit)
+
+	startKey := db.lEncodeListKey(key, headSeq)
+	rit := store.NewRangeLimitIterator(it,
+		&store.Range{
+			Min:  startKey,
+			Max:  nil,
+			Type: store.RangeClose},
+		&store.Limit{
+			Offset: 0,
+			Count:  int(limit)})
+
+	for ; rit.Valid(); rit.Next() {
+		v = append(v, rit.Value())
+	}
+
+	return v, nil
+}
+
+// RPop rpops the value.
+func (db *DB) RPop(key []byte) ([]byte, error) {
+	return db.lpop(key, listTailSeq)
+}
+
+// RPush rpushs the value .
+func (db *DB) RPush(key []byte, args ...[]byte) (int64, error) {
+	return db.lpush(key, listTailSeq, args...)
+}
+
+// LClear clears the list.
+func (db *DB) LClear(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	num := db.lDelete(t, key)
+	db.rmExpire(t, ListType, key)
+
+	err := t.Commit()
+	return num, err
+}
+
+// LMclear clears multi lists.
+func (db *DB) LMclear(keys ...[]byte) (int64, error) {
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return 0, err
+		}
+
+		db.lDelete(t, key)
+		db.rmExpire(t, ListType, key)
+
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+func (db *DB) lFlush() (drop int64, err error) {
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+	return db.flushType(t, ListType)
+}
+
+// LExpire expires the list.
+func (db *DB) LExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.lExpireAt(key, time.Now().Unix()+duration)
+}
+
+// LExpireAt expires the list at when.
+func (db *DB) LExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.lExpireAt(key, when)
+}
+
+// LTTL gets the TTL of list.
+func (db *DB) LTTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(ListType, key)
+}
+
+// LPersist removes the TTL of list.
+func (db *DB) LPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, ListType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+func (db *DB) lEncodeMinKey() []byte {
+	return db.lEncodeMetaKey(nil)
+}
+
+func (db *DB) lEncodeMaxKey() []byte {
+	ek := db.lEncodeMetaKey(nil)
+	ek[len(ek)-1] = LMetaType + 1
+	return ek
+}
+
+// BLPop pops the list with block way.
+func (db *DB) BLPop(keys [][]byte, timeout time.Duration) ([]interface{}, error) {
+	return db.lblockPop(keys, listHeadSeq, timeout)
+}
+
+// BRPop bpops the list with block way.
+func (db *DB) BRPop(keys [][]byte, timeout time.Duration) ([]interface{}, error) {
+	return db.lblockPop(keys, listTailSeq, timeout)
+}
+
+// LKeyExists check list existed or not.
+func (db *DB) LKeyExists(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+	sk := db.lEncodeMetaKey(key)
+	v, err := db.bucket.Get(sk)
+	if v != nil && err == nil {
+		return 1, nil
+	}
+	return 0, err
+}
+
+func (db *DB) lblockPop(keys [][]byte, whereSeq int32, timeout time.Duration) ([]interface{}, error) {
+	for {
+		var ctx context.Context
+		var cancel context.CancelFunc
+		if timeout > 0 {
+			ctx, cancel = context.WithTimeout(context.Background(), timeout)
+		} else {
+			ctx, cancel = context.WithCancel(context.Background())
+		}
+
+		for _, key := range keys {
+			v, err := db.lbkeys.popOrWait(db, key, whereSeq, cancel)
+
+			if err != nil {
+				cancel()
+				return nil, err
+			} else if v != nil {
+				cancel()
+				return []interface{}{key, v}, nil
+			}
+		}
+
+		//blocking wait
+		<-ctx.Done()
+		cancel()
+
+		//if ctx.Err() is a deadline exceeded (timeout) we return
+		//otherwise we try to pop one of the keys again.
+		if ctx.Err() == context.DeadlineExceeded {
+			return nil, nil
+		}
+	}
+}
+
+func (db *DB) lSignalAsReady(key []byte) {
+	db.lbkeys.signal(key)
+}
+
+type lBlockKeys struct {
+	sync.Mutex
+
+	keys map[string]*list.List
+}
+
+func newLBlockKeys() *lBlockKeys {
+	l := new(lBlockKeys)
+
+	l.keys = make(map[string]*list.List)
+	return l
+}
+
+func (l *lBlockKeys) signal(key []byte) {
+	l.Lock()
+	defer l.Unlock()
+
+	s := hack.String(key)
+	fns, ok := l.keys[s]
+	if !ok {
+		return
+	}
+	for e := fns.Front(); e != nil; e = e.Next() {
+		fn := e.Value.(context.CancelFunc)
+		fn()
+	}
+
+	delete(l.keys, s)
+}
+
+func (l *lBlockKeys) popOrWait(db *DB, key []byte, whereSeq int32, fn context.CancelFunc) ([]interface{}, error) {
+	v, err := db.lpop(key, whereSeq)
+	if err != nil {
+		return nil, err
+	} else if v != nil {
+		return []interface{}{key, v}, nil
+	}
+
+	l.Lock()
+
+	s := hack.String(key)
+	chs, ok := l.keys[s]
+	if !ok {
+		chs = list.New()
+		l.keys[s] = chs
+	}
+
+	chs.PushBack(fn)
+	l.Unlock()
+	return nil, nil
+}

vendor/github.com/siddontang/ledisdb/ledis/t_set.go

@@ -0,0 +1,644 @@
+package ledis
+
+import (
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/siddontang/go/hack"
+	"github.com/siddontang/ledisdb/store"
+)
+
+var errSetKey = errors.New("invalid set key")
+var errSSizeKey = errors.New("invalid ssize key")
+
+// For set operation type.
+const (
+	setStartSep byte = ':'
+	setStopSep  byte = setStartSep + 1
+	UnionType   byte = 51
+	DiffType    byte = 52
+	InterType   byte = 53
+)
+
+func checkSetKMSize(key []byte, member []byte) error {
+	if len(key) > MaxKeySize || len(key) == 0 {
+		return errKeySize
+	} else if len(member) > MaxSetMemberSize || len(member) == 0 {
+		return errSetMemberSize
+	}
+	return nil
+}
+
+func (db *DB) sEncodeSizeKey(key []byte) []byte {
+	buf := make([]byte, len(key)+1+len(db.indexVarBuf))
+
+	pos := copy(buf, db.indexVarBuf)
+	buf[pos] = SSizeType
+
+	pos++
+
+	copy(buf[pos:], key)
+	return buf
+}
+
+func (db *DB) sDecodeSizeKey(ek []byte) ([]byte, error) {
+	pos, err := db.checkKeyIndex(ek)
+	if err != nil {
+		return nil, err
+	}
+
+	if pos+1 > len(ek) || ek[pos] != SSizeType {
+		return nil, errSSizeKey
+	}
+	pos++
+
+	return ek[pos:], nil
+}
+
+func (db *DB) sEncodeSetKey(key []byte, member []byte) []byte {
+	buf := make([]byte, len(key)+len(member)+1+1+2+len(db.indexVarBuf))
+
+	pos := copy(buf, db.indexVarBuf)
+
+	buf[pos] = SetType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	buf[pos] = setStartSep
+	pos++
+	copy(buf[pos:], member)
+
+	return buf
+}
+
+func (db *DB) sDecodeSetKey(ek []byte) ([]byte, []byte, error) {
+	pos, err := db.checkKeyIndex(ek)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if pos+1 > len(ek) || ek[pos] != SetType {
+		return nil, nil, errSetKey
+	}
+
+	pos++
+
+	if pos+2 > len(ek) {
+		return nil, nil, errSetKey
+	}
+
+	keyLen := int(binary.BigEndian.Uint16(ek[pos:]))
+	pos += 2
+
+	if keyLen+pos > len(ek) {
+		return nil, nil, errSetKey
+	}
+
+	key := ek[pos : pos+keyLen]
+	pos += keyLen
+
+	if ek[pos] != hashStartSep {
+		return nil, nil, errSetKey
+	}
+
+	pos++
+	member := ek[pos:]
+	return key, member, nil
+}
+
+func (db *DB) sEncodeStartKey(key []byte) []byte {
+	return db.sEncodeSetKey(key, nil)
+}
+
+func (db *DB) sEncodeStopKey(key []byte) []byte {
+	k := db.sEncodeSetKey(key, nil)
+
+	k[len(k)-1] = setStopSep
+
+	return k
+}
+
+func (db *DB) sFlush() (drop int64, err error) {
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	return db.flushType(t, SetType)
+}
+
+func (db *DB) sDelete(t *batch, key []byte) int64 {
+	sk := db.sEncodeSizeKey(key)
+	start := db.sEncodeStartKey(key)
+	stop := db.sEncodeStopKey(key)
+
+	var num int64
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		t.Delete(it.RawKey())
+		num++
+	}
+
+	it.Close()
+
+	t.Delete(sk)
+	return num
+}
+
+func (db *DB) sIncrSize(key []byte, delta int64) (int64, error) {
+	t := db.setBatch
+	sk := db.sEncodeSizeKey(key)
+
+	var err error
+	var size int64
+	if size, err = Int64(db.bucket.Get(sk)); err != nil {
+		return 0, err
+	}
+
+	size += delta
+	if size <= 0 {
+		size = 0
+		t.Delete(sk)
+		db.rmExpire(t, SetType, key)
+	} else {
+		t.Put(sk, PutInt64(size))
+	}
+
+	return size, nil
+}
+
+func (db *DB) sExpireAt(key []byte, when int64) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if scnt, err := db.SCard(key); err != nil || scnt == 0 {
+		return 0, err
+	}
+	db.expireAt(t, SetType, key, when)
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+
+	return 1, nil
+}
+
+func (db *DB) sSetItem(key []byte, member []byte) (int64, error) {
+	t := db.setBatch
+	ek := db.sEncodeSetKey(key, member)
+
+	var n int64 = 1
+	if v, _ := db.bucket.Get(ek); v != nil {
+		n = 0
+	} else {
+		if _, err := db.sIncrSize(key, 1); err != nil {
+			return 0, err
+		}
+	}
+
+	t.Put(ek, nil)
+	return n, nil
+}
+
+// SAdd adds the value to the set.
+func (db *DB) SAdd(key []byte, args ...[]byte) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var err error
+	var ek []byte
+	var num int64
+	for i := 0; i < len(args); i++ {
+		if err := checkSetKMSize(key, args[i]); err != nil {
+			return 0, err
+		}
+
+		ek = db.sEncodeSetKey(key, args[i])
+
+		if v, err := db.bucket.Get(ek); err != nil {
+			return 0, err
+		} else if v == nil {
+			num++
+		}
+
+		t.Put(ek, nil)
+	}
+
+	if _, err = db.sIncrSize(key, num); err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return num, err
+
+}
+
+// SCard gets the size of set.
+func (db *DB) SCard(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	sk := db.sEncodeSizeKey(key)
+
+	return Int64(db.bucket.Get(sk))
+}
+
+func (db *DB) sDiffGeneric(keys ...[]byte) ([][]byte, error) {
+	destMap := make(map[string]bool)
+
+	members, err := db.SMembers(keys[0])
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range members {
+		destMap[hack.String(m)] = true
+	}
+
+	for _, k := range keys[1:] {
+		members, err := db.SMembers(k)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, m := range members {
+			if _, ok := destMap[hack.String(m)]; !ok {
+				continue
+			} else if ok {
+				delete(destMap, hack.String(m))
+			}
+		}
+		// O - A = O, O is zero set.
+		if len(destMap) == 0 {
+			return nil, nil
+		}
+	}
+
+	slice := make([][]byte, len(destMap))
+	idx := 0
+	for k, v := range destMap {
+		if !v {
+			continue
+		}
+		slice[idx] = []byte(k)
+		idx++
+	}
+
+	return slice, nil
+}
+
+// SDiff gets the different of sets.
+func (db *DB) SDiff(keys ...[]byte) ([][]byte, error) {
+	v, err := db.sDiffGeneric(keys...)
+	return v, err
+}
+
+// SDiffStore gets the different of sets and stores to dest set.
+func (db *DB) SDiffStore(dstKey []byte, keys ...[]byte) (int64, error) {
+	n, err := db.sStoreGeneric(dstKey, DiffType, keys...)
+	return n, err
+}
+
+// SKeyExists checks whether set existed or not.
+func (db *DB) SKeyExists(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+	sk := db.sEncodeSizeKey(key)
+	v, err := db.bucket.Get(sk)
+	if v != nil && err == nil {
+		return 1, nil
+	}
+	return 0, err
+}
+
+func (db *DB) sInterGeneric(keys ...[]byte) ([][]byte, error) {
+	destMap := make(map[string]bool)
+
+	members, err := db.SMembers(keys[0])
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range members {
+		destMap[hack.String(m)] = true
+	}
+
+	for _, key := range keys[1:] {
+		if err := checkKeySize(key); err != nil {
+			return nil, err
+		}
+
+		members, err := db.SMembers(key)
+		if err != nil {
+			return nil, err
+		} else if len(members) == 0 {
+			return nil, err
+		}
+
+		tempMap := make(map[string]bool)
+		for _, member := range members {
+			if err := checkKeySize(member); err != nil {
+				return nil, err
+			}
+			if _, ok := destMap[hack.String(member)]; ok {
+				tempMap[hack.String(member)] = true //mark this item as selected
+			}
+		}
+		destMap = tempMap //reduce the size of the result set
+		if len(destMap) == 0 {
+			return nil, nil
+		}
+	}
+
+	slice := make([][]byte, len(destMap))
+	idx := 0
+	for k, v := range destMap {
+		if !v {
+			continue
+		}
+
+		slice[idx] = []byte(k)
+		idx++
+	}
+
+	return slice, nil
+
+}
+
+// SInter intersects the sets.
+func (db *DB) SInter(keys ...[]byte) ([][]byte, error) {
+	v, err := db.sInterGeneric(keys...)
+	return v, err
+
+}
+
+// SInterStore intersects the sets and stores to dest set.
+func (db *DB) SInterStore(dstKey []byte, keys ...[]byte) (int64, error) {
+	n, err := db.sStoreGeneric(dstKey, InterType, keys...)
+	return n, err
+}
+
+// SIsMember checks member in set.
+func (db *DB) SIsMember(key []byte, member []byte) (int64, error) {
+	ek := db.sEncodeSetKey(key, member)
+
+	var n int64 = 1
+	if v, err := db.bucket.Get(ek); err != nil {
+		return 0, err
+	} else if v == nil {
+		n = 0
+	}
+	return n, nil
+}
+
+// SMembers gets members of set.
+func (db *DB) SMembers(key []byte) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.sEncodeStartKey(key)
+	stop := db.sEncodeStopKey(key)
+
+	v := make([][]byte, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	defer it.Close()
+
+	for ; it.Valid(); it.Next() {
+		_, m, err := db.sDecodeSetKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		v = append(v, m)
+	}
+
+	return v, nil
+}
+
+// SRem removes the members of set.
+func (db *DB) SRem(key []byte, args ...[]byte) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var ek []byte
+	var v []byte
+	var err error
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	var num int64
+	for i := 0; i < len(args); i++ {
+		if err := checkSetKMSize(key, args[i]); err != nil {
+			return 0, err
+		}
+
+		ek = db.sEncodeSetKey(key, args[i])
+
+		v = it.RawFind(ek)
+		if v == nil {
+			continue
+		} else {
+			num++
+			t.Delete(ek)
+		}
+	}
+
+	if _, err = db.sIncrSize(key, -num); err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return num, err
+
+}
+
+func (db *DB) sUnionGeneric(keys ...[]byte) ([][]byte, error) {
+	dstMap := make(map[string]bool)
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return nil, err
+		}
+
+		members, err := db.SMembers(key)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, member := range members {
+			dstMap[hack.String(member)] = true
+		}
+	}
+
+	slice := make([][]byte, len(dstMap))
+	idx := 0
+	for k, v := range dstMap {
+		if !v {
+			continue
+		}
+		slice[idx] = []byte(k)
+		idx++
+	}
+
+	return slice, nil
+}
+
+// SUnion unions the sets.
+func (db *DB) SUnion(keys ...[]byte) ([][]byte, error) {
+	v, err := db.sUnionGeneric(keys...)
+	return v, err
+}
+
+// SUnionStore unions the sets and stores to the dest set.
+func (db *DB) SUnionStore(dstKey []byte, keys ...[]byte) (int64, error) {
+	n, err := db.sStoreGeneric(dstKey, UnionType, keys...)
+	return n, err
+}
+
+func (db *DB) sStoreGeneric(dstKey []byte, optType byte, keys ...[]byte) (int64, error) {
+	if err := checkKeySize(dstKey); err != nil {
+		return 0, err
+	}
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	db.sDelete(t, dstKey)
+
+	var err error
+	var ek []byte
+	var v [][]byte
+
+	switch optType {
+	case UnionType:
+		v, err = db.sUnionGeneric(keys...)
+	case DiffType:
+		v, err = db.sDiffGeneric(keys...)
+	case InterType:
+		v, err = db.sInterGeneric(keys...)
+	}
+
+	if err != nil {
+		return 0, err
+	}
+
+	for _, m := range v {
+		if err := checkSetKMSize(dstKey, m); err != nil {
+			return 0, err
+		}
+
+		ek = db.sEncodeSetKey(dstKey, m)
+
+		if _, err := db.bucket.Get(ek); err != nil {
+			return 0, err
+		}
+
+		t.Put(ek, nil)
+	}
+
+	var n = int64(len(v))
+	sk := db.sEncodeSizeKey(dstKey)
+	t.Put(sk, PutInt64(n))
+
+	if err = t.Commit(); err != nil {
+		return 0, err
+	}
+	return n, nil
+}
+
+// SClear clears the set.
+func (db *DB) SClear(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	num := db.sDelete(t, key)
+	db.rmExpire(t, SetType, key)
+
+	err := t.Commit()
+	return num, err
+}
+
+// SMclear clears multi sets.
+func (db *DB) SMclear(keys ...[]byte) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return 0, err
+		}
+
+		db.sDelete(t, key)
+		db.rmExpire(t, SetType, key)
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+// SExpire expries the set.
+func (db *DB) SExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.sExpireAt(key, time.Now().Unix()+duration)
+
+}
+
+// SExpireAt expires the set at when.
+func (db *DB) SExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.sExpireAt(key, when)
+
+}
+
+// STTL gets the TTL of set.
+func (db *DB) STTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(SetType, key)
+}
+
+// SPersist removes the TTL of set.
+func (db *DB) SPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, SetType, key)
+	if err != nil {
+		return 0, err
+	}
+	err = t.Commit()
+	return n, err
+}

vendor/github.com/siddontang/ledisdb/ledis/t_ttl.go

@@ -0,0 +1,217 @@
+package ledis
+
+import (
+	"encoding/binary"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/siddontang/ledisdb/store"
+)
+
+var (
+	errExpMetaKey = errors.New("invalid expire meta key")
+	errExpTimeKey = errors.New("invalid expire time key")
+)
+
+type onExpired func(*batch, []byte) int64
+
+type ttlChecker struct {
+	sync.Mutex
+	db  *DB
+	txs []*batch
+	cbs []onExpired
+
+	//next check time
+	nc int64
+}
+
+var errExpType = errors.New("invalid expire type")
+
+func (db *DB) expEncodeTimeKey(dataType byte, key []byte, when int64) []byte {
+	buf := make([]byte, len(key)+10+len(db.indexVarBuf))
+
+	pos := copy(buf, db.indexVarBuf)
+
+	buf[pos] = ExpTimeType
+	pos++
+
+	binary.BigEndian.PutUint64(buf[pos:], uint64(when))
+	pos += 8
+
+	buf[pos] = dataType
+	pos++
+
+	copy(buf[pos:], key)
+
+	return buf
+}
+
+func (db *DB) expEncodeMetaKey(dataType byte, key []byte) []byte {
+	buf := make([]byte, len(key)+2+len(db.indexVarBuf))
+
+	pos := copy(buf, db.indexVarBuf)
+	buf[pos] = ExpMetaType
+	pos++
+	buf[pos] = dataType
+	pos++
+
+	copy(buf[pos:], key)
+
+	return buf
+}
+
+func (db *DB) expDecodeMetaKey(mk []byte) (byte, []byte, error) {
+	pos, err := db.checkKeyIndex(mk)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	if pos+2 > len(mk) || mk[pos] != ExpMetaType {
+		return 0, nil, errExpMetaKey
+	}
+
+	return mk[pos+1], mk[pos+2:], nil
+}
+
+func (db *DB) expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) {
+	pos, err := db.checkKeyIndex(tk)
+	if err != nil {
+		return 0, nil, 0, err
+	}
+
+	if pos+10 > len(tk) || tk[pos] != ExpTimeType {
+		return 0, nil, 0, errExpTimeKey
+	}
+
+	return tk[pos+9], tk[pos+10:], int64(binary.BigEndian.Uint64(tk[pos+1:])), nil
+}
+
+func (db *DB) expire(t *batch, dataType byte, key []byte, duration int64) {
+	db.expireAt(t, dataType, key, time.Now().Unix()+duration)
+}
+
+func (db *DB) expireAt(t *batch, dataType byte, key []byte, when int64) {
+	mk := db.expEncodeMetaKey(dataType, key)
+	tk := db.expEncodeTimeKey(dataType, key, when)
+
+	t.Put(tk, mk)
+	t.Put(mk, PutInt64(when))
+
+	db.ttlChecker.setNextCheckTime(when, false)
+}
+
+func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) {
+	mk := db.expEncodeMetaKey(dataType, key)
+
+	if t, err = Int64(db.bucket.Get(mk)); err != nil || t == 0 {
+		t = -1
+	} else {
+		t -= time.Now().Unix()
+		if t <= 0 {
+			t = -1
+		}
+		// if t == -1 : to remove ????
+	}
+
+	return t, err
+}
+
+func (db *DB) rmExpire(t *batch, dataType byte, key []byte) (int64, error) {
+	mk := db.expEncodeMetaKey(dataType, key)
+	v, err := db.bucket.Get(mk)
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	}
+
+	when, err2 := Int64(v, nil)
+	if err2 != nil {
+		return 0, err2
+	}
+
+	tk := db.expEncodeTimeKey(dataType, key, when)
+	t.Delete(mk)
+	t.Delete(tk)
+	return 1, nil
+}
+
+func (c *ttlChecker) register(dataType byte, t *batch, f onExpired) {
+	c.txs[dataType] = t
+	c.cbs[dataType] = f
+}
+
+func (c *ttlChecker) setNextCheckTime(when int64, force bool) {
+	c.Lock()
+	if force {
+		c.nc = when
+	} else if c.nc > when {
+		c.nc = when
+	}
+	c.Unlock()
+}
+
+func (c *ttlChecker) check() {
+	now := time.Now().Unix()
+
+	c.Lock()
+	nc := c.nc
+	c.Unlock()
+
+	if now < nc {
+		return
+	}
+
+	nc = now + 3600
+
+	db := c.db
+	dbGet := db.bucket.Get
+
+	minKey := db.expEncodeTimeKey(NoneType, nil, 0)
+	maxKey := db.expEncodeTimeKey(maxDataType, nil, nc)
+
+	it := db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		tk := it.RawKey()
+		mk := it.RawValue()
+
+		dt, k, nt, err := db.expDecodeTimeKey(tk)
+		if err != nil {
+			continue
+		}
+
+		if nt > now {
+			//the next ttl check time is nt!
+			nc = nt
+			break
+		}
+
+		t := c.txs[dt]
+		cb := c.cbs[dt]
+		if tk == nil || cb == nil {
+			continue
+		}
+
+		t.Lock()
+
+		if exp, err := Int64(dbGet(mk)); err == nil {
+			// check expire again
+			if exp <= now {
+				cb(t, k)
+				t.Delete(tk)
+				t.Delete(mk)
+
+				t.Commit()
+			}
+
+		}
+
+		t.Unlock()
+	}
+	it.Close()
+
+	c.setNextCheckTime(nc, true)
+
+	return
+}

vendor/github.com/siddontang/ledisdb/ledis/util.go

@@ -0,0 +1,103 @@
+package ledis
+
+import (
+	"encoding/binary"
+	"errors"
+	"strconv"
+
+	"github.com/siddontang/go/hack"
+)
+
+var errIntNumber = errors.New("invalid integer")
+
+/*
+	Below I forget why I use little endian to store int.
+	Maybe I was foolish at that time.
+*/
+
+// Int64 gets 64 integer with the little endian format.
+func Int64(v []byte, err error) (int64, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil || len(v) == 0 {
+		return 0, nil
+	} else if len(v) != 8 {
+		return 0, errIntNumber
+	}
+
+	return int64(binary.LittleEndian.Uint64(v)), nil
+}
+
+// Uint64 gets unsigned 64 integer.
+func Uint64(v []byte, err error) (uint64, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil || len(v) == 0 {
+		return 0, nil
+	} else if len(v) != 8 {
+		return 0, errIntNumber
+	}
+
+	return binary.LittleEndian.Uint64(v), nil
+}
+
+// PutInt64 puts the 64 integer.
+func PutInt64(v int64) []byte {
+	b := make([]byte, 8)
+	binary.LittleEndian.PutUint64(b, uint64(v))
+	return b
+}
+
+// StrInt64 gets the 64 integer with string format.
+func StrInt64(v []byte, err error) (int64, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else {
+		return strconv.ParseInt(hack.String(v), 10, 64)
+	}
+}
+
+// StrUint64 gets the unsigned 64 integer with string format.
+func StrUint64(v []byte, err error) (uint64, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else {
+		return strconv.ParseUint(hack.String(v), 10, 64)
+	}
+}
+
+// StrInt32 gets the 32 integer with string format.
+func StrInt32(v []byte, err error) (int32, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else {
+		res, err := strconv.ParseInt(hack.String(v), 10, 32)
+		return int32(res), err
+	}
+}
+
+// StrInt8 ets the 8 integer with string format.
+func StrInt8(v []byte, err error) (int8, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else {
+		res, err := strconv.ParseInt(hack.String(v), 10, 8)
+		return int8(res), err
+	}
+}
+
+// AsyncNotify notices the channel.
+func AsyncNotify(ch chan struct{}) {
+	select {
+	case ch <- struct{}{}:
+	default:
+	}
+}

vendor/github.com/siddontang/ledisdb/rpl/file_io.go

@@ -0,0 +1,363 @@
+package rpl
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/edsrzf/mmap-go"
+	"github.com/siddontang/go/log"
+)
+
+//like leveldb or rocksdb file interface, haha!
+
+type writeFile interface {
+	Sync() error
+	Write(b []byte) (n int, err error)
+	Close() error
+	ReadAt(buf []byte, offset int64) (int, error)
+	Truncate(size int64) error
+	SetOffset(o int64)
+	Name() string
+	Size() int
+	Offset() int64
+}
+
+type readFile interface {
+	ReadAt(buf []byte, offset int64) (int, error)
+	Close() error
+	Size() int
+	Name() string
+}
+
+type rawWriteFile struct {
+	writeFile
+	f      *os.File
+	offset int64
+	name   string
+}
+
+func newRawWriteFile(name string, size int64) (writeFile, error) {
+	m := new(rawWriteFile)
+	var err error
+
+	m.name = name
+
+	m.f, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func (m *rawWriteFile) Close() error {
+	if err := m.f.Truncate(m.offset); err != nil {
+		return fmt.Errorf("close truncate %s error %s", m.name, err.Error())
+	}
+
+	if err := m.f.Close(); err != nil {
+		return fmt.Errorf("close %s error %s", m.name, err.Error())
+	}
+
+	return nil
+}
+
+func (m *rawWriteFile) Sync() error {
+	return m.f.Sync()
+}
+
+func (m *rawWriteFile) Write(b []byte) (n int, err error) {
+	n, err = m.f.WriteAt(b, m.offset)
+	if err != nil {
+		return
+	} else if n != len(b) {
+		err = io.ErrShortWrite
+		return
+	}
+
+	m.offset += int64(n)
+	return
+}
+
+func (m *rawWriteFile) ReadAt(buf []byte, offset int64) (int, error) {
+	return m.f.ReadAt(buf, offset)
+}
+
+func (m *rawWriteFile) Truncate(size int64) error {
+	var err error
+	if err = m.f.Truncate(size); err != nil {
+		return err
+	}
+
+	if m.offset > size {
+		m.offset = size
+	}
+	return nil
+}
+
+func (m *rawWriteFile) SetOffset(o int64) {
+	m.offset = o
+}
+
+func (m *rawWriteFile) Offset() int64 {
+	return m.offset
+}
+
+func (m *rawWriteFile) Name() string {
+	return m.name
+}
+
+func (m *rawWriteFile) Size() int {
+	st, _ := m.f.Stat()
+	return int(st.Size())
+}
+
+type rawReadFile struct {
+	readFile
+
+	f    *os.File
+	name string
+}
+
+func newRawReadFile(name string) (readFile, error) {
+	m := new(rawReadFile)
+
+	var err error
+	m.f, err = os.Open(name)
+	m.name = name
+
+	if err != nil {
+		return nil, err
+	}
+
+	return m, err
+}
+
+func (m *rawReadFile) Close() error {
+	return m.f.Close()
+}
+
+func (m *rawReadFile) Size() int {
+	st, _ := m.f.Stat()
+	return int(st.Size())
+}
+
+func (m *rawReadFile) ReadAt(b []byte, offset int64) (int, error) {
+	return m.f.ReadAt(b, offset)
+}
+
+func (m *rawReadFile) Name() string {
+	return m.name
+}
+
+/////////////////////////////////////////////////
+
+type mmapWriteFile struct {
+	writeFile
+
+	f      *os.File
+	m      mmap.MMap
+	name   string
+	size   int64
+	offset int64
+}
+
+func newMmapWriteFile(name string, size int64) (writeFile, error) {
+	m := new(mmapWriteFile)
+
+	m.name = name
+
+	var err error
+
+	m.f, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	if size == 0 {
+		st, _ := m.f.Stat()
+		size = st.Size()
+	}
+
+	if err = m.f.Truncate(size); err != nil {
+		return nil, err
+	}
+
+	if m.m, err = mmap.Map(m.f, mmap.RDWR, 0); err != nil {
+		return nil, err
+	}
+
+	m.size = size
+	m.offset = 0
+	return m, nil
+}
+
+func (m *mmapWriteFile) Size() int {
+	return int(m.size)
+}
+
+func (m *mmapWriteFile) Sync() error {
+	return m.m.Flush()
+}
+
+func (m *mmapWriteFile) Close() error {
+	if err := m.m.Unmap(); err != nil {
+		return fmt.Errorf("unmap %s error %s", m.name, err.Error())
+	}
+
+	if err := m.f.Truncate(m.offset); err != nil {
+		return fmt.Errorf("close truncate %s error %s", m.name, err.Error())
+	}
+
+	if err := m.f.Close(); err != nil {
+		return fmt.Errorf("close %s error %s", m.name, err.Error())
+	}
+
+	return nil
+}
+
+func (m *mmapWriteFile) Write(b []byte) (n int, err error) {
+	extra := int64(len(b)) - (m.size - m.offset)
+	if extra > 0 {
+		newSize := m.size + extra + m.size/10
+		if err = m.Truncate(newSize); err != nil {
+			return
+		}
+		m.size = newSize
+	}
+
+	n = copy(m.m[m.offset:], b)
+	if n != len(b) {
+		return 0, io.ErrShortWrite
+	}
+
+	m.offset += int64(len(b))
+	return len(b), nil
+}
+
+func (m *mmapWriteFile) ReadAt(buf []byte, offset int64) (int, error) {
+	if offset > m.offset {
+		return 0, fmt.Errorf("invalid offset %d", offset)
+	}
+
+	n := copy(buf, m.m[offset:m.offset])
+	if n != len(buf) {
+		return n, io.ErrUnexpectedEOF
+	}
+
+	return n, nil
+}
+
+func (m *mmapWriteFile) Truncate(size int64) error {
+	var err error
+	if err = m.m.Unmap(); err != nil {
+		return err
+	}
+
+	if err = m.f.Truncate(size); err != nil {
+		return err
+	}
+
+	if m.m, err = mmap.Map(m.f, mmap.RDWR, 0); err != nil {
+		return err
+	}
+
+	m.size = size
+	if m.offset > m.size {
+		m.offset = m.size
+	}
+	return nil
+}
+
+func (m *mmapWriteFile) SetOffset(o int64) {
+	m.offset = o
+}
+
+func (m *mmapWriteFile) Offset() int64 {
+	return m.offset
+}
+
+func (m *mmapWriteFile) Name() string {
+	return m.name
+}
+
+type mmapReadFile struct {
+	readFile
+
+	f    *os.File
+	m    mmap.MMap
+	name string
+}
+
+func newMmapReadFile(name string) (readFile, error) {
+	m := new(mmapReadFile)
+
+	m.name = name
+
+	var err error
+	m.f, err = os.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	m.m, err = mmap.Map(m.f, mmap.RDONLY, 0)
+	return m, err
+}
+
+func (m *mmapReadFile) ReadAt(buf []byte, offset int64) (int, error) {
+	if int64(offset) > int64(len(m.m)) {
+		return 0, fmt.Errorf("invalid offset %d", offset)
+	}
+
+	n := copy(buf, m.m[offset:])
+	if n != len(buf) {
+		return n, io.ErrUnexpectedEOF
+	}
+
+	return n, nil
+}
+
+func (m *mmapReadFile) Close() error {
+	if m.m != nil {
+		if err := m.m.Unmap(); err != nil {
+			log.Errorf("unmap %s error %s", m.name, err.Error())
+		}
+		m.m = nil
+	}
+
+	if m.f != nil {
+		if err := m.f.Close(); err != nil {
+			log.Errorf("close %s error %s", m.name, err.Error())
+		}
+		m.f = nil
+	}
+
+	return nil
+}
+
+func (m *mmapReadFile) Size() int {
+	return len(m.m)
+}
+
+func (m *mmapReadFile) Name() string {
+	return m.name
+}
+
+/////////////////////////////////////
+
+func newWriteFile(useMmap bool, name string, size int64) (writeFile, error) {
+	if useMmap {
+		return newMmapWriteFile(name, size)
+	} else {
+		return newRawWriteFile(name, size)
+	}
+}
+
+func newReadFile(useMmap bool, name string) (readFile, error) {
+	if useMmap {
+		return newMmapReadFile(name)
+	} else {
+		return newRawReadFile(name)
+	}
+}

vendor/github.com/siddontang/ledisdb/rpl/file_store.go

@@ -0,0 +1,416 @@
+package rpl
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/siddontang/go/log"
+	"github.com/siddontang/go/num"
+	"github.com/siddontang/ledisdb/config"
+)
+
+const (
+	defaultMaxLogFileSize = int64(256 * 1024 * 1024)
+
+	maxLogFileSize = int64(1024 * 1024 * 1024)
+
+	defaultLogNumInFile = int64(1024 * 1024)
+)
+
+/*
+	File Store:
+	00000001.data
+	00000001.meta
+	00000002.data
+	00000002.meta
+
+	data: log1 data | log2 data | magic data
+
+	if data has no magic data, it means that we don't close replication gracefully.
+	so we must repair the log data
+	log data: id (bigendian uint64), create time (bigendian uint32), compression (byte), data len(bigendian uint32), data
+	split data = log0 data + [padding 0] -> file % pagesize() == 0
+
+	meta: log1 offset | log2 offset
+	log offset: bigendian uint32 | bigendian uint32
+
+	//sha1 of github.com/siddontang/ledisdb 20 bytes
+	magic data = "\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17"
+
+	we must guarantee that the log id is monotonic increment strictly.
+	if log1's id is 1, log2 must be 2
+*/
+
+type FileStore struct {
+	LogStore
+
+	cfg *config.Config
+
+	base string
+
+	rm sync.RWMutex
+	wm sync.Mutex
+
+	rs tableReaders
+	w  *tableWriter
+
+	quit chan struct{}
+}
+
+func NewFileStore(base string, cfg *config.Config) (*FileStore, error) {
+	s := new(FileStore)
+
+	s.quit = make(chan struct{})
+
+	var err error
+
+	if err = os.MkdirAll(base, 0755); err != nil {
+		return nil, err
+	}
+
+	s.base = base
+
+	if cfg.Replication.MaxLogFileSize == 0 {
+		cfg.Replication.MaxLogFileSize = defaultMaxLogFileSize
+	}
+
+	cfg.Replication.MaxLogFileSize = num.MinInt64(cfg.Replication.MaxLogFileSize, maxLogFileSize)
+
+	s.cfg = cfg
+
+	if err = s.load(); err != nil {
+		return nil, err
+	}
+
+	index := int64(1)
+	if len(s.rs) != 0 {
+		index = s.rs[len(s.rs)-1].index + 1
+	}
+
+	s.w = newTableWriter(s.base, index, cfg.Replication.MaxLogFileSize, cfg.Replication.UseMmap)
+	s.w.SetSyncType(cfg.Replication.SyncLog)
+
+	go s.checkTableReaders()
+
+	return s, nil
+}
+
+func (s *FileStore) GetLog(id uint64, l *Log) error {
+	//first search in table writer
+	if err := s.w.GetLog(id, l); err == nil {
+		return nil
+	} else if err != ErrLogNotFound {
+		return err
+	}
+
+	s.rm.RLock()
+	t := s.rs.Search(id)
+
+	if t == nil {
+		s.rm.RUnlock()
+
+		return ErrLogNotFound
+	}
+
+	err := t.GetLog(id, l)
+	s.rm.RUnlock()
+
+	return err
+}
+
+func (s *FileStore) FirstID() (uint64, error) {
+	id := uint64(0)
+
+	s.rm.RLock()
+	if len(s.rs) > 0 {
+		id = s.rs[0].first
+	} else {
+		id = 0
+	}
+	s.rm.RUnlock()
+
+	if id > 0 {
+		return id, nil
+	}
+
+	//if id = 0,
+
+	return s.w.First(), nil
+}
+
+func (s *FileStore) LastID() (uint64, error) {
+	id := s.w.Last()
+	if id > 0 {
+		return id, nil
+	}
+
+	//if table writer has no last id, we may find in the last table reader
+
+	s.rm.RLock()
+	if len(s.rs) > 0 {
+		id = s.rs[len(s.rs)-1].last
+	}
+	s.rm.RUnlock()
+
+	return id, nil
+}
+
+func (s *FileStore) StoreLog(l *Log) error {
+	s.wm.Lock()
+	err := s.storeLog(l)
+	s.wm.Unlock()
+	return err
+}
+
+func (s *FileStore) storeLog(l *Log) error {
+	err := s.w.StoreLog(l)
+	if err == nil {
+		return nil
+	} else if err != errTableNeedFlush {
+		return err
+	}
+
+	var r *tableReader
+	r, err = s.w.Flush()
+
+	if err != nil {
+		log.Fatalf("write table flush error %s, can not store!!!", err.Error())
+
+		s.w.Close()
+
+		return err
+	}
+
+	s.rm.Lock()
+	s.rs = append(s.rs, r)
+	s.rm.Unlock()
+
+	err = s.w.StoreLog(l)
+
+	return err
+}
+
+func (s *FileStore) PurgeExpired(n int64) error {
+	s.rm.Lock()
+
+	var purges []*tableReader
+
+	t := uint32(time.Now().Unix() - int64(n))
+
+	for i, r := range s.rs {
+		if r.lastTime > t {
+			purges = append([]*tableReader{}, s.rs[0:i]...)
+			n := copy(s.rs, s.rs[i:])
+			s.rs = s.rs[0:n]
+			break
+		}
+	}
+
+	s.rm.Unlock()
+
+	s.purgeTableReaders(purges)
+
+	return nil
+}
+
+func (s *FileStore) Sync() error {
+	return s.w.Sync()
+}
+
+func (s *FileStore) Clear() error {
+	s.wm.Lock()
+	s.rm.Lock()
+
+	defer func() {
+		s.rm.Unlock()
+		s.wm.Unlock()
+	}()
+
+	s.w.Close()
+
+	for i := range s.rs {
+		s.rs[i].Close()
+	}
+
+	s.rs = tableReaders{}
+
+	if err := os.RemoveAll(s.base); err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(s.base, 0755); err != nil {
+		return err
+	}
+
+	s.w = newTableWriter(s.base, 1, s.cfg.Replication.MaxLogFileSize, s.cfg.Replication.UseMmap)
+
+	return nil
+}
+
+func (s *FileStore) Close() error {
+	close(s.quit)
+
+	s.wm.Lock()
+	s.rm.Lock()
+
+	if r, err := s.w.Flush(); err != nil {
+		if err != errNilHandler {
+			log.Errorf("close err: %s", err.Error())
+		}
+	} else {
+		r.Close()
+		s.w.Close()
+	}
+
+	for i := range s.rs {
+		s.rs[i].Close()
+	}
+
+	s.rs = tableReaders{}
+
+	s.rm.Unlock()
+	s.wm.Unlock()
+
+	return nil
+}
+
+func (s *FileStore) checkTableReaders() {
+	t := time.NewTicker(60 * time.Second)
+	defer t.Stop()
+	for {
+		select {
+		case <-t.C:
+			s.rm.Lock()
+
+			for _, r := range s.rs {
+				if !r.Keepalived() {
+					r.Close()
+				}
+			}
+
+			purges := []*tableReader{}
+			maxNum := s.cfg.Replication.MaxLogFileNum
+			num := len(s.rs)
+			if num > maxNum {
+				purges = s.rs[:num-maxNum]
+				s.rs = s.rs[num-maxNum:]
+			}
+
+			s.rm.Unlock()
+
+			s.purgeTableReaders(purges)
+
+		case <-s.quit:
+			return
+		}
+	}
+}
+
+func (s *FileStore) purgeTableReaders(purges []*tableReader) {
+	for _, r := range purges {
+		dataName := fmtTableDataName(r.base, r.index)
+		metaName := fmtTableMetaName(r.base, r.index)
+		r.Close()
+		if err := os.Remove(dataName); err != nil {
+			log.Errorf("purge table data %s err: %s", dataName, err.Error())
+		}
+		if err := os.Remove(metaName); err != nil {
+			log.Errorf("purge table meta %s err: %s", metaName, err.Error())
+		}
+
+	}
+}
+
+func (s *FileStore) load() error {
+	fs, err := ioutil.ReadDir(s.base)
+	if err != nil {
+		return err
+	}
+
+	s.rs = make(tableReaders, 0, len(fs))
+
+	var r *tableReader
+	var index int64
+	for _, f := range fs {
+		if _, err := fmt.Sscanf(f.Name(), "%08d.data", &index); err == nil {
+			if r, err = newTableReader(s.base, index, s.cfg.Replication.UseMmap); err != nil {
+				log.Errorf("load table %s err: %s", f.Name(), err.Error())
+			} else {
+				s.rs = append(s.rs, r)
+			}
+		}
+	}
+
+	if err := s.rs.check(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type tableReaders []*tableReader
+
+func (ts tableReaders) Len() int {
+	return len(ts)
+}
+
+func (ts tableReaders) Swap(i, j int) {
+	ts[i], ts[j] = ts[j], ts[i]
+}
+
+func (ts tableReaders) Less(i, j int) bool {
+	return ts[i].first < ts[j].first
+}
+
+func (ts tableReaders) Search(id uint64) *tableReader {
+	i, j := 0, len(ts)-1
+
+	for i <= j {
+		h := i + (j-i)/2
+
+		if ts[h].first <= id && id <= ts[h].last {
+			return ts[h]
+		} else if ts[h].last < id {
+			i = h + 1
+		} else {
+			j = h - 1
+		}
+	}
+
+	return nil
+}
+
+func (ts tableReaders) check() error {
+	if len(ts) == 0 {
+		return nil
+	}
+
+	sort.Sort(ts)
+
+	first := ts[0].first
+	last := ts[0].last
+	index := ts[0].index
+
+	if first == 0 || first > last {
+		return fmt.Errorf("invalid log in table %s", ts[0])
+	}
+
+	for i := 1; i < len(ts); i++ {
+		if ts[i].first <= last {
+			return fmt.Errorf("invalid first log id %d in table %s", ts[i].first, ts[i])
+		}
+
+		if ts[i].index <= index {
+			return fmt.Errorf("invalid index %d in table %s", ts[i].index, ts[i])
+		}
+
+		first = ts[i].first
+		last = ts[i].last
+		index = ts[i].index
+	}
+	return nil
+}

vendor/github.com/siddontang/ledisdb/rpl/file_table.go

@@ -0,0 +1,571 @@
+package rpl
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/siddontang/go/log"
+	"github.com/siddontang/go/sync2"
+)
+
+var (
+	magic             = []byte("\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17")
+	errTableNeedFlush = errors.New("write table need flush")
+	errNilHandler     = errors.New("nil write handler")
+)
+
+const tableReaderKeepaliveInterval int64 = 30
+
+func fmtTableDataName(base string, index int64) string {
+	return path.Join(base, fmt.Sprintf("%08d.data", index))
+}
+
+func fmtTableMetaName(base string, index int64) string {
+	return path.Join(base, fmt.Sprintf("%08d.meta", index))
+}
+
+type tableReader struct {
+	sync.Mutex
+
+	base  string
+	index int64
+
+	data readFile
+	meta readFile
+
+	first uint64
+	last  uint64
+
+	lastTime uint32
+
+	lastReadTime sync2.AtomicInt64
+
+	useMmap bool
+}
+
+func newTableReader(base string, index int64, useMmap bool) (*tableReader, error) {
+	if index <= 0 {
+		return nil, fmt.Errorf("invalid index %d", index)
+	}
+	t := new(tableReader)
+	t.base = base
+	t.index = index
+
+	t.useMmap = useMmap
+
+	var err error
+
+	if err = t.check(); err != nil {
+		log.Errorf("check %d error: %s, try to repair", t.index, err.Error())
+
+		if err = t.repair(); err != nil {
+			log.Errorf("repair %d error: %s", t.index, err.Error())
+			return nil, err
+		}
+	}
+
+	t.close()
+
+	return t, nil
+}
+
+func (t *tableReader) String() string {
+	return fmt.Sprintf("%d", t.index)
+}
+
+func (t *tableReader) Close() {
+	t.Lock()
+
+	t.close()
+
+	t.Unlock()
+}
+
+func (t *tableReader) close() {
+	if t.data != nil {
+		t.data.Close()
+		t.data = nil
+	}
+
+	if t.meta != nil {
+		t.meta.Close()
+		t.meta = nil
+	}
+}
+
+func (t *tableReader) Keepalived() bool {
+	l := t.lastReadTime.Get()
+	if l > 0 && time.Now().Unix()-l > tableReaderKeepaliveInterval {
+		return false
+	}
+
+	return true
+}
+
+func (t *tableReader) getLogPos(index int) (uint32, error) {
+	var buf [4]byte
+	if _, err := t.meta.ReadAt(buf[0:4], int64(index)*4); err != nil {
+		return 0, err
+	}
+
+	return binary.BigEndian.Uint32(buf[0:4]), nil
+}
+
+func (t *tableReader) checkData() error {
+	var err error
+	//check will use raw file mode
+	if t.data, err = newReadFile(false, fmtTableDataName(t.base, t.index)); err != nil {
+		return err
+	}
+
+	if t.data.Size() < len(magic) {
+		return fmt.Errorf("data file %s size %d too short", t.data.Name(), t.data.Size())
+	}
+
+	buf := make([]byte, len(magic))
+	if _, err := t.data.ReadAt(buf, int64(t.data.Size()-len(magic))); err != nil {
+		return err
+	}
+
+	if !bytes.Equal(magic, buf) {
+		return fmt.Errorf("data file %s invalid magic data %q", t.data.Name(), buf)
+	}
+
+	return nil
+}
+
+func (t *tableReader) checkMeta() error {
+	var err error
+	//check will use raw file mode
+	if t.meta, err = newReadFile(false, fmtTableMetaName(t.base, t.index)); err != nil {
+		return err
+	}
+
+	if t.meta.Size()%4 != 0 || t.meta.Size() == 0 {
+		return fmt.Errorf("meta file %s invalid offset len %d, must 4 multiple and not 0", t.meta.Name(), t.meta.Size())
+	}
+
+	return nil
+}
+
+func (t *tableReader) check() error {
+	var err error
+
+	if err := t.checkMeta(); err != nil {
+		return err
+	}
+
+	if err := t.checkData(); err != nil {
+		return err
+	}
+
+	firstLogPos, _ := t.getLogPos(0)
+	lastLogPos, _ := t.getLogPos(t.meta.Size()/4 - 1)
+
+	if firstLogPos != 0 {
+		return fmt.Errorf("invalid first log pos %d, must 0", firstLogPos)
+	}
+
+	var l Log
+	if _, err = t.decodeLogHead(&l, t.data, int64(firstLogPos)); err != nil {
+		return fmt.Errorf("decode first log err %s", err.Error())
+	}
+
+	t.first = l.ID
+	var n int64
+	if n, err = t.decodeLogHead(&l, t.data, int64(lastLogPos)); err != nil {
+		return fmt.Errorf("decode last log err %s", err.Error())
+	} else if n+int64(len(magic)) != int64(t.data.Size()) {
+		return fmt.Errorf("extra log data at offset %d", n)
+	}
+
+	t.last = l.ID
+	t.lastTime = l.CreateTime
+
+	if t.first > t.last {
+		return fmt.Errorf("invalid log table first %d > last %d", t.first, t.last)
+	} else if (t.last - t.first + 1) != uint64(t.meta.Size()/4) {
+		return fmt.Errorf("invalid log table, first %d, last %d, and log num %d", t.first, t.last, t.meta.Size()/4)
+	}
+
+	return nil
+}
+
+func (t *tableReader) repair() error {
+	t.close()
+
+	var err error
+	var data writeFile
+	var meta writeFile
+
+	//repair will use raw file mode
+	data, err = newWriteFile(false, fmtTableDataName(t.base, t.index), 0)
+	data.SetOffset(int64(data.Size()))
+
+	meta, err = newWriteFile(false, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4))
+
+	var l Log
+	var pos int64 = 0
+	var nextPos int64 = 0
+	b := make([]byte, 4)
+
+	t.first = 0
+	t.last = 0
+
+	for {
+		nextPos, err = t.decodeLogHead(&l, data, pos)
+		if err != nil {
+			//if error, we may lost all logs from pos
+			log.Errorf("%s may lost logs from %d", data.Name(), pos)
+			break
+		}
+
+		if l.ID == 0 {
+			log.Errorf("%s may lost logs from %d, invalid log 0", data.Name(), pos)
+			break
+		}
+
+		if t.first == 0 {
+			t.first = l.ID
+		}
+
+		if t.last == 0 {
+			t.last = l.ID
+		} else if l.ID <= t.last {
+			log.Errorf("%s may lost logs from %d, invalid logid %d", t.data.Name(), pos, l.ID)
+			break
+		}
+
+		t.last = l.ID
+		t.lastTime = l.CreateTime
+
+		binary.BigEndian.PutUint32(b, uint32(pos))
+		meta.Write(b)
+
+		pos = nextPos
+
+		t.lastTime = l.CreateTime
+	}
+
+	var e error
+	if err := meta.Close(); err != nil {
+		e = err
+	}
+
+	data.SetOffset(pos)
+
+	if _, err = data.Write(magic); err != nil {
+		log.Errorf("write magic error %s", err.Error())
+	}
+
+	if err = data.Close(); err != nil {
+		return err
+	}
+
+	return e
+}
+
+func (t *tableReader) decodeLogHead(l *Log, r io.ReaderAt, pos int64) (int64, error) {
+	dataLen, err := l.DecodeHeadAt(r, pos)
+	if err != nil {
+		return 0, err
+	}
+
+	return pos + int64(l.HeadSize()) + int64(dataLen), nil
+}
+
+func (t *tableReader) GetLog(id uint64, l *Log) error {
+	if id < t.first || id > t.last {
+		return ErrLogNotFound
+	}
+
+	t.lastReadTime.Set(time.Now().Unix())
+
+	t.Lock()
+
+	if err := t.openTable(); err != nil {
+		t.close()
+		t.Unlock()
+		return err
+	}
+	t.Unlock()
+
+	pos, err := t.getLogPos(int(id - t.first))
+	if err != nil {
+		return err
+	}
+
+	if err := l.DecodeAt(t.data, int64(pos)); err != nil {
+		return err
+	} else if l.ID != id {
+		return fmt.Errorf("invalid log id %d != %d", l.ID, id)
+	}
+
+	return nil
+}
+
+func (t *tableReader) openTable() error {
+	var err error
+	if t.data == nil {
+		if t.data, err = newReadFile(t.useMmap, fmtTableDataName(t.base, t.index)); err != nil {
+			return err
+		}
+	}
+
+	if t.meta == nil {
+		if t.meta, err = newReadFile(t.useMmap, fmtTableMetaName(t.base, t.index)); err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}
+
+type tableWriter struct {
+	sync.RWMutex
+
+	data writeFile
+	meta writeFile
+
+	base  string
+	index int64
+
+	first    uint64
+	last     uint64
+	lastTime uint32
+
+	maxLogSize int64
+
+	closed bool
+
+	syncType int
+
+	posBuf []byte
+
+	useMmap bool
+}
+
+func newTableWriter(base string, index int64, maxLogSize int64, useMmap bool) *tableWriter {
+	if index <= 0 {
+		panic(fmt.Errorf("invalid index %d", index))
+	}
+
+	t := new(tableWriter)
+
+	t.base = base
+	t.index = index
+
+	t.maxLogSize = maxLogSize
+
+	t.closed = false
+
+	t.posBuf = make([]byte, 4)
+
+	t.useMmap = useMmap
+
+	return t
+}
+
+func (t *tableWriter) String() string {
+	return fmt.Sprintf("%d", t.index)
+}
+
+func (t *tableWriter) SetMaxLogSize(s int64) {
+	t.maxLogSize = s
+}
+
+func (t *tableWriter) SetSyncType(tp int) {
+	t.syncType = tp
+}
+
+func (t *tableWriter) close() {
+	if t.meta != nil {
+		if err := t.meta.Close(); err != nil {
+			log.Fatalf("close log meta error %s", err.Error())
+		}
+		t.meta = nil
+	}
+
+	if t.data != nil {
+		if _, err := t.data.Write(magic); err != nil {
+			log.Fatalf("write magic error %s", err.Error())
+		}
+
+		if err := t.data.Close(); err != nil {
+			log.Fatalf("close log data error %s", err.Error())
+		}
+		t.data = nil
+	}
+}
+
+func (t *tableWriter) Close() {
+	t.Lock()
+	t.closed = true
+
+	t.close()
+	t.Unlock()
+}
+
+func (t *tableWriter) First() uint64 {
+	t.Lock()
+	id := t.first
+	t.Unlock()
+	return id
+}
+
+func (t *tableWriter) Last() uint64 {
+	t.Lock()
+	id := t.last
+	t.Unlock()
+	return id
+}
+
+func (t *tableWriter) Flush() (*tableReader, error) {
+	t.Lock()
+
+	if t.data == nil || t.meta == nil {
+		t.Unlock()
+		return nil, errNilHandler
+	}
+
+	tr := new(tableReader)
+	tr.base = t.base
+	tr.index = t.index
+
+	tr.first = t.first
+	tr.last = t.last
+	tr.lastTime = t.lastTime
+	tr.useMmap = t.useMmap
+
+	t.close()
+
+	t.first = 0
+	t.last = 0
+	t.index = t.index + 1
+
+	t.Unlock()
+
+	return tr, nil
+}
+
+func (t *tableWriter) StoreLog(l *Log) error {
+	t.Lock()
+	err := t.storeLog(l)
+	t.Unlock()
+
+	return err
+}
+
+func (t *tableWriter) openFile() error {
+	var err error
+	if t.data == nil {
+		if t.data, err = newWriteFile(t.useMmap, fmtTableDataName(t.base, t.index), t.maxLogSize+t.maxLogSize/10+int64(len(magic))); err != nil {
+			return err
+		}
+	}
+
+	if t.meta == nil {
+		if t.meta, err = newWriteFile(t.useMmap, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4)); err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+func (t *tableWriter) storeLog(l *Log) error {
+	if l.ID == 0 {
+		return ErrStoreLogID
+	}
+
+	if t.closed {
+		return fmt.Errorf("table writer is closed")
+	}
+
+	if t.last > 0 && l.ID != t.last+1 {
+		return ErrStoreLogID
+	}
+
+	if t.data != nil && t.data.Offset() > t.maxLogSize {
+		return errTableNeedFlush
+	}
+
+	var err error
+	if err = t.openFile(); err != nil {
+		return err
+	}
+
+	offsetPos := t.data.Offset()
+	if err = l.Encode(t.data); err != nil {
+		return err
+	}
+
+	binary.BigEndian.PutUint32(t.posBuf, uint32(offsetPos))
+	if _, err = t.meta.Write(t.posBuf); err != nil {
+		return err
+	}
+
+	if t.first == 0 {
+		t.first = l.ID
+	}
+
+	t.last = l.ID
+	t.lastTime = l.CreateTime
+
+	if t.syncType == 2 {
+		if err := t.data.Sync(); err != nil {
+			log.Errorf("sync table error %s", err.Error())
+		}
+	}
+
+	return nil
+}
+
+func (t *tableWriter) GetLog(id uint64, l *Log) error {
+	t.RLock()
+	defer t.RUnlock()
+
+	if id < t.first || id > t.last {
+		return ErrLogNotFound
+	}
+
+	var buf [4]byte
+	if _, err := t.meta.ReadAt(buf[0:4], int64((id-t.first)*4)); err != nil {
+		return err
+	}
+
+	offset := binary.BigEndian.Uint32(buf[0:4])
+
+	if err := l.DecodeAt(t.data, int64(offset)); err != nil {
+		return err
+	} else if l.ID != id {
+		return fmt.Errorf("invalid log id %d != %d", id, l.ID)
+	}
+
+	return nil
+}
+
+func (t *tableWriter) Sync() error {
+	t.Lock()
+
+	var err error
+	if t.data != nil {
+		err = t.data.Sync()
+		t.Unlock()
+		return err
+	}
+
+	if t.meta != nil {
+		err = t.meta.Sync()
+	}
+
+	t.Unlock()
+
+	return err
+}

vendor/github.com/siddontang/ledisdb/rpl/goleveldb_store.go

@@ -0,0 +1,225 @@
+package rpl
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/siddontang/go/num"
+	"github.com/siddontang/ledisdb/config"
+	"github.com/siddontang/ledisdb/store"
+)
+
+type GoLevelDBStore struct {
+	LogStore
+
+	m  sync.Mutex
+	db *store.DB
+
+	cfg *config.Config
+
+	first uint64
+	last  uint64
+
+	buf bytes.Buffer
+}
+
+func (s *GoLevelDBStore) FirstID() (uint64, error) {
+	s.m.Lock()
+	id, err := s.firstID()
+	s.m.Unlock()
+
+	return id, err
+}
+
+func (s *GoLevelDBStore) LastID() (uint64, error) {
+	s.m.Lock()
+	id, err := s.lastID()
+	s.m.Unlock()
+
+	return id, err
+}
+
+func (s *GoLevelDBStore) firstID() (uint64, error) {
+	if s.first != InvalidLogID {
+		return s.first, nil
+	}
+
+	it := s.db.NewIterator()
+	defer it.Close()
+
+	it.SeekToFirst()
+
+	if it.Valid() {
+		s.first = num.BytesToUint64(it.RawKey())
+	}
+
+	return s.first, nil
+}
+
+func (s *GoLevelDBStore) lastID() (uint64, error) {
+	if s.last != InvalidLogID {
+		return s.last, nil
+	}
+
+	it := s.db.NewIterator()
+	defer it.Close()
+
+	it.SeekToLast()
+
+	if it.Valid() {
+		s.last = num.BytesToUint64(it.RawKey())
+	}
+
+	return s.last, nil
+}
+
+func (s *GoLevelDBStore) GetLog(id uint64, log *Log) error {
+	v, err := s.db.Get(num.Uint64ToBytes(id))
+	if err != nil {
+		return err
+	} else if v == nil {
+		return ErrLogNotFound
+	} else {
+		return log.Decode(bytes.NewBuffer(v))
+	}
+}
+
+func (s *GoLevelDBStore) StoreLog(log *Log) error {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	last, err := s.lastID()
+	if err != nil {
+		return err
+	}
+
+	s.last = InvalidLogID
+
+	s.buf.Reset()
+
+	if log.ID != last+1 {
+		return ErrStoreLogID
+	}
+
+	last = log.ID
+	key := num.Uint64ToBytes(log.ID)
+
+	if err := log.Encode(&s.buf); err != nil {
+		return err
+	}
+
+	if err = s.db.Put(key, s.buf.Bytes()); err != nil {
+		return err
+	}
+
+	s.last = last
+	return nil
+}
+
+func (s *GoLevelDBStore) PurgeExpired(n int64) error {
+	if n <= 0 {
+		return fmt.Errorf("invalid expired time %d", n)
+	}
+
+	t := uint32(time.Now().Unix() - int64(n))
+
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	s.reset()
+
+	it := s.db.NewIterator()
+	it.SeekToFirst()
+
+	w := s.db.NewWriteBatch()
+	defer w.Rollback()
+
+	l := new(Log)
+	for ; it.Valid(); it.Next() {
+		v := it.RawValue()
+
+		if err := l.Unmarshal(v); err != nil {
+			return err
+		} else if l.CreateTime > t {
+			break
+		} else {
+			w.Delete(it.RawKey())
+		}
+	}
+
+	if err := w.Commit(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *GoLevelDBStore) Sync() error {
+	//no other way for sync, so ignore here
+	return nil
+}
+
+func (s *GoLevelDBStore) reset() {
+	s.first = InvalidLogID
+	s.last = InvalidLogID
+}
+
+func (s *GoLevelDBStore) Clear() error {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	if s.db != nil {
+		s.db.Close()
+	}
+
+	s.reset()
+	os.RemoveAll(s.cfg.DBPath)
+
+	return s.open()
+}
+
+func (s *GoLevelDBStore) Close() error {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	if s.db == nil {
+		return nil
+	}
+
+	err := s.db.Close()
+	s.db = nil
+	return err
+}
+
+func (s *GoLevelDBStore) open() error {
+	var err error
+
+	s.first = InvalidLogID
+	s.last = InvalidLogID
+
+	s.db, err = store.Open(s.cfg)
+	return err
+}
+
+func NewGoLevelDBStore(base string, syncLog int) (*GoLevelDBStore, error) {
+	cfg := config.NewConfigDefault()
+	cfg.DBName = "goleveldb"
+	cfg.DBPath = base
+	cfg.LevelDB.BlockSize = 16 * 1024 * 1024
+	cfg.LevelDB.CacheSize = 64 * 1024 * 1024
+	cfg.LevelDB.WriteBufferSize = 64 * 1024 * 1024
+	cfg.LevelDB.Compression = false
+	cfg.DBSyncCommit = syncLog
+
+	s := new(GoLevelDBStore)
+	s.cfg = cfg
+
+	if err := s.open(); err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}

vendor/github.com/siddontang/ledisdb/rpl/log.go

@@ -0,0 +1,167 @@
+package rpl
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"sync"
+)
+
+const LogHeadSize = 17
+
+type Log struct {
+	ID          uint64
+	CreateTime  uint32
+	Compression uint8
+
+	Data []byte
+}
+
+func (l *Log) HeadSize() int {
+	return LogHeadSize
+}
+
+func (l *Log) Size() int {
+	return l.HeadSize() + len(l.Data)
+}
+
+func (l *Log) Marshal() ([]byte, error) {
+	buf := bytes.NewBuffer(make([]byte, l.Size()))
+	buf.Reset()
+
+	if err := l.Encode(buf); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func (l *Log) Unmarshal(b []byte) error {
+	buf := bytes.NewBuffer(b)
+
+	return l.Decode(buf)
+}
+
+var headPool = sync.Pool{
+	New: func() interface{} { return make([]byte, LogHeadSize) },
+}
+
+func (l *Log) Encode(w io.Writer) error {
+	b := headPool.Get().([]byte)
+	pos := 0
+
+	binary.BigEndian.PutUint64(b[pos:], l.ID)
+	pos += 8
+	binary.BigEndian.PutUint32(b[pos:], uint32(l.CreateTime))
+	pos += 4
+	b[pos] = l.Compression
+	pos++
+	binary.BigEndian.PutUint32(b[pos:], uint32(len(l.Data)))
+
+	n, err := w.Write(b)
+	headPool.Put(b)
+
+	if err != nil {
+		return err
+	} else if n != LogHeadSize {
+		return io.ErrShortWrite
+	}
+
+	if n, err = w.Write(l.Data); err != nil {
+		return err
+	} else if n != len(l.Data) {
+		return io.ErrShortWrite
+	}
+	return nil
+}
+
+func (l *Log) Decode(r io.Reader) error {
+	length, err := l.DecodeHead(r)
+	if err != nil {
+		return err
+	}
+
+	l.growData(int(length))
+
+	if _, err := io.ReadFull(r, l.Data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *Log) DecodeHead(r io.Reader) (uint32, error) {
+	buf := headPool.Get().([]byte)
+
+	if _, err := io.ReadFull(r, buf); err != nil {
+		headPool.Put(buf)
+		return 0, err
+	}
+
+	length := l.decodeHeadBuf(buf)
+
+	headPool.Put(buf)
+
+	return length, nil
+}
+
+func (l *Log) DecodeAt(r io.ReaderAt, pos int64) error {
+	length, err := l.DecodeHeadAt(r, pos)
+	if err != nil {
+		return err
+	}
+
+	l.growData(int(length))
+	var n int
+	n, err = r.ReadAt(l.Data, pos+int64(LogHeadSize))
+	if err == io.EOF && n == len(l.Data) {
+		err = nil
+	}
+
+	return err
+}
+
+func (l *Log) growData(length int) {
+	l.Data = l.Data[0:0]
+
+	if cap(l.Data) >= length {
+		l.Data = l.Data[0:length]
+	} else {
+		l.Data = make([]byte, length)
+	}
+}
+
+func (l *Log) DecodeHeadAt(r io.ReaderAt, pos int64) (uint32, error) {
+	buf := headPool.Get().([]byte)
+
+	n, err := r.ReadAt(buf, pos)
+	if err != nil && err != io.EOF {
+		headPool.Put(buf)
+
+		return 0, err
+	}
+
+	length := l.decodeHeadBuf(buf)
+	headPool.Put(buf)
+
+	if err == io.EOF && (length != 0 || n != len(buf)) {
+		return 0, err
+	}
+
+	return length, nil
+}
+
+func (l *Log) decodeHeadBuf(buf []byte) uint32 {
+	pos := 0
+	l.ID = binary.BigEndian.Uint64(buf[pos:])
+	pos += 8
+
+	l.CreateTime = binary.BigEndian.Uint32(buf[pos:])
+	pos += 4
+
+	l.Compression = uint8(buf[pos])
+	pos++
+
+	length := binary.BigEndian.Uint32(buf[pos:])
+	return length
+}

vendor/github.com/siddontang/ledisdb/rpl/rpl.go

@@ -0,0 +1,336 @@
+package rpl
+
+import (
+	"encoding/binary"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/siddontang/go/log"
+	"github.com/siddontang/go/snappy"
+	"github.com/siddontang/ledisdb/config"
+)
+
+type Stat struct {
+	FirstID  uint64
+	LastID   uint64
+	CommitID uint64
+}
+
+type Replication struct {
+	m sync.Mutex
+
+	cfg *config.Config
+
+	s LogStore
+
+	commitID  uint64
+	commitLog *os.File
+
+	quit chan struct{}
+
+	wg sync.WaitGroup
+
+	nc chan struct{}
+
+	ncm sync.Mutex
+}
+
+func NewReplication(cfg *config.Config) (*Replication, error) {
+	if len(cfg.Replication.Path) == 0 {
+		cfg.Replication.Path = path.Join(cfg.DataDir, "rpl")
+	}
+
+	base := cfg.Replication.Path
+
+	r := new(Replication)
+
+	r.quit = make(chan struct{})
+	r.nc = make(chan struct{})
+
+	r.cfg = cfg
+
+	var err error
+
+	switch cfg.Replication.StoreName {
+	case "goleveldb":
+		if r.s, err = NewGoLevelDBStore(path.Join(base, "wal"), cfg.Replication.SyncLog); err != nil {
+			return nil, err
+		}
+	default:
+		if r.s, err = NewFileStore(path.Join(base, "ldb"), cfg); err != nil {
+			return nil, err
+		}
+	}
+
+	if r.commitLog, err = os.OpenFile(path.Join(base, "commit.log"), os.O_RDWR|os.O_CREATE, 0644); err != nil {
+		return nil, err
+	}
+
+	if s, _ := r.commitLog.Stat(); s.Size() == 0 {
+		r.commitID = 0
+	} else if err = binary.Read(r.commitLog, binary.BigEndian, &r.commitID); err != nil {
+		return nil, err
+	}
+
+	log.Infof("staring replication with commit ID %d", r.commitID)
+
+	r.wg.Add(1)
+	go r.run()
+
+	return r, nil
+}
+
+func (r *Replication) Close() error {
+	close(r.quit)
+
+	r.wg.Wait()
+
+	r.m.Lock()
+	defer r.m.Unlock()
+
+	log.Infof("closing replication with commit ID %d", r.commitID)
+
+	if r.s != nil {
+		r.s.Close()
+		r.s = nil
+	}
+
+	if err := r.updateCommitID(r.commitID, true); err != nil {
+		log.Errorf("update commit id err %s", err.Error())
+	}
+
+	if r.commitLog != nil {
+		r.commitLog.Close()
+		r.commitLog = nil
+	}
+
+	return nil
+}
+
+func (r *Replication) Log(data []byte) (*Log, error) {
+	if r.cfg.Replication.Compression {
+		//todo optimize
+		var err error
+		if data, err = snappy.Encode(nil, data); err != nil {
+			return nil, err
+		}
+	}
+
+	r.m.Lock()
+
+	lastID, err := r.s.LastID()
+	if err != nil {
+		r.m.Unlock()
+		return nil, err
+	}
+
+	commitId := r.commitID
+	if lastID < commitId {
+		lastID = commitId
+	} else if lastID > commitId {
+		r.m.Unlock()
+		return nil, ErrCommitIDBehind
+	}
+
+	l := new(Log)
+	l.ID = lastID + 1
+	l.CreateTime = uint32(time.Now().Unix())
+
+	if r.cfg.Replication.Compression {
+		l.Compression = 1
+	} else {
+		l.Compression = 0
+	}
+
+	l.Data = data
+
+	if err = r.s.StoreLog(l); err != nil {
+		r.m.Unlock()
+		return nil, err
+	}
+
+	r.m.Unlock()
+
+	r.ncm.Lock()
+	close(r.nc)
+	r.nc = make(chan struct{})
+	r.ncm.Unlock()
+
+	return l, nil
+}
+
+func (r *Replication) WaitLog() <-chan struct{} {
+	r.ncm.Lock()
+	ch := r.nc
+	r.ncm.Unlock()
+	return ch
+}
+
+func (r *Replication) StoreLog(log *Log) error {
+	r.m.Lock()
+	err := r.s.StoreLog(log)
+	r.m.Unlock()
+
+	return err
+}
+
+func (r *Replication) FirstLogID() (uint64, error) {
+	r.m.Lock()
+	id, err := r.s.FirstID()
+	r.m.Unlock()
+
+	return id, err
+}
+
+func (r *Replication) LastLogID() (uint64, error) {
+	r.m.Lock()
+	id, err := r.s.LastID()
+	r.m.Unlock()
+	return id, err
+}
+
+func (r *Replication) LastCommitID() (uint64, error) {
+	r.m.Lock()
+	id := r.commitID
+	r.m.Unlock()
+	return id, nil
+}
+
+func (r *Replication) UpdateCommitID(id uint64) error {
+	r.m.Lock()
+	err := r.updateCommitID(id, r.cfg.Replication.SyncLog == 2)
+	r.m.Unlock()
+
+	return err
+}
+
+func (r *Replication) Stat() (*Stat, error) {
+	r.m.Lock()
+	defer r.m.Unlock()
+
+	s := &Stat{}
+	var err error
+
+	if s.FirstID, err = r.s.FirstID(); err != nil {
+		return nil, err
+	}
+
+	if s.LastID, err = r.s.LastID(); err != nil {
+		return nil, err
+	}
+
+	s.CommitID = r.commitID
+	return s, nil
+}
+
+func (r *Replication) updateCommitID(id uint64, force bool) error {
+	if force {
+		if _, err := r.commitLog.Seek(0, os.SEEK_SET); err != nil {
+			return err
+		}
+
+		if err := binary.Write(r.commitLog, binary.BigEndian, id); err != nil {
+			return err
+		}
+	}
+
+	r.commitID = id
+
+	return nil
+}
+
+func (r *Replication) CommitIDBehind() (bool, error) {
+	r.m.Lock()
+
+	id, err := r.s.LastID()
+	if err != nil {
+		r.m.Unlock()
+		return false, err
+	}
+
+	behind := id > r.commitID
+	r.m.Unlock()
+
+	return behind, nil
+}
+
+func (r *Replication) GetLog(id uint64, log *Log) error {
+	return r.s.GetLog(id, log)
+}
+
+func (r *Replication) NextNeedCommitLog(log *Log) error {
+	r.m.Lock()
+	defer r.m.Unlock()
+
+	id, err := r.s.LastID()
+	if err != nil {
+		return err
+	}
+
+	if id <= r.commitID {
+		return ErrNoBehindLog
+	}
+
+	return r.s.GetLog(r.commitID+1, log)
+
+}
+
+func (r *Replication) Clear() error {
+	return r.ClearWithCommitID(0)
+}
+
+func (r *Replication) ClearWithCommitID(id uint64) error {
+	r.m.Lock()
+	defer r.m.Unlock()
+
+	if err := r.s.Clear(); err != nil {
+		return err
+	}
+
+	return r.updateCommitID(id, true)
+}
+
+func (r *Replication) run() {
+	defer r.wg.Done()
+
+	syncTc := time.NewTicker(1 * time.Second)
+	purgeTc := time.NewTicker(1 * time.Hour)
+
+	for {
+		select {
+		case <-purgeTc.C:
+			n := (r.cfg.Replication.ExpiredLogDays * 24 * 3600)
+			r.m.Lock()
+			err := r.s.PurgeExpired(int64(n))
+			r.m.Unlock()
+			if err != nil {
+				log.Errorf("purge expired log error %s", err.Error())
+			}
+		case <-syncTc.C:
+			if r.cfg.Replication.SyncLog == 1 {
+				r.m.Lock()
+				err := r.s.Sync()
+				r.m.Unlock()
+				if err != nil {
+					log.Errorf("sync store error %s", err.Error())
+				}
+			}
+			if r.cfg.Replication.SyncLog != 2 {
+				//we will sync commit id every 1 second
+				r.m.Lock()
+				err := r.updateCommitID(r.commitID, true)
+				r.m.Unlock()
+
+				if err != nil {
+					log.Errorf("sync commitid error %s", err.Error())
+				}
+			}
+		case <-r.quit:
+			syncTc.Stop()
+			purgeTc.Stop()
+			return
+		}
+	}
+}

vendor/github.com/siddontang/ledisdb/rpl/store.go

@@ -0,0 +1,36 @@
+package rpl
+
+import (
+	"errors"
+)
+
+const (
+	InvalidLogID uint64 = 0
+)
+
+var (
+	ErrLogNotFound    = errors.New("log not found")
+	ErrStoreLogID     = errors.New("log id is less")
+	ErrNoBehindLog    = errors.New("no behind commit log")
+	ErrCommitIDBehind = errors.New("commit id is behind last log id")
+)
+
+type LogStore interface {
+	GetLog(id uint64, log *Log) error
+
+	FirstID() (uint64, error)
+	LastID() (uint64, error)
+
+	// if log id is less than current last id, return error
+	StoreLog(log *Log) error
+
+	// Delete logs before n seconds
+	PurgeExpired(n int64) error
+
+	Sync() error
+
+	// Clear all logs
+	Clear() error
+
+	Close() error
+}

vendor/github.com/siddontang/ledisdb/store/db.go

@@ -0,0 +1,169 @@
+package store
+
+import (
+	"sync"
+	"time"
+
+	"github.com/siddontang/ledisdb/config"
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+type DB struct {
+	db   driver.IDB
+	name string
+
+	st *Stat
+
+	cfg *config.Config
+
+	lastCommit time.Time
+
+	m sync.Mutex
+}
+
+func (db *DB) Close() error {
+	return db.db.Close()
+}
+
+func (db *DB) String() string {
+	return db.name
+}
+
+func (db *DB) NewIterator() *Iterator {
+	db.st.IterNum.Add(1)
+
+	it := new(Iterator)
+	it.it = db.db.NewIterator()
+	it.st = db.st
+
+	return it
+}
+
+func (db *DB) Get(key []byte) ([]byte, error) {
+	t := time.Now()
+	v, err := db.db.Get(key)
+	db.st.statGet(v, err)
+	db.st.GetTotalTime.Add(time.Now().Sub(t))
+	return v, err
+}
+
+func (db *DB) Put(key []byte, value []byte) error {
+	db.st.PutNum.Add(1)
+
+	if db.needSyncCommit() {
+		return db.db.SyncPut(key, value)
+
+	} else {
+		return db.db.Put(key, value)
+
+	}
+}
+
+func (db *DB) Delete(key []byte) error {
+	db.st.DeleteNum.Add(1)
+
+	if db.needSyncCommit() {
+		return db.db.SyncDelete(key)
+	} else {
+		return db.db.Delete(key)
+	}
+}
+
+func (db *DB) NewWriteBatch() *WriteBatch {
+	db.st.BatchNum.Add(1)
+	wb := new(WriteBatch)
+	wb.wb = db.db.NewWriteBatch()
+	wb.st = db.st
+	wb.db = db
+	return wb
+}
+
+func (db *DB) NewSnapshot() (*Snapshot, error) {
+	db.st.SnapshotNum.Add(1)
+
+	var err error
+	s := &Snapshot{}
+	if s.ISnapshot, err = db.db.NewSnapshot(); err != nil {
+		return nil, err
+	}
+	s.st = db.st
+
+	return s, nil
+}
+
+func (db *DB) Compact() error {
+	db.st.CompactNum.Add(1)
+
+	t := time.Now()
+	err := db.db.Compact()
+
+	db.st.CompactTotalTime.Add(time.Now().Sub(t))
+
+	return err
+}
+
+func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
+	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1})
+}
+
+func (db *DB) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
+	return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1})
+}
+
+//count < 0, unlimit.
+//
+//offset must >= 0, if < 0, will get nothing.
+func (db *DB) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator {
+	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count})
+}
+
+//count < 0, unlimit.
+//
+//offset must >= 0, if < 0, will get nothing.
+func (db *DB) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator {
+	return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count})
+}
+
+func (db *DB) Stat() *Stat {
+	return db.st
+}
+
+func (db *DB) needSyncCommit() bool {
+	if db.cfg.DBSyncCommit == 0 {
+		return false
+	} else if db.cfg.DBSyncCommit == 2 {
+		return true
+	} else {
+		n := time.Now()
+		need := false
+		db.m.Lock()
+
+		if n.Sub(db.lastCommit) > time.Second {
+			need = true
+		}
+		db.lastCommit = n
+
+		db.m.Unlock()
+		return need
+	}
+
+}
+
+func (db *DB) GetSlice(key []byte) (Slice, error) {
+	if d, ok := db.db.(driver.ISliceGeter); ok {
+		t := time.Now()
+		v, err := d.GetSlice(key)
+		db.st.statGet(v, err)
+		db.st.GetTotalTime.Add(time.Now().Sub(t))
+		return v, err
+	} else {
+		v, err := db.Get(key)
+		if err != nil {
+			return nil, err
+		} else if v == nil {
+			return nil, nil
+		} else {
+			return driver.GoSlice(v), nil
+		}
+	}
+}

vendor/github.com/siddontang/ledisdb/store/driver/driver.go

@@ -0,0 +1,57 @@
+package driver
+
+type IDB interface {
+	Close() error
+
+	Get(key []byte) ([]byte, error)
+
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+
+	SyncPut(key []byte, value []byte) error
+	SyncDelete(key []byte) error
+
+	NewIterator() IIterator
+
+	NewWriteBatch() IWriteBatch
+
+	NewSnapshot() (ISnapshot, error)
+
+	Compact() error
+}
+
+type ISnapshot interface {
+	Get(key []byte) ([]byte, error)
+	NewIterator() IIterator
+	Close()
+}
+
+type IIterator interface {
+	Close() error
+
+	First()
+	Last()
+	Seek(key []byte)
+
+	Next()
+	Prev()
+
+	Valid() bool
+
+	Key() []byte
+	Value() []byte
+}
+
+type IWriteBatch interface {
+	Put(key []byte, value []byte)
+	Delete(key []byte)
+	Commit() error
+	SyncCommit() error
+	Rollback() error
+	Data() []byte
+	Close()
+}
+
+type ISliceGeter interface {
+	GetSlice(key []byte) (ISlice, error)
+}

vendor/github.com/siddontang/ledisdb/store/driver/slice.go

@@ -0,0 +1,21 @@
+package driver
+
+type ISlice interface {
+	Data() []byte
+	Size() int
+	Free()
+}
+
+type GoSlice []byte
+
+func (s GoSlice) Data() []byte {
+	return []byte(s)
+}
+
+func (s GoSlice) Size() int {
+	return len(s)
+}
+
+func (s GoSlice) Free() {
+
+}

vendor/github.com/siddontang/ledisdb/store/driver/store.go

@@ -0,0 +1,46 @@
+package driver
+
+import (
+	"fmt"
+
+	"github.com/siddontang/ledisdb/config"
+)
+
+type Store interface {
+	String() string
+	Open(path string, cfg *config.Config) (IDB, error)
+	Repair(path string, cfg *config.Config) error
+}
+
+var dbs = map[string]Store{}
+
+func Register(s Store) {
+	name := s.String()
+	if _, ok := dbs[name]; ok {
+		panic(fmt.Errorf("store %s is registered", s))
+	}
+
+	dbs[name] = s
+}
+
+func ListStores() []string {
+	s := []string{}
+	for k := range dbs {
+		s = append(s, k)
+	}
+
+	return s
+}
+
+func GetStore(cfg *config.Config) (Store, error) {
+	if len(cfg.DBName) == 0 {
+		cfg.DBName = config.DefaultDBName
+	}
+
+	s, ok := dbs[cfg.DBName]
+	if !ok {
+		return nil, fmt.Errorf("store %s is not registered", cfg.DBName)
+	}
+
+	return s, nil
+}

vendor/github.com/siddontang/ledisdb/store/goleveldb/batch.go

@@ -0,0 +1,39 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type WriteBatch struct {
+	db     *DB
+	wbatch *leveldb.Batch
+}
+
+func (w *WriteBatch) Put(key, value []byte) {
+	w.wbatch.Put(key, value)
+}
+
+func (w *WriteBatch) Delete(key []byte) {
+	w.wbatch.Delete(key)
+}
+
+func (w *WriteBatch) Commit() error {
+	return w.db.db.Write(w.wbatch, nil)
+}
+
+func (w *WriteBatch) SyncCommit() error {
+	return w.db.db.Write(w.wbatch, w.db.syncOpts)
+}
+
+func (w *WriteBatch) Rollback() error {
+	w.wbatch.Reset()
+	return nil
+}
+
+func (w *WriteBatch) Close() {
+	w.wbatch.Reset()
+}
+
+func (w *WriteBatch) Data() []byte {
+	return w.wbatch.Dump()
+}

vendor/github.com/siddontang/ledisdb/store/goleveldb/const.go

@@ -0,0 +1,4 @@
+package goleveldb
+
+const DBName = "goleveldb"
+const MemDBName = "memory"

vendor/github.com/siddontang/ledisdb/store/goleveldb/db.go

@@ -0,0 +1,204 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/siddontang/ledisdb/config"
+	"github.com/siddontang/ledisdb/store/driver"
+
+	"os"
+)
+
+const defaultFilterBits int = 10
+
+type Store struct {
+}
+
+func (s Store) String() string {
+	return DBName
+}
+
+type MemStore struct {
+}
+
+func (s MemStore) String() string {
+	return MemDBName
+}
+
+type DB struct {
+	path string
+
+	cfg *config.LevelDBConfig
+
+	db *leveldb.DB
+
+	opts *opt.Options
+
+	iteratorOpts *opt.ReadOptions
+
+	syncOpts *opt.WriteOptions
+
+	cache cache.Cache
+
+	filter filter.Filter
+}
+
+func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) {
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return nil, err
+	}
+
+	db := new(DB)
+	db.path = path
+	db.cfg = &cfg.LevelDB
+
+	db.initOpts()
+
+	var err error
+	db.db, err = leveldb.OpenFile(db.path, db.opts)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func (s Store) Repair(path string, cfg *config.Config) error {
+	db, err := leveldb.RecoverFile(path, newOptions(&cfg.LevelDB))
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	return nil
+}
+
+func (s MemStore) Open(path string, cfg *config.Config) (driver.IDB, error) {
+	db := new(DB)
+	db.path = path
+	db.cfg = &cfg.LevelDB
+
+	db.initOpts()
+
+	var err error
+	db.db, err = leveldb.Open(storage.NewMemStorage(), db.opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func (s MemStore) Repair(path string, cfg *config.Config) error {
+	return nil
+}
+
+func (db *DB) initOpts() {
+	db.opts = newOptions(db.cfg)
+
+	db.iteratorOpts = &opt.ReadOptions{}
+	db.iteratorOpts.DontFillCache = true
+
+	db.syncOpts = &opt.WriteOptions{}
+	db.syncOpts.Sync = true
+}
+
+func newOptions(cfg *config.LevelDBConfig) *opt.Options {
+	opts := &opt.Options{}
+	opts.ErrorIfMissing = false
+
+	opts.BlockCacheCapacity = cfg.CacheSize
+
+	//we must use bloomfilter
+	opts.Filter = filter.NewBloomFilter(defaultFilterBits)
+
+	if !cfg.Compression {
+		opts.Compression = opt.NoCompression
+	} else {
+		opts.Compression = opt.SnappyCompression
+	}
+
+	opts.BlockSize = cfg.BlockSize
+	opts.WriteBuffer = cfg.WriteBufferSize
+	opts.OpenFilesCacheCapacity = cfg.MaxOpenFiles
+
+	//here we use default value, later add config support
+	opts.CompactionTableSize = 32 * 1024 * 1024
+	opts.WriteL0SlowdownTrigger = 16
+	opts.WriteL0PauseTrigger = 64
+
+	return opts
+}
+
+func (db *DB) Close() error {
+	return db.db.Close()
+}
+
+func (db *DB) Put(key, value []byte) error {
+	return db.db.Put(key, value, nil)
+}
+
+func (db *DB) Get(key []byte) ([]byte, error) {
+	v, err := db.db.Get(key, nil)
+	if err == leveldb.ErrNotFound {
+		return nil, nil
+	}
+	return v, nil
+}
+
+func (db *DB) Delete(key []byte) error {
+	return db.db.Delete(key, nil)
+}
+
+func (db *DB) SyncPut(key []byte, value []byte) error {
+	return db.db.Put(key, value, db.syncOpts)
+}
+
+func (db *DB) SyncDelete(key []byte) error {
+	return db.db.Delete(key, db.syncOpts)
+}
+
+func (db *DB) NewWriteBatch() driver.IWriteBatch {
+	wb := &WriteBatch{
+		db:     db,
+		wbatch: new(leveldb.Batch),
+	}
+	return wb
+}
+
+func (db *DB) NewIterator() driver.IIterator {
+	it := &Iterator{
+		db.db.NewIterator(nil, db.iteratorOpts),
+	}
+
+	return it
+}
+
+func (db *DB) NewSnapshot() (driver.ISnapshot, error) {
+	snapshot, err := db.db.GetSnapshot()
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Snapshot{
+		db:  db,
+		snp: snapshot,
+	}
+
+	return s, nil
+}
+
+func (db *DB) Compact() error {
+	return db.db.CompactRange(util.Range{nil, nil})
+}
+
+func init() {
+	driver.Register(Store{})
+	driver.Register(MemStore{})
+}

vendor/github.com/siddontang/ledisdb/store/goleveldb/iterator.go

@@ -0,0 +1,49 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+)
+
+type Iterator struct {
+	it iterator.Iterator
+}
+
+func (it *Iterator) Key() []byte {
+	return it.it.Key()
+}
+
+func (it *Iterator) Value() []byte {
+	return it.it.Value()
+}
+
+func (it *Iterator) Close() error {
+	if it.it != nil {
+		it.it.Release()
+		it.it = nil
+	}
+	return nil
+}
+
+func (it *Iterator) Valid() bool {
+	return it.it.Valid()
+}
+
+func (it *Iterator) Next() {
+	it.it.Next()
+}
+
+func (it *Iterator) Prev() {
+	it.it.Prev()
+}
+
+func (it *Iterator) First() {
+	it.it.First()
+}
+
+func (it *Iterator) Last() {
+	it.it.Last()
+}
+
+func (it *Iterator) Seek(key []byte) {
+	it.it.Seek(key)
+}

vendor/github.com/siddontang/ledisdb/store/goleveldb/snapshot.go

@@ -0,0 +1,26 @@
+package goleveldb
+
+import (
+	"github.com/siddontang/ledisdb/store/driver"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type Snapshot struct {
+	db  *DB
+	snp *leveldb.Snapshot
+}
+
+func (s *Snapshot) Get(key []byte) ([]byte, error) {
+	return s.snp.Get(key, s.db.iteratorOpts)
+}
+
+func (s *Snapshot) NewIterator() driver.IIterator {
+	it := &Iterator{
+		s.snp.NewIterator(nil, s.db.iteratorOpts),
+	}
+	return it
+}
+
+func (s *Snapshot) Close() {
+	s.snp.Release()
+}

vendor/github.com/siddontang/ledisdb/store/iterator.go

@@ -0,0 +1,334 @@
+package store
+
+import (
+	"bytes"
+
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+const (
+	IteratorForward  uint8 = 0
+	IteratorBackward uint8 = 1
+)
+
+const (
+	RangeClose uint8 = 0x00
+	RangeLOpen uint8 = 0x01
+	RangeROpen uint8 = 0x10
+	RangeOpen  uint8 = 0x11
+)
+
+// min must less or equal than max
+//
+// range type:
+//
+//  close: [min, max]
+//  open: (min, max)
+//  lopen: (min, max]
+//  ropen: [min, max)
+//
+type Range struct {
+	Min []byte
+	Max []byte
+
+	Type uint8
+}
+
+type Limit struct {
+	Offset int
+	Count  int
+}
+
+type Iterator struct {
+	it driver.IIterator
+	st *Stat
+}
+
+// Returns a copy of key.
+func (it *Iterator) Key() []byte {
+	k := it.it.Key()
+	if k == nil {
+		return nil
+	}
+
+	return append([]byte{}, k...)
+}
+
+// Returns a copy of value.
+func (it *Iterator) Value() []byte {
+	v := it.it.Value()
+	if v == nil {
+		return nil
+	}
+
+	return append([]byte{}, v...)
+}
+
+// Returns a reference of key.
+// you must be careful that it will be changed after next iterate.
+func (it *Iterator) RawKey() []byte {
+	return it.it.Key()
+}
+
+// Returns a reference of value.
+// you must be careful that it will be changed after next iterate.
+func (it *Iterator) RawValue() []byte {
+	return it.it.Value()
+}
+
+// Copy key to b, if b len is small or nil, returns a new one.
+func (it *Iterator) BufKey(b []byte) []byte {
+	k := it.RawKey()
+	if k == nil {
+		return nil
+	}
+	if b == nil {
+		b = []byte{}
+	}
+
+	b = b[0:0]
+	return append(b, k...)
+}
+
+// Copy value to b, if b len is small or nil, returns a new one.
+func (it *Iterator) BufValue(b []byte) []byte {
+	v := it.RawValue()
+	if v == nil {
+		return nil
+	}
+
+	if b == nil {
+		b = []byte{}
+	}
+
+	b = b[0:0]
+	return append(b, v...)
+}
+
+func (it *Iterator) Close() {
+	if it.it != nil {
+		it.st.IterCloseNum.Add(1)
+		it.it.Close()
+		it.it = nil
+	}
+}
+
+func (it *Iterator) Valid() bool {
+	return it.it.Valid()
+}
+
+func (it *Iterator) Next() {
+	it.st.IterSeekNum.Add(1)
+	it.it.Next()
+}
+
+func (it *Iterator) Prev() {
+	it.st.IterSeekNum.Add(1)
+	it.it.Prev()
+}
+
+func (it *Iterator) SeekToFirst() {
+	it.st.IterSeekNum.Add(1)
+	it.it.First()
+}
+
+func (it *Iterator) SeekToLast() {
+	it.st.IterSeekNum.Add(1)
+	it.it.Last()
+}
+
+func (it *Iterator) Seek(key []byte) {
+	it.st.IterSeekNum.Add(1)
+	it.it.Seek(key)
+}
+
+// Finds by key, if not found, nil returns.
+func (it *Iterator) Find(key []byte) []byte {
+	it.Seek(key)
+	if it.Valid() {
+		k := it.RawKey()
+		if k == nil {
+			return nil
+		} else if bytes.Equal(k, key) {
+			return it.Value()
+		}
+	}
+
+	return nil
+}
+
+// Finds by key, if not found, nil returns, else a reference of value returns.
+// you must be careful that it will be changed after next iterate.
+func (it *Iterator) RawFind(key []byte) []byte {
+	it.Seek(key)
+	if it.Valid() {
+		k := it.RawKey()
+		if k == nil {
+			return nil
+		} else if bytes.Equal(k, key) {
+			return it.RawValue()
+		}
+	}
+
+	return nil
+}
+
+type RangeLimitIterator struct {
+	it *Iterator
+
+	r *Range
+	l *Limit
+
+	step int
+
+	//0 for IteratorForward, 1 for IteratorBackward
+	direction uint8
+}
+
+func (it *RangeLimitIterator) Key() []byte {
+	return it.it.Key()
+}
+
+func (it *RangeLimitIterator) Value() []byte {
+	return it.it.Value()
+}
+
+func (it *RangeLimitIterator) RawKey() []byte {
+	return it.it.RawKey()
+}
+
+func (it *RangeLimitIterator) RawValue() []byte {
+	return it.it.RawValue()
+}
+
+func (it *RangeLimitIterator) BufKey(b []byte) []byte {
+	return it.it.BufKey(b)
+}
+
+func (it *RangeLimitIterator) BufValue(b []byte) []byte {
+	return it.it.BufValue(b)
+}
+
+func (it *RangeLimitIterator) Valid() bool {
+	if it.l.Offset < 0 {
+		return false
+	} else if !it.it.Valid() {
+		return false
+	} else if it.l.Count >= 0 && it.step >= it.l.Count {
+		return false
+	}
+
+	if it.direction == IteratorForward {
+		if it.r.Max != nil {
+			r := bytes.Compare(it.it.RawKey(), it.r.Max)
+			if it.r.Type&RangeROpen > 0 {
+				return !(r >= 0)
+			} else {
+				return !(r > 0)
+			}
+		}
+	} else {
+		if it.r.Min != nil {
+			r := bytes.Compare(it.it.RawKey(), it.r.Min)
+			if it.r.Type&RangeLOpen > 0 {
+				return !(r <= 0)
+			} else {
+				return !(r < 0)
+			}
+		}
+	}
+
+	return true
+}
+
+func (it *RangeLimitIterator) Next() {
+	it.step++
+
+	if it.direction == IteratorForward {
+		it.it.Next()
+	} else {
+		it.it.Prev()
+	}
+}
+
+func (it *RangeLimitIterator) Close() {
+	it.it.Close()
+}
+
+func NewRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, l, IteratorForward)
+}
+
+func NewRevRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, l, IteratorBackward)
+}
+
+func NewRangeIterator(i *Iterator, r *Range) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorForward)
+}
+
+func NewRevRangeIterator(i *Iterator, r *Range) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorBackward)
+}
+
+func rangeLimitIterator(i *Iterator, r *Range, l *Limit, direction uint8) *RangeLimitIterator {
+	it := new(RangeLimitIterator)
+
+	it.it = i
+
+	it.r = r
+	it.l = l
+	it.direction = direction
+
+	it.step = 0
+
+	if l.Offset < 0 {
+		return it
+	}
+
+	if direction == IteratorForward {
+		if r.Min == nil {
+			it.it.SeekToFirst()
+		} else {
+			it.it.Seek(r.Min)
+
+			if r.Type&RangeLOpen > 0 {
+				if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Min) {
+					it.it.Next()
+				}
+			}
+		}
+	} else {
+		if r.Max == nil {
+			it.it.SeekToLast()
+		} else {
+			it.it.Seek(r.Max)
+
+			if !it.it.Valid() {
+				it.it.SeekToLast()
+			} else {
+				if !bytes.Equal(it.it.RawKey(), r.Max) {
+					it.it.Prev()
+				}
+			}
+
+			if r.Type&RangeROpen > 0 {
+				if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Max) {
+					it.it.Prev()
+				}
+			}
+		}
+	}
+
+	for i := 0; i < l.Offset; i++ {
+		if it.it.Valid() {
+			if it.direction == IteratorForward {
+				it.it.Next()
+			} else {
+				it.it.Prev()
+			}
+		}
+	}
+
+	return it
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/batch.go

@@ -0,0 +1,99 @@
+// +build leveldb
+
+package leveldb
+
+// #cgo LDFLAGS: -lleveldb
+// #include "leveldb/c.h"
+// #include "leveldb_ext.h"
+import "C"
+
+import (
+	"unsafe"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type WriteBatch struct {
+	db     *DB
+	wbatch *C.leveldb_writebatch_t
+}
+
+func newWriteBatch(db *DB) *WriteBatch {
+	w := new(WriteBatch)
+	w.db = db
+	w.wbatch = C.leveldb_writebatch_create()
+
+	return w
+}
+
+func (w *WriteBatch) Close() {
+	if w.wbatch != nil {
+		C.leveldb_writebatch_destroy(w.wbatch)
+		w.wbatch = nil
+	}
+}
+
+func (w *WriteBatch) Put(key, value []byte) {
+	var k, v *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+	if len(value) != 0 {
+		v = (*C.char)(unsafe.Pointer(&value[0]))
+	}
+
+	lenk := len(key)
+	lenv := len(value)
+
+	C.leveldb_writebatch_put(w.wbatch, k, C.size_t(lenk), v, C.size_t(lenv))
+}
+
+func (w *WriteBatch) Delete(key []byte) {
+	C.leveldb_writebatch_delete(w.wbatch,
+		(*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)))
+}
+
+func (w *WriteBatch) Commit() error {
+	return w.commit(w.db.writeOpts)
+}
+
+func (w *WriteBatch) SyncCommit() error {
+	return w.commit(w.db.syncOpts)
+}
+
+func (w *WriteBatch) Rollback() error {
+	C.leveldb_writebatch_clear(w.wbatch)
+
+	return nil
+}
+
+func (w *WriteBatch) commit(wb *WriteOptions) error {
+	var errStr *C.char
+	C.leveldb_write(w.db.db, wb.Opt, w.wbatch, &errStr)
+	if errStr != nil {
+		return saveError(errStr)
+	}
+	return nil
+}
+
+//export leveldb_writebatch_iterate_put
+func leveldb_writebatch_iterate_put(p unsafe.Pointer, k *C.char, klen C.size_t, v *C.char, vlen C.size_t) {
+	b := (*leveldb.Batch)(p)
+	key := slice(unsafe.Pointer(k), int(klen))
+	value := slice(unsafe.Pointer(v), int(vlen))
+	b.Put(key, value)
+}
+
+//export leveldb_writebatch_iterate_delete
+func leveldb_writebatch_iterate_delete(p unsafe.Pointer, k *C.char, klen C.size_t) {
+	b := (*leveldb.Batch)(p)
+	key := slice(unsafe.Pointer(k), int(klen))
+	b.Delete(key)
+}
+
+func (w *WriteBatch) Data() []byte {
+	gbatch := leveldb.Batch{}
+	C.leveldb_writebatch_iterate_ext(w.wbatch,
+		unsafe.Pointer(&gbatch))
+	return gbatch.Dump()
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/cache.go

@@ -0,0 +1,20 @@
+// +build leveldb
+
+package leveldb
+
+// #cgo LDFLAGS: -lleveldb
+// #include <stdint.h>
+// #include "leveldb/c.h"
+import "C"
+
+type Cache struct {
+	Cache *C.leveldb_cache_t
+}
+
+func NewLRUCache(capacity int) *Cache {
+	return &Cache{C.leveldb_cache_create_lru(C.size_t(capacity))}
+}
+
+func (c *Cache) Close() {
+	C.leveldb_cache_destroy(c.Cache)
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/const.go

@@ -0,0 +1,3 @@
+package leveldb
+
+const DBName = "leveldb"

vendor/github.com/siddontang/ledisdb/store/leveldb/db.go

@@ -0,0 +1,314 @@
+// +build leveldb
+
+// Package leveldb is a wrapper for c++ leveldb
+package leveldb
+
+/*
+#cgo LDFLAGS: -lleveldb
+#include <leveldb/c.h>
+#include "leveldb_ext.h"
+*/
+import "C"
+
+import (
+	"os"
+	"runtime"
+	"unsafe"
+
+	"github.com/siddontang/ledisdb/config"
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+const defaultFilterBits int = 10
+
+type Store struct {
+}
+
+func (s Store) String() string {
+	return DBName
+}
+
+func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) {
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return nil, err
+	}
+
+	db := new(DB)
+	db.path = path
+	db.cfg = &cfg.LevelDB
+
+	if err := db.open(); err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func (s Store) Repair(path string, cfg *config.Config) error {
+	db := new(DB)
+	db.cfg = &cfg.LevelDB
+	db.path = path
+
+	err := db.open()
+	defer db.Close()
+
+	//open ok, do not need repair
+	if err == nil {
+		return nil
+	}
+
+	var errStr *C.char
+	ldbname := C.CString(path)
+	defer C.leveldb_free(unsafe.Pointer(ldbname))
+
+	C.leveldb_repair_db(db.opts.Opt, ldbname, &errStr)
+	if errStr != nil {
+		return saveError(errStr)
+	}
+	return nil
+}
+
+type DB struct {
+	path string
+
+	cfg *config.LevelDBConfig
+
+	db *C.leveldb_t
+
+	opts *Options
+
+	//for default read and write options
+	readOpts     *ReadOptions
+	writeOpts    *WriteOptions
+	iteratorOpts *ReadOptions
+
+	syncOpts *WriteOptions
+
+	cache *Cache
+
+	filter *FilterPolicy
+}
+
+func (db *DB) open() error {
+	db.initOptions(db.cfg)
+
+	var errStr *C.char
+	ldbname := C.CString(db.path)
+	defer C.leveldb_free(unsafe.Pointer(ldbname))
+
+	db.db = C.leveldb_open(db.opts.Opt, ldbname, &errStr)
+	if errStr != nil {
+		db.db = nil
+		return saveError(errStr)
+	}
+	return nil
+}
+
+func (db *DB) initOptions(cfg *config.LevelDBConfig) {
+	opts := NewOptions()
+
+	opts.SetCreateIfMissing(true)
+
+	db.cache = NewLRUCache(cfg.CacheSize)
+	opts.SetCache(db.cache)
+
+	//we must use bloomfilter
+	db.filter = NewBloomFilter(defaultFilterBits)
+	opts.SetFilterPolicy(db.filter)
+
+	if !cfg.Compression {
+		opts.SetCompression(NoCompression)
+	} else {
+		opts.SetCompression(SnappyCompression)
+	}
+
+	opts.SetBlockSize(cfg.BlockSize)
+
+	opts.SetWriteBufferSize(cfg.WriteBufferSize)
+
+	opts.SetMaxOpenFiles(cfg.MaxOpenFiles)
+
+	opts.SetMaxFileSize(cfg.MaxFileSize)
+
+	db.opts = opts
+
+	db.readOpts = NewReadOptions()
+	db.writeOpts = NewWriteOptions()
+
+	db.syncOpts = NewWriteOptions()
+	db.syncOpts.SetSync(true)
+
+	db.iteratorOpts = NewReadOptions()
+	db.iteratorOpts.SetFillCache(false)
+}
+
+func (db *DB) Close() error {
+	if db.db != nil {
+		C.leveldb_close(db.db)
+		db.db = nil
+	}
+
+	db.opts.Close()
+
+	if db.cache != nil {
+		db.cache.Close()
+	}
+
+	if db.filter != nil {
+		db.filter.Close()
+	}
+
+	db.readOpts.Close()
+	db.writeOpts.Close()
+	db.iteratorOpts.Close()
+
+	return nil
+}
+
+func (db *DB) Put(key, value []byte) error {
+	return db.put(db.writeOpts, key, value)
+}
+
+func (db *DB) Get(key []byte) ([]byte, error) {
+	return db.get(db.readOpts, key)
+}
+
+func (db *DB) Delete(key []byte) error {
+	return db.delete(db.writeOpts, key)
+}
+
+func (db *DB) SyncPut(key []byte, value []byte) error {
+	return db.put(db.syncOpts, key, value)
+}
+
+func (db *DB) SyncDelete(key []byte) error {
+	return db.delete(db.syncOpts, key)
+}
+
+func (db *DB) NewWriteBatch() driver.IWriteBatch {
+	wb := newWriteBatch(db)
+
+	runtime.SetFinalizer(wb, func(w *WriteBatch) {
+		w.Close()
+	})
+
+	return wb
+}
+
+func (db *DB) NewIterator() driver.IIterator {
+	it := new(Iterator)
+
+	it.it = C.leveldb_create_iterator(db.db, db.iteratorOpts.Opt)
+
+	return it
+}
+
+func (db *DB) NewSnapshot() (driver.ISnapshot, error) {
+	snap := &Snapshot{
+		db:           db,
+		snap:         C.leveldb_create_snapshot(db.db),
+		readOpts:     NewReadOptions(),
+		iteratorOpts: NewReadOptions(),
+	}
+	snap.readOpts.SetSnapshot(snap)
+	snap.iteratorOpts.SetSnapshot(snap)
+	snap.iteratorOpts.SetFillCache(false)
+
+	return snap, nil
+}
+
+func (db *DB) put(wo *WriteOptions, key, value []byte) error {
+	var errStr *C.char
+	var k, v *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+	if len(value) != 0 {
+		v = (*C.char)(unsafe.Pointer(&value[0]))
+	}
+
+	lenk := len(key)
+	lenv := len(value)
+	C.leveldb_put(
+		db.db, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr)
+
+	if errStr != nil {
+		return saveError(errStr)
+	}
+	return nil
+}
+
+func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) {
+	var errStr *C.char
+	var vallen C.size_t
+	var k *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+
+	value := C.leveldb_get(
+		db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr)
+
+	if errStr != nil {
+		return nil, saveError(errStr)
+	}
+
+	if value == nil {
+		return nil, nil
+	}
+
+	defer C.leveldb_free(unsafe.Pointer(value))
+
+	return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil
+}
+
+func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) {
+	var errStr *C.char
+	var vallen C.size_t
+	var k *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+
+	value := C.leveldb_get(
+		db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr)
+
+	if errStr != nil {
+		return nil, saveError(errStr)
+	}
+
+	if value == nil {
+		return nil, nil
+	}
+
+	return NewCSlice(unsafe.Pointer(value), int(vallen)), nil
+}
+
+func (db *DB) delete(wo *WriteOptions, key []byte) error {
+	var errStr *C.char
+	var k *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+
+	C.leveldb_delete(
+		db.db, wo.Opt, k, C.size_t(len(key)), &errStr)
+
+	if errStr != nil {
+		return saveError(errStr)
+	}
+	return nil
+}
+
+func (db *DB) Compact() error {
+	C.leveldb_compact_range(db.db, nil, 0, nil, 0)
+	return nil
+}
+
+func (db *DB) GetSlice(key []byte) (driver.ISlice, error) {
+	return db.getSlice(db.readOpts, key)
+}
+
+func init() {
+	driver.Register(Store{})
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/filterpolicy.go

@@ -0,0 +1,21 @@
+// +build leveldb
+
+package leveldb
+
+// #cgo LDFLAGS: -lleveldb
+// #include <stdlib.h>
+// #include "leveldb/c.h"
+import "C"
+
+type FilterPolicy struct {
+	Policy *C.leveldb_filterpolicy_t
+}
+
+func NewBloomFilter(bitsPerKey int) *FilterPolicy {
+	policy := C.leveldb_filterpolicy_create_bloom(C.int(bitsPerKey))
+	return &FilterPolicy{policy}
+}
+
+func (fp *FilterPolicy) Close() {
+	C.leveldb_filterpolicy_destroy(fp.Policy)
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/iterator.go

@@ -0,0 +1,70 @@
+// +build leveldb
+
+package leveldb
+
+// #cgo LDFLAGS: -lleveldb
+// #include <stdlib.h>
+// #include "leveldb/c.h"
+// #include "leveldb_ext.h"
+import "C"
+
+import (
+	"unsafe"
+)
+
+type Iterator struct {
+	it      *C.leveldb_iterator_t
+	isValid C.uchar
+}
+
+func (it *Iterator) Key() []byte {
+	var klen C.size_t
+	kdata := C.leveldb_iter_key(it.it, &klen)
+	if kdata == nil {
+		return nil
+	}
+
+	return slice(unsafe.Pointer(kdata), int(C.int(klen)))
+}
+
+func (it *Iterator) Value() []byte {
+	var vlen C.size_t
+	vdata := C.leveldb_iter_value(it.it, &vlen)
+	if vdata == nil {
+		return nil
+	}
+
+	return slice(unsafe.Pointer(vdata), int(C.int(vlen)))
+}
+
+func (it *Iterator) Close() error {
+	if it.it != nil {
+		C.leveldb_iter_destroy(it.it)
+		it.it = nil
+	}
+	return nil
+}
+
+func (it *Iterator) Valid() bool {
+	return ucharToBool(it.isValid)
+}
+
+func (it *Iterator) Next() {
+	it.isValid = C.leveldb_iter_next_ext(it.it)
+}
+
+func (it *Iterator) Prev() {
+	it.isValid = C.leveldb_iter_prev_ext(it.it)
+}
+
+func (it *Iterator) First() {
+	it.isValid = C.leveldb_iter_seek_to_first_ext(it.it)
+}
+
+func (it *Iterator) Last() {
+	it.isValid = C.leveldb_iter_seek_to_last_ext(it.it)
+}
+
+func (it *Iterator) Seek(key []byte) {
+	it.isValid = C.leveldb_iter_seek_ext(it.it, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)))
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.cc

@@ -0,0 +1,95 @@
+// +build leveldb
+
+#include "leveldb_ext.h"
+
+#include <stdlib.h>
+//#include <string>
+
+//#include "leveldb/db.h"
+
+//using namespace leveldb;
+
+extern "C" {
+
+// static bool SaveError(char** errptr, const Status& s) {
+//   assert(errptr != NULL);
+//   if (s.ok()) {
+//     return false;
+//   } else if (*errptr == NULL) {
+//     *errptr = strdup(s.ToString().c_str());
+//   } else {
+//     free(*errptr);
+//     *errptr = strdup(s.ToString().c_str());
+//   }
+//   return true;
+// }
+
+// void* leveldb_get_ext(
+//     leveldb_t* db,
+//     const leveldb_readoptions_t* options,
+//     const char* key, size_t keylen,
+//     char** valptr,
+//     size_t* vallen,
+//     char** errptr) {
+
+//     std::string *tmp = new(std::string);
+
+//     //very tricky, maybe changed with c++ leveldb upgrade
+//     Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp);
+
+//     if (s.ok()) {
+//         *valptr = (char*)tmp->data();
+//         *vallen = tmp->size();
+//     } else {
+//         delete(tmp);
+//         tmp = NULL;
+//         *valptr = NULL;
+//         *vallen = 0;
+//         if (!s.IsNotFound()) {
+//             SaveError(errptr, s);
+//         }
+//     }
+//     return tmp;
+// }
+
+// void leveldb_get_free_ext(void* context) {
+//     std::string* s = (std::string*)context;
+
+//     delete(s);
+// }
+
+
+unsigned char leveldb_iter_seek_to_first_ext(leveldb_iterator_t* iter) {
+    leveldb_iter_seek_to_first(iter);
+    return leveldb_iter_valid(iter);
+}
+
+unsigned char leveldb_iter_seek_to_last_ext(leveldb_iterator_t* iter) {
+    leveldb_iter_seek_to_last(iter);
+    return leveldb_iter_valid(iter);
+}
+
+unsigned char leveldb_iter_seek_ext(leveldb_iterator_t* iter, const char* k, size_t klen) {
+    leveldb_iter_seek(iter, k, klen);
+    return leveldb_iter_valid(iter);
+}
+
+unsigned char leveldb_iter_next_ext(leveldb_iterator_t* iter) {
+    leveldb_iter_next(iter);
+    return leveldb_iter_valid(iter);
+}
+
+unsigned char leveldb_iter_prev_ext(leveldb_iterator_t* iter) {
+    leveldb_iter_prev(iter);
+    return leveldb_iter_valid(iter);
+}
+
+extern void leveldb_writebatch_iterate_put(void*, const char* k, size_t klen, const char* v, size_t vlen);
+extern void leveldb_writebatch_iterate_delete(void*, const char* k, size_t klen);
+
+void leveldb_writebatch_iterate_ext(leveldb_writebatch_t* w, void *p) {
+    leveldb_writebatch_iterate(w, p, 
+        leveldb_writebatch_iterate_put, leveldb_writebatch_iterate_delete);
+}
+
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.h

@@ -0,0 +1,41 @@
+// +build leveldb
+
+#ifndef LEVELDB_EXT_H
+#define LEVELDB_EXT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "leveldb/c.h"
+
+
+// /* Returns NULL if not found. Otherwise stores the value in **valptr.
+//    Stores the length of the value in *vallen. 
+//    Returns a context must be later to free*/
+// extern void* leveldb_get_ext(
+//     leveldb_t* db,
+//     const leveldb_readoptions_t* options,
+//     const char* key, size_t keylen,
+//     char** valptr,
+//     size_t* vallen,
+//     char** errptr);
+
+// // Free context returns by leveldb_get_ext
+// extern void leveldb_get_free_ext(void* context);
+
+
+// Below iterator functions like leveldb iterator but returns valid status for iterator
+extern unsigned char leveldb_iter_seek_to_first_ext(leveldb_iterator_t*);
+extern unsigned char leveldb_iter_seek_to_last_ext(leveldb_iterator_t*);
+extern unsigned char leveldb_iter_seek_ext(leveldb_iterator_t*, const char* k, size_t klen);
+extern unsigned char leveldb_iter_next_ext(leveldb_iterator_t*);
+extern unsigned char leveldb_iter_prev_ext(leveldb_iterator_t*);
+
+extern void leveldb_writebatch_iterate_ext(leveldb_writebatch_t*, void* p);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

vendor/github.com/siddontang/ledisdb/store/leveldb/levigo-license

@@ -0,0 +1,7 @@
+Copyright (c) 2012 Jeffrey M Hodges
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/siddontang/ledisdb/store/leveldb/options.go

@@ -0,0 +1,126 @@
+// +build leveldb
+
+package leveldb
+
+// #cgo LDFLAGS: -lleveldb
+// #include "leveldb/c.h"
+import "C"
+
+type CompressionOpt int
+
+const (
+	NoCompression     = CompressionOpt(0)
+	SnappyCompression = CompressionOpt(1)
+)
+
+type Options struct {
+	Opt *C.leveldb_options_t
+}
+
+type ReadOptions struct {
+	Opt *C.leveldb_readoptions_t
+}
+
+type WriteOptions struct {
+	Opt *C.leveldb_writeoptions_t
+}
+
+func NewOptions() *Options {
+	opt := C.leveldb_options_create()
+	return &Options{opt}
+}
+
+func NewReadOptions() *ReadOptions {
+	opt := C.leveldb_readoptions_create()
+	return &ReadOptions{opt}
+}
+
+func NewWriteOptions() *WriteOptions {
+	opt := C.leveldb_writeoptions_create()
+	return &WriteOptions{opt}
+}
+
+func (o *Options) Close() {
+	C.leveldb_options_destroy(o.Opt)
+}
+
+func (o *Options) SetComparator(cmp *C.leveldb_comparator_t) {
+	C.leveldb_options_set_comparator(o.Opt, cmp)
+}
+
+func (o *Options) SetErrorIfExists(error_if_exists bool) {
+	eie := boolToUchar(error_if_exists)
+	C.leveldb_options_set_error_if_exists(o.Opt, eie)
+}
+
+func (o *Options) SetCache(cache *Cache) {
+	C.leveldb_options_set_cache(o.Opt, cache.Cache)
+}
+
+func (o *Options) SetWriteBufferSize(s int) {
+	C.leveldb_options_set_write_buffer_size(o.Opt, C.size_t(s))
+}
+
+func (o *Options) SetParanoidChecks(pc bool) {
+	C.leveldb_options_set_paranoid_checks(o.Opt, boolToUchar(pc))
+}
+
+func (o *Options) SetMaxOpenFiles(n int) {
+	C.leveldb_options_set_max_open_files(o.Opt, C.int(n))
+}
+
+func (o *Options) SetMaxFileSize(n int) {
+	C.leveldb_options_set_max_file_size(o.Opt, C.size_t(n))
+}
+
+func (o *Options) SetBlockSize(s int) {
+	C.leveldb_options_set_block_size(o.Opt, C.size_t(s))
+}
+
+func (o *Options) SetBlockRestartInterval(n int) {
+	C.leveldb_options_set_block_restart_interval(o.Opt, C.int(n))
+}
+
+func (o *Options) SetCompression(t CompressionOpt) {
+	C.leveldb_options_set_compression(o.Opt, C.int(t))
+}
+
+func (o *Options) SetCreateIfMissing(b bool) {
+	C.leveldb_options_set_create_if_missing(o.Opt, boolToUchar(b))
+}
+
+func (o *Options) SetFilterPolicy(fp *FilterPolicy) {
+	var policy *C.leveldb_filterpolicy_t
+	if fp != nil {
+		policy = fp.Policy
+	}
+	C.leveldb_options_set_filter_policy(o.Opt, policy)
+}
+
+func (ro *ReadOptions) Close() {
+	C.leveldb_readoptions_destroy(ro.Opt)
+}
+
+func (ro *ReadOptions) SetVerifyChecksums(b bool) {
+	C.leveldb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b))
+}
+
+func (ro *ReadOptions) SetFillCache(b bool) {
+	C.leveldb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b))
+}
+
+func (ro *ReadOptions) SetSnapshot(snap *Snapshot) {
+	var s *C.leveldb_snapshot_t
+	if snap != nil {
+		s = snap.snap
+	}
+	C.leveldb_readoptions_set_snapshot(ro.Opt, s)
+}
+
+func (wo *WriteOptions) Close() {
+	C.leveldb_writeoptions_destroy(wo.Opt)
+}
+
+func (wo *WriteOptions) SetSync(b bool) {
+	C.leveldb_writeoptions_set_sync(wo.Opt, boolToUchar(b))
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/slice.go

@@ -0,0 +1,40 @@
+// +build leveldb
+
+package leveldb
+
+// #cgo LDFLAGS: -lleveldb
+// #include "leveldb/c.h"
+import "C"
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+type CSlice struct {
+	data unsafe.Pointer
+	size int
+}
+
+func NewCSlice(p unsafe.Pointer, n int) *CSlice {
+	return &CSlice{p, n}
+}
+
+func (s *CSlice) Data() []byte {
+	var value []byte
+
+	sH := (*reflect.SliceHeader)(unsafe.Pointer(&value))
+	sH.Cap = int(s.size)
+	sH.Len = int(s.size)
+	sH.Data = uintptr(s.data)
+
+	return value
+}
+
+func (s *CSlice) Size() int {
+	return int(s.size)
+}
+
+func (s *CSlice) Free() {
+	C.leveldb_free(s.data)
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/snapshot.go

@@ -0,0 +1,39 @@
+// +build leveldb
+
+package leveldb
+
+// #cgo LDFLAGS: -lleveldb
+// #include "leveldb/c.h"
+import "C"
+
+import (
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+type Snapshot struct {
+	db           *DB
+	snap         *C.leveldb_snapshot_t
+	readOpts     *ReadOptions
+	iteratorOpts *ReadOptions
+}
+
+func (s *Snapshot) Get(key []byte) ([]byte, error) {
+	return s.db.get(s.readOpts, key)
+}
+
+func (s *Snapshot) GetSlice(key []byte) (driver.ISlice, error) {
+	return s.db.getSlice(s.readOpts, key)
+}
+
+func (s *Snapshot) NewIterator() driver.IIterator {
+	it := new(Iterator)
+	it.it = C.leveldb_create_iterator(s.db.db, s.db.iteratorOpts.Opt)
+	return it
+
+}
+
+func (s *Snapshot) Close() {
+	C.leveldb_release_snapshot(s.db.db, s.snap)
+	s.iteratorOpts.Close()
+	s.readOpts.Close()
+}

vendor/github.com/siddontang/ledisdb/store/leveldb/util.go

@@ -0,0 +1,45 @@
+// +build leveldb
+
+package leveldb
+
+// #include "leveldb/c.h"
+import "C"
+
+import (
+	"fmt"
+	"reflect"
+	"unsafe"
+)
+
+func boolToUchar(b bool) C.uchar {
+	uc := C.uchar(0)
+	if b {
+		uc = C.uchar(1)
+	}
+	return uc
+}
+
+func ucharToBool(uc C.uchar) bool {
+	if uc == C.uchar(0) {
+		return false
+	}
+	return true
+}
+
+func saveError(errStr *C.char) error {
+	if errStr != nil {
+		gs := C.GoString(errStr)
+		C.leveldb_free(unsafe.Pointer(errStr))
+		return fmt.Errorf(gs)
+	}
+	return nil
+}
+
+func slice(p unsafe.Pointer, n int) []byte {
+	var b []byte
+	pbyte := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	pbyte.Data = uintptr(p)
+	pbyte.Len = n
+	pbyte.Cap = n
+	return b
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/batch.go

@@ -0,0 +1,83 @@
+// +build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include "rocksdb/c.h"
+// #include "rocksdb_ext.h"
+import "C"
+
+import (
+	"unsafe"
+)
+
+type WriteBatch struct {
+	db       *DB
+	wbatch   *C.rocksdb_writebatch_t
+	commitOk bool
+}
+
+func (w *WriteBatch) Close() {
+	if w.wbatch != nil {
+		C.rocksdb_writebatch_destroy(w.wbatch)
+		w.wbatch = nil
+	}
+}
+
+func (w *WriteBatch) Put(key, value []byte) {
+	w.commitOk = false
+
+	var k, v *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+	if len(value) != 0 {
+		v = (*C.char)(unsafe.Pointer(&value[0]))
+	}
+
+	lenk := len(key)
+	lenv := len(value)
+
+	C.rocksdb_writebatch_put(w.wbatch, k, C.size_t(lenk), v, C.size_t(lenv))
+}
+
+func (w *WriteBatch) Delete(key []byte) {
+	w.commitOk = false
+
+	C.rocksdb_writebatch_delete(w.wbatch,
+		(*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)))
+}
+
+func (w *WriteBatch) Commit() error {
+	return w.commit(w.db.writeOpts)
+}
+
+func (w *WriteBatch) SyncCommit() error {
+	return w.commit(w.db.syncOpts)
+}
+
+func (w *WriteBatch) Rollback() error {
+	if !w.commitOk {
+		C.rocksdb_writebatch_clear(w.wbatch)
+	}
+	return nil
+}
+
+func (w *WriteBatch) commit(wb *WriteOptions) error {
+	w.commitOk = true
+
+	var errStr *C.char
+	C.rocksdb_write_ext(w.db.db, wb.Opt, w.wbatch, &errStr)
+	if errStr != nil {
+		w.commitOk = false
+		return saveError(errStr)
+	}
+	return nil
+}
+
+func (w *WriteBatch) Data() []byte {
+	var vallen C.size_t
+	value := C.rocksdb_writebatch_data(w.wbatch, &vallen)
+
+	return slice(unsafe.Pointer(value), int(vallen))
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/cache.go

@@ -0,0 +1,20 @@
+// +build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include <stdint.h>
+// #include "rocksdb/c.h"
+import "C"
+
+type Cache struct {
+	Cache *C.rocksdb_cache_t
+}
+
+func NewLRUCache(capacity int) *Cache {
+	return &Cache{C.rocksdb_cache_create_lru(C.size_t(capacity))}
+}
+
+func (c *Cache) Close() {
+	C.rocksdb_cache_destroy(c.Cache)
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/const.go

@@ -0,0 +1,3 @@
+package rocksdb
+
+const DBName = "rocksdb"

vendor/github.com/siddontang/ledisdb/store/rocksdb/db.go

@@ -0,0 +1,342 @@
+// +build rocksdb
+
+// Package rocksdb is a wrapper for c++ rocksdb
+package rocksdb
+
+/*
+#cgo LDFLAGS: -lrocksdb
+#include <rocksdb/c.h>
+#include <stdlib.h>
+#include "rocksdb_ext.h"
+*/
+import "C"
+
+import (
+	"os"
+	"runtime"
+	"unsafe"
+
+	"github.com/siddontang/ledisdb/config"
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+const defaultFilterBits int = 10
+
+type Store struct {
+}
+
+func (s Store) String() string {
+	return DBName
+}
+
+func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) {
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return nil, err
+	}
+
+	db := new(DB)
+	db.path = path
+	db.cfg = &cfg.RocksDB
+
+	if err := db.open(); err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func (s Store) Repair(path string, cfg *config.Config) error {
+	db := new(DB)
+	db.path = path
+	db.cfg = &cfg.RocksDB
+
+	err := db.open()
+	defer db.Close()
+
+	//open ok, do not need repair
+	if err == nil {
+		return nil
+	}
+
+	var errStr *C.char
+	ldbname := C.CString(path)
+	defer C.free(unsafe.Pointer(ldbname))
+
+	C.rocksdb_repair_db(db.opts.Opt, ldbname, &errStr)
+	if errStr != nil {
+		return saveError(errStr)
+	}
+	return nil
+}
+
+type DB struct {
+	path string
+
+	cfg *config.RocksDBConfig
+
+	db *C.rocksdb_t
+
+	env *Env
+
+	opts      *Options
+	blockOpts *BlockBasedTableOptions
+
+	//for default read and write options
+	readOpts     *ReadOptions
+	writeOpts    *WriteOptions
+	iteratorOpts *ReadOptions
+
+	syncOpts *WriteOptions
+
+	cache *Cache
+
+	filter *FilterPolicy
+}
+
+func (db *DB) open() error {
+	db.initOptions(db.cfg)
+
+	var errStr *C.char
+	ldbname := C.CString(db.path)
+	defer C.free(unsafe.Pointer(ldbname))
+
+	db.db = C.rocksdb_open(db.opts.Opt, ldbname, &errStr)
+	if errStr != nil {
+		db.db = nil
+		return saveError(errStr)
+	}
+	return nil
+}
+
+func (db *DB) initOptions(cfg *config.RocksDBConfig) {
+	opts := NewOptions()
+	blockOpts := NewBlockBasedTableOptions()
+
+	opts.SetCreateIfMissing(true)
+
+	db.env = NewDefaultEnv()
+	db.env.SetBackgroundThreads(cfg.BackgroundThreads)
+	db.env.SetHighPriorityBackgroundThreads(cfg.HighPriorityBackgroundThreads)
+	opts.SetEnv(db.env)
+
+	db.cache = NewLRUCache(cfg.CacheSize)
+	blockOpts.SetCache(db.cache)
+
+	//we must use bloomfilter
+	db.filter = NewBloomFilter(defaultFilterBits)
+	blockOpts.SetFilterPolicy(db.filter)
+	blockOpts.SetBlockSize(cfg.BlockSize)
+	opts.SetBlockBasedTableFactory(blockOpts)
+
+	opts.SetCompression(CompressionOpt(cfg.Compression))
+	opts.SetWriteBufferSize(cfg.WriteBufferSize)
+	opts.SetMaxOpenFiles(cfg.MaxOpenFiles)
+	opts.SetMaxBackgroundCompactions(cfg.MaxBackgroundCompactions)
+	opts.SetMaxBackgroundFlushes(cfg.MaxBackgroundFlushes)
+	opts.SetLevel0FileNumCompactionTrigger(cfg.Level0FileNumCompactionTrigger)
+	opts.SetLevel0SlowdownWritesTrigger(cfg.Level0SlowdownWritesTrigger)
+	opts.SetLevel0StopWritesTrigger(cfg.Level0StopWritesTrigger)
+	opts.SetTargetFileSizeBase(cfg.TargetFileSizeBase)
+	opts.SetTargetFileSizeMultiplier(cfg.TargetFileSizeMultiplier)
+	opts.SetMaxBytesForLevelBase(cfg.MaxBytesForLevelBase)
+	opts.SetMaxBytesForLevelMultiplier(cfg.MaxBytesForLevelMultiplier)
+	opts.SetMinWriteBufferNumberToMerge(cfg.MinWriteBufferNumberToMerge)
+	opts.DisableAutoCompactions(cfg.DisableAutoCompactions)
+	opts.EnableStatistics(cfg.EnableStatistics)
+	opts.UseFsync(cfg.UseFsync)
+	opts.SetStatsDumpPeriodSec(cfg.StatsDumpPeriodSec)
+	opts.SetMaxManifestFileSize(cfg.MaxManifestFileSize)
+
+	db.opts = opts
+	db.blockOpts = blockOpts
+
+	db.readOpts = NewReadOptions()
+	db.writeOpts = NewWriteOptions()
+	db.writeOpts.DisableWAL(cfg.DisableWAL)
+
+	db.syncOpts = NewWriteOptions()
+	db.syncOpts.SetSync(true)
+	db.syncOpts.DisableWAL(cfg.DisableWAL)
+
+	db.iteratorOpts = NewReadOptions()
+	db.iteratorOpts.SetFillCache(false)
+}
+
+func (db *DB) Close() error {
+	if db.db != nil {
+		C.rocksdb_close(db.db)
+		db.db = nil
+	}
+
+	if db.filter != nil {
+		db.filter.Close()
+	}
+
+	if db.cache != nil {
+		db.cache.Close()
+	}
+
+	if db.env != nil {
+		db.env.Close()
+	}
+
+	//db.blockOpts.Close()
+
+	db.opts.Close()
+
+	db.readOpts.Close()
+	db.writeOpts.Close()
+	db.iteratorOpts.Close()
+
+	return nil
+}
+
+func (db *DB) Put(key, value []byte) error {
+	return db.put(db.writeOpts, key, value)
+}
+
+func (db *DB) Get(key []byte) ([]byte, error) {
+	return db.get(db.readOpts, key)
+}
+
+func (db *DB) Delete(key []byte) error {
+	return db.delete(db.writeOpts, key)
+}
+
+func (db *DB) SyncPut(key []byte, value []byte) error {
+	return db.put(db.syncOpts, key, value)
+}
+
+func (db *DB) SyncDelete(key []byte) error {
+	return db.delete(db.syncOpts, key)
+}
+
+func (db *DB) NewWriteBatch() driver.IWriteBatch {
+	wb := &WriteBatch{
+		db:     db,
+		wbatch: C.rocksdb_writebatch_create(),
+	}
+
+	runtime.SetFinalizer(wb, func(w *WriteBatch) {
+		w.Close()
+	})
+
+	return wb
+}
+
+func (db *DB) NewIterator() driver.IIterator {
+	it := new(Iterator)
+
+	it.it = C.rocksdb_create_iterator(db.db, db.iteratorOpts.Opt)
+
+	return it
+}
+
+func (db *DB) NewSnapshot() (driver.ISnapshot, error) {
+	snap := &Snapshot{
+		db:           db,
+		snap:         C.rocksdb_create_snapshot(db.db),
+		readOpts:     NewReadOptions(),
+		iteratorOpts: NewReadOptions(),
+	}
+	snap.readOpts.SetSnapshot(snap)
+	snap.iteratorOpts.SetSnapshot(snap)
+	snap.iteratorOpts.SetFillCache(false)
+
+	return snap, nil
+}
+
+func (db *DB) put(wo *WriteOptions, key, value []byte) error {
+	var errStr *C.char
+	var k, v *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+	if len(value) != 0 {
+		v = (*C.char)(unsafe.Pointer(&value[0]))
+	}
+
+	lenk := len(key)
+	lenv := len(value)
+	C.rocksdb_put(
+		db.db, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr)
+
+	if errStr != nil {
+		return saveError(errStr)
+	}
+	return nil
+}
+
+func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) {
+	var errStr *C.char
+	var vallen C.size_t
+	var k *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+
+	value := C.rocksdb_get(
+		db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr)
+
+	if errStr != nil {
+		return nil, saveError(errStr)
+	}
+
+	if value == nil {
+		return nil, nil
+	}
+
+	defer C.free(unsafe.Pointer(value))
+	return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil
+}
+
+func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) {
+	var errStr *C.char
+	var vallen C.size_t
+	var k *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+
+	value := C.rocksdb_get(
+		db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr)
+
+	if errStr != nil {
+		return nil, saveError(errStr)
+	}
+
+	if value == nil {
+		return nil, nil
+	}
+
+	return NewCSlice(unsafe.Pointer(value), int(vallen)), nil
+}
+
+func (db *DB) delete(wo *WriteOptions, key []byte) error {
+	var errStr *C.char
+	var k *C.char
+	if len(key) != 0 {
+		k = (*C.char)(unsafe.Pointer(&key[0]))
+	}
+
+	C.rocksdb_delete(
+		db.db, wo.Opt, k, C.size_t(len(key)), &errStr)
+
+	if errStr != nil {
+		return saveError(errStr)
+	}
+	return nil
+}
+
+func (db *DB) Compact() error {
+	C.rocksdb_compact_range(db.db, nil, 0, nil, 0)
+	return nil
+}
+
+func (db *DB) GetSlice(key []byte) (driver.ISlice, error) {
+	return db.getSlice(db.readOpts, key)
+}
+
+func init() {
+	driver.Register(Store{})
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/env.go

@@ -0,0 +1,27 @@
+// +build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include "rocksdb/c.h"
+import "C"
+
+type Env struct {
+	Env *C.rocksdb_env_t
+}
+
+func NewDefaultEnv() *Env {
+	return &Env{C.rocksdb_create_default_env()}
+}
+
+func (env *Env) SetHighPriorityBackgroundThreads(n int) {
+	C.rocksdb_env_set_high_priority_background_threads(env.Env, C.int(n))
+}
+
+func (env *Env) SetBackgroundThreads(n int) {
+	C.rocksdb_env_set_background_threads(env.Env, C.int(n))
+}
+
+func (env *Env) Close() {
+	C.rocksdb_env_destroy(env.Env)
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/filterpolicy.go

@@ -0,0 +1,21 @@
+// +build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include <stdlib.h>
+// #include "rocksdb/c.h"
+import "C"
+
+type FilterPolicy struct {
+	Policy *C.rocksdb_filterpolicy_t
+}
+
+func NewBloomFilter(bitsPerKey int) *FilterPolicy {
+	policy := C.rocksdb_filterpolicy_create_bloom(C.int(bitsPerKey))
+	return &FilterPolicy{policy}
+}
+
+func (fp *FilterPolicy) Close() {
+	C.rocksdb_filterpolicy_destroy(fp.Policy)
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/iterator.go

@@ -0,0 +1,70 @@
+// +build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include <stdlib.h>
+// #include "rocksdb/c.h"
+// #include "rocksdb_ext.h"
+import "C"
+
+import (
+	"unsafe"
+)
+
+type Iterator struct {
+	it      *C.rocksdb_iterator_t
+	isValid C.uchar
+}
+
+func (it *Iterator) Key() []byte {
+	var klen C.size_t
+	kdata := C.rocksdb_iter_key(it.it, &klen)
+	if kdata == nil {
+		return nil
+	}
+
+	return slice(unsafe.Pointer(kdata), int(C.int(klen)))
+}
+
+func (it *Iterator) Value() []byte {
+	var vlen C.size_t
+	vdata := C.rocksdb_iter_value(it.it, &vlen)
+	if vdata == nil {
+		return nil
+	}
+
+	return slice(unsafe.Pointer(vdata), int(C.int(vlen)))
+}
+
+func (it *Iterator) Close() error {
+	if it.it != nil {
+		C.rocksdb_iter_destroy(it.it)
+		it.it = nil
+	}
+	return nil
+}
+
+func (it *Iterator) Valid() bool {
+	return ucharToBool(it.isValid)
+}
+
+func (it *Iterator) Next() {
+	it.isValid = C.rocksdb_iter_next_ext(it.it)
+}
+
+func (it *Iterator) Prev() {
+	it.isValid = C.rocksdb_iter_prev_ext(it.it)
+}
+
+func (it *Iterator) First() {
+	it.isValid = C.rocksdb_iter_seek_to_first_ext(it.it)
+}
+
+func (it *Iterator) Last() {
+	it.isValid = C.rocksdb_iter_seek_to_last_ext(it.it)
+}
+
+func (it *Iterator) Seek(key []byte) {
+	it.isValid = C.rocksdb_iter_seek_ext(it.it, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)))
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/options.go

@@ -0,0 +1,229 @@
+// +build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include "rocksdb/c.h"
+import "C"
+
+type CompressionOpt int
+
+const (
+	NoCompression     = CompressionOpt(0)
+	SnappyCompression = CompressionOpt(1)
+	ZlibCompression   = CompressionOpt(2)
+	Bz2Compression    = CompressionOpt(3)
+	Lz4Compression    = CompressionOpt(4)
+	Lz4hcCompression  = CompressionOpt(5)
+)
+
+type Options struct {
+	Opt *C.rocksdb_options_t
+}
+
+type ReadOptions struct {
+	Opt *C.rocksdb_readoptions_t
+}
+
+type WriteOptions struct {
+	Opt *C.rocksdb_writeoptions_t
+}
+
+type BlockBasedTableOptions struct {
+	Opt *C.rocksdb_block_based_table_options_t
+}
+
+func NewOptions() *Options {
+	opt := C.rocksdb_options_create()
+	return &Options{opt}
+}
+
+func NewReadOptions() *ReadOptions {
+	opt := C.rocksdb_readoptions_create()
+	return &ReadOptions{opt}
+}
+
+func NewWriteOptions() *WriteOptions {
+	opt := C.rocksdb_writeoptions_create()
+	return &WriteOptions{opt}
+}
+
+func NewBlockBasedTableOptions() *BlockBasedTableOptions {
+	opt := C.rocksdb_block_based_options_create()
+	return &BlockBasedTableOptions{opt}
+}
+
+func (o *Options) Close() {
+	C.rocksdb_options_destroy(o.Opt)
+}
+
+func (o *Options) IncreaseParallelism(n int) {
+	C.rocksdb_options_increase_parallelism(o.Opt, C.int(n))
+}
+
+func (o *Options) OptimizeLevelStyleCompaction(n int) {
+	C.rocksdb_options_optimize_level_style_compaction(o.Opt, C.uint64_t(n))
+}
+
+func (o *Options) SetComparator(cmp *C.rocksdb_comparator_t) {
+	C.rocksdb_options_set_comparator(o.Opt, cmp)
+}
+
+func (o *Options) SetErrorIfExists(error_if_exists bool) {
+	eie := boolToUchar(error_if_exists)
+	C.rocksdb_options_set_error_if_exists(o.Opt, eie)
+}
+
+func (o *Options) SetEnv(env *Env) {
+	C.rocksdb_options_set_env(o.Opt, env.Env)
+}
+
+func (o *Options) SetWriteBufferSize(s int) {
+	C.rocksdb_options_set_write_buffer_size(o.Opt, C.size_t(s))
+}
+
+func (o *Options) SetParanoidChecks(pc bool) {
+	C.rocksdb_options_set_paranoid_checks(o.Opt, boolToUchar(pc))
+}
+
+func (o *Options) SetMaxOpenFiles(n int) {
+	C.rocksdb_options_set_max_open_files(o.Opt, C.int(n))
+}
+
+func (o *Options) SetCompression(t CompressionOpt) {
+	C.rocksdb_options_set_compression(o.Opt, C.int(t))
+}
+
+func (o *Options) SetCreateIfMissing(b bool) {
+	C.rocksdb_options_set_create_if_missing(o.Opt, boolToUchar(b))
+}
+
+func (o *Options) SetMaxWriteBufferNumber(n int) {
+	C.rocksdb_options_set_max_write_buffer_number(o.Opt, C.int(n))
+}
+
+func (o *Options) SetMaxBackgroundCompactions(n int) {
+	C.rocksdb_options_set_max_background_compactions(o.Opt, C.int(n))
+}
+
+func (o *Options) SetMaxBackgroundFlushes(n int) {
+	C.rocksdb_options_set_max_background_flushes(o.Opt, C.int(n))
+}
+
+func (o *Options) SetNumLevels(n int) {
+	C.rocksdb_options_set_num_levels(o.Opt, C.int(n))
+}
+
+func (o *Options) SetLevel0FileNumCompactionTrigger(n int) {
+	C.rocksdb_options_set_level0_file_num_compaction_trigger(o.Opt, C.int(n))
+}
+
+func (o *Options) SetLevel0SlowdownWritesTrigger(n int) {
+	C.rocksdb_options_set_level0_slowdown_writes_trigger(o.Opt, C.int(n))
+}
+
+func (o *Options) SetLevel0StopWritesTrigger(n int) {
+	C.rocksdb_options_set_level0_stop_writes_trigger(o.Opt, C.int(n))
+}
+
+func (o *Options) SetTargetFileSizeBase(n int) {
+	C.rocksdb_options_set_target_file_size_base(o.Opt, C.uint64_t(uint64(n)))
+}
+
+func (o *Options) SetTargetFileSizeMultiplier(n int) {
+	C.rocksdb_options_set_target_file_size_multiplier(o.Opt, C.int(n))
+}
+
+func (o *Options) SetMaxBytesForLevelBase(n int) {
+	C.rocksdb_options_set_max_bytes_for_level_base(o.Opt, C.uint64_t(uint64(n)))
+}
+
+func (o *Options) SetMaxBytesForLevelMultiplier(n int) {
+	C.rocksdb_options_set_max_bytes_for_level_multiplier(o.Opt, C.double(n))
+}
+
+func (o *Options) SetBlockBasedTableFactory(opt *BlockBasedTableOptions) {
+	C.rocksdb_options_set_block_based_table_factory(o.Opt, opt.Opt)
+}
+
+func (o *Options) SetMinWriteBufferNumberToMerge(n int) {
+	C.rocksdb_options_set_min_write_buffer_number_to_merge(o.Opt, C.int(n))
+}
+
+func (o *Options) DisableAutoCompactions(b bool) {
+	C.rocksdb_options_set_disable_auto_compactions(o.Opt, boolToInt(b))
+}
+
+func (o *Options) UseFsync(b bool) {
+	C.rocksdb_options_set_use_fsync(o.Opt, boolToInt(b))
+}
+
+func (o *Options) EnableStatistics(b bool) {
+	if b {
+		C.rocksdb_options_enable_statistics(o.Opt)
+	}
+}
+
+func (o *Options) SetStatsDumpPeriodSec(n int) {
+	C.rocksdb_options_set_stats_dump_period_sec(o.Opt, C.uint(n))
+}
+
+func (o *Options) SetMaxManifestFileSize(n int) {
+	C.rocksdb_options_set_max_manifest_file_size(o.Opt, C.size_t(n))
+}
+
+func (o *BlockBasedTableOptions) Close() {
+	C.rocksdb_block_based_options_destroy(o.Opt)
+}
+
+func (o *BlockBasedTableOptions) SetFilterPolicy(fp *FilterPolicy) {
+	var policy *C.rocksdb_filterpolicy_t
+	if fp != nil {
+		policy = fp.Policy
+	}
+	C.rocksdb_block_based_options_set_filter_policy(o.Opt, policy)
+}
+
+func (o *BlockBasedTableOptions) SetBlockSize(s int) {
+	C.rocksdb_block_based_options_set_block_size(o.Opt, C.size_t(s))
+}
+
+func (o *BlockBasedTableOptions) SetBlockRestartInterval(n int) {
+	C.rocksdb_block_based_options_set_block_restart_interval(o.Opt, C.int(n))
+}
+
+func (o *BlockBasedTableOptions) SetCache(cache *Cache) {
+	C.rocksdb_block_based_options_set_block_cache(o.Opt, cache.Cache)
+}
+
+func (ro *ReadOptions) Close() {
+	C.rocksdb_readoptions_destroy(ro.Opt)
+}
+
+func (ro *ReadOptions) SetVerifyChecksums(b bool) {
+	C.rocksdb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b))
+}
+
+func (ro *ReadOptions) SetFillCache(b bool) {
+	C.rocksdb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b))
+}
+
+func (ro *ReadOptions) SetSnapshot(snap *Snapshot) {
+	var s *C.rocksdb_snapshot_t
+	if snap != nil {
+		s = snap.snap
+	}
+	C.rocksdb_readoptions_set_snapshot(ro.Opt, s)
+}
+
+func (wo *WriteOptions) Close() {
+	C.rocksdb_writeoptions_destroy(wo.Opt)
+}
+
+func (wo *WriteOptions) SetSync(b bool) {
+	C.rocksdb_writeoptions_set_sync(wo.Opt, boolToUchar(b))
+}
+
+func (wo *WriteOptions) DisableWAL(b bool) {
+	C.rocksdb_writeoptions_disable_WAL(wo.Opt, boolToInt(b))
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.cc

@@ -0,0 +1,44 @@
+// +build rocksdb
+
+#include "rocksdb_ext.h"
+
+#include <stdlib.h>
+#include <string>
+
+extern "C" {
+
+unsigned char rocksdb_iter_seek_to_first_ext(rocksdb_iterator_t* iter) {
+    rocksdb_iter_seek_to_first(iter);
+    return rocksdb_iter_valid(iter);
+}
+
+unsigned char rocksdb_iter_seek_to_last_ext(rocksdb_iterator_t* iter) {
+    rocksdb_iter_seek_to_last(iter);
+    return rocksdb_iter_valid(iter);
+}
+
+unsigned char rocksdb_iter_seek_ext(rocksdb_iterator_t* iter, const char* k, size_t klen) {
+    rocksdb_iter_seek(iter, k, klen);
+    return rocksdb_iter_valid(iter);
+}
+
+unsigned char rocksdb_iter_next_ext(rocksdb_iterator_t* iter) {
+    rocksdb_iter_next(iter);
+    return rocksdb_iter_valid(iter);
+}
+
+unsigned char rocksdb_iter_prev_ext(rocksdb_iterator_t* iter) {
+    rocksdb_iter_prev(iter);
+    return rocksdb_iter_valid(iter);
+}
+
+void rocksdb_write_ext(rocksdb_t* db, 
+    const rocksdb_writeoptions_t* options, 
+    rocksdb_writebatch_t* batch, char** errptr) {
+    rocksdb_write(db, options, batch, errptr);
+    if(*errptr == NULL) {
+        rocksdb_writebatch_clear(batch);
+    }
+}
+
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.h

@@ -0,0 +1,24 @@
+// +build rocksdb
+
+#ifndef ROCKSDB_EXT_H
+#define ROCKSDB_EXT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "rocksdb/c.h"
+
+// Below iterator functions like rocksdb iterator but returns valid status for iterator
+extern unsigned char rocksdb_iter_seek_to_first_ext(rocksdb_iterator_t*);
+extern unsigned char rocksdb_iter_seek_to_last_ext(rocksdb_iterator_t*);
+extern unsigned char rocksdb_iter_seek_ext(rocksdb_iterator_t*, const char* k, size_t klen);
+extern unsigned char rocksdb_iter_next_ext(rocksdb_iterator_t*);
+extern unsigned char rocksdb_iter_prev_ext(rocksdb_iterator_t*);
+extern void rocksdb_write_ext(rocksdb_t* db, const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, char** errptr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

vendor/github.com/siddontang/ledisdb/store/rocksdb/slice.go

@@ -0,0 +1,41 @@
+//+build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include <rocksdb/c.h>
+// #include <stdlib.h>
+import "C"
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+type CSlice struct {
+	data unsafe.Pointer
+	size int
+}
+
+func NewCSlice(p unsafe.Pointer, n int) *CSlice {
+	return &CSlice{p, n}
+}
+
+func (s *CSlice) Data() []byte {
+	var value []byte
+
+	sH := (*reflect.SliceHeader)(unsafe.Pointer(&value))
+	sH.Cap = int(s.size)
+	sH.Len = int(s.size)
+	sH.Data = uintptr(s.data)
+
+	return value
+}
+
+func (s *CSlice) Size() int {
+	return int(s.size)
+}
+
+func (s *CSlice) Free() {
+	C.free(s.data)
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/snapshot.go

@@ -0,0 +1,39 @@
+// +build rocksdb
+
+package rocksdb
+
+// #cgo LDFLAGS: -lrocksdb
+// #include "rocksdb/c.h"
+import "C"
+
+import (
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+type Snapshot struct {
+	db           *DB
+	snap         *C.rocksdb_snapshot_t
+	readOpts     *ReadOptions
+	iteratorOpts *ReadOptions
+}
+
+func (s *Snapshot) Get(key []byte) ([]byte, error) {
+	return s.db.get(s.readOpts, key)
+}
+
+func (s *Snapshot) GetSlice(key []byte) (driver.ISlice, error) {
+	return s.db.getSlice(s.readOpts, key)
+}
+
+func (s *Snapshot) NewIterator() driver.IIterator {
+	it := new(Iterator)
+	it.it = C.rocksdb_create_iterator(s.db.db, s.db.iteratorOpts.Opt)
+	return it
+
+}
+
+func (s *Snapshot) Close() {
+	C.rocksdb_release_snapshot(s.db.db, s.snap)
+	s.iteratorOpts.Close()
+	s.readOpts.Close()
+}

vendor/github.com/siddontang/ledisdb/store/rocksdb/util.go

@@ -0,0 +1,54 @@
+// +build rocksdb
+
+package rocksdb
+
+// #include <stdlib.h>
+// #include "rocksdb/c.h"
+import "C"
+
+import (
+	"fmt"
+	"reflect"
+	"unsafe"
+)
+
+func boolToUchar(b bool) C.uchar {
+	uc := C.uchar(0)
+	if b {
+		uc = C.uchar(1)
+	}
+	return uc
+}
+
+func ucharToBool(uc C.uchar) bool {
+	if uc == C.uchar(0) {
+		return false
+	}
+	return true
+}
+
+func boolToInt(b bool) C.int {
+	uc := C.int(0)
+	if b {
+		uc = C.int(1)
+	}
+	return uc
+}
+
+func saveError(errStr *C.char) error {
+	if errStr != nil {
+		gs := C.GoString(errStr)
+		C.free(unsafe.Pointer(errStr))
+		return fmt.Errorf(gs)
+	}
+	return nil
+}
+
+func slice(p unsafe.Pointer, n int) []byte {
+	var b []byte
+	pbyte := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	pbyte.Data = uintptr(p)
+	pbyte.Len = n
+	pbyte.Cap = n
+	return b
+}

vendor/github.com/siddontang/ledisdb/store/slice.go

@@ -0,0 +1,9 @@
+package store
+
+import (
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+type Slice interface {
+	driver.ISlice
+}

vendor/github.com/siddontang/ledisdb/store/snapshot.go

@@ -0,0 +1,48 @@
+package store
+
+import (
+	"github.com/siddontang/ledisdb/store/driver"
+)
+
+type Snapshot struct {
+	driver.ISnapshot
+	st *Stat
+}
+
+func (s *Snapshot) NewIterator() *Iterator {
+	it := new(Iterator)
+	it.it = s.ISnapshot.NewIterator()
+	it.st = s.st
+
+	s.st.IterNum.Add(1)
+
+	return it
+}
+
+func (s *Snapshot) Get(key []byte) ([]byte, error) {
+	v, err := s.ISnapshot.Get(key)
+	s.st.statGet(v, err)
+	return v, err
+}
+
+func (s *Snapshot) GetSlice(key []byte) (Slice, error) {
+	if d, ok := s.ISnapshot.(driver.ISliceGeter); ok {
+		v, err := d.GetSlice(key)
+		s.st.statGet(v, err)
+		return v, err
+	} else {
+		v, err := s.Get(key)
+		if err != nil {
+			return nil, err
+		} else if v == nil {
+			return nil, nil
+		} else {
+			return driver.GoSlice(v), nil
+		}
+	}
+}
+
+func (s *Snapshot) Close() {
+	s.st.SnapshotCloseNum.Add(1)
+	s.ISnapshot.Close()
+}

vendor/github.com/siddontang/ledisdb/store/stat.go

@@ -0,0 +1,37 @@
+package store
+
+import (
+	"github.com/siddontang/go/sync2"
+)
+
+type Stat struct {
+	GetNum               sync2.AtomicInt64
+	GetMissingNum        sync2.AtomicInt64
+	GetTotalTime         sync2.AtomicDuration
+	PutNum               sync2.AtomicInt64
+	DeleteNum            sync2.AtomicInt64
+	IterNum              sync2.AtomicInt64
+	IterSeekNum          sync2.AtomicInt64
+	IterCloseNum         sync2.AtomicInt64
+	SnapshotNum          sync2.AtomicInt64
+	SnapshotCloseNum     sync2.AtomicInt64
+	BatchNum             sync2.AtomicInt64
+	BatchCommitNum       sync2.AtomicInt64
+	BatchCommitTotalTime sync2.AtomicDuration
+	TxNum                sync2.AtomicInt64
+	TxCommitNum          sync2.AtomicInt64
+	TxCloseNum           sync2.AtomicInt64
+	CompactNum           sync2.AtomicInt64
+	CompactTotalTime     sync2.AtomicDuration
+}
+
+func (st *Stat) statGet(v interface{}, err error) {
+	st.GetNum.Add(1)
+	if v == nil && err == nil {
+		st.GetMissingNum.Add(1)
+	}
+}
+
+func (st *Stat) Reset() {
+	*st = Stat{}
+}

vendor/github.com/siddontang/ledisdb/store/store.go

@@ -0,0 +1,62 @@
+package store
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/siddontang/ledisdb/config"
+	"github.com/siddontang/ledisdb/store/driver"
+
+	_ "github.com/siddontang/ledisdb/store/goleveldb"
+	_ "github.com/siddontang/ledisdb/store/leveldb"
+	_ "github.com/siddontang/ledisdb/store/rocksdb"
+)
+
+func getStorePath(cfg *config.Config) string {
+	if len(cfg.DBPath) > 0 {
+		return cfg.DBPath
+	} else {
+		return path.Join(cfg.DataDir, fmt.Sprintf("%s_data", cfg.DBName))
+	}
+}
+
+func Open(cfg *config.Config) (*DB, error) {
+	s, err := driver.GetStore(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	path := getStorePath(cfg)
+
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return nil, err
+	}
+
+	idb, err := s.Open(path, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	db := new(DB)
+	db.db = idb
+	db.name = s.String()
+	db.st = &Stat{}
+	db.cfg = cfg
+
+	return db, nil
+}
+
+func Repair(cfg *config.Config) error {
+	s, err := driver.GetStore(cfg)
+	if err != nil {
+		return err
+	}
+
+	path := getStorePath(cfg)
+
+	return s.Repair(path, cfg)
+}
+
+func init() {
+}

vendor/github.com/siddontang/ledisdb/store/writebatch.go

@@ -0,0 +1,136 @@
+package store
+
+import (
+	"time"
+
+	"github.com/siddontang/ledisdb/store/driver"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type WriteBatch struct {
+	wb driver.IWriteBatch
+	st *Stat
+
+	putNum    int64
+	deleteNum int64
+	db        *DB
+
+	data *BatchData
+}
+
+func (wb *WriteBatch) Close() {
+	wb.wb.Close()
+}
+
+func (wb *WriteBatch) Put(key []byte, value []byte) {
+	wb.putNum++
+	wb.wb.Put(key, value)
+}
+
+func (wb *WriteBatch) Delete(key []byte) {
+	wb.deleteNum++
+	wb.wb.Delete(key)
+}
+
+func (wb *WriteBatch) Commit() error {
+	wb.st.BatchCommitNum.Add(1)
+	wb.st.PutNum.Add(wb.putNum)
+	wb.st.DeleteNum.Add(wb.deleteNum)
+	wb.putNum = 0
+	wb.deleteNum = 0
+
+	var err error
+	t := time.Now()
+	if wb.db == nil || !wb.db.needSyncCommit() {
+		err = wb.wb.Commit()
+	} else {
+		err = wb.wb.SyncCommit()
+	}
+
+	wb.st.BatchCommitTotalTime.Add(time.Now().Sub(t))
+
+	return err
+}
+
+func (wb *WriteBatch) Rollback() error {
+	wb.putNum = 0
+	wb.deleteNum = 0
+
+	return wb.wb.Rollback()
+}
+
+// the data will be undefined after commit or rollback
+func (wb *WriteBatch) BatchData() *BatchData {
+	data := wb.wb.Data()
+	if wb.data == nil {
+		wb.data = new(BatchData)
+	}
+
+	wb.data.Load(data)
+	return wb.data
+}
+
+func (wb *WriteBatch) Data() []byte {
+	b := wb.BatchData()
+	return b.Data()
+}
+
+/*
+	see leveldb batch data format for more information
+*/
+
+type BatchData struct {
+	leveldb.Batch
+}
+
+func NewBatchData(data []byte) (*BatchData, error) {
+	b := new(BatchData)
+
+	if err := b.Load(data); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (d *BatchData) Data() []byte {
+	return d.Dump()
+}
+
+func (d *BatchData) Reset() {
+	d.Batch.Reset()
+}
+
+type BatchDataReplay interface {
+	Put(key, value []byte)
+	Delete(key []byte)
+}
+
+type BatchItem struct {
+	Key   []byte
+	Value []byte
+}
+
+type batchItems []BatchItem
+
+func (bs *batchItems) Put(key, value []byte) {
+	*bs = append(*bs, BatchItem{key, value})
+}
+
+func (bs *batchItems) Delete(key []byte) {
+	*bs = append(*bs, BatchItem{key, nil})
+}
+
+func (d *BatchData) Replay(r BatchDataReplay) error {
+	return d.Batch.Replay(r)
+}
+
+func (d *BatchData) Items() ([]BatchItem, error) {
+	is := make(batchItems, 0, d.Len())
+
+	if err := d.Replay(&is); err != nil {
+		return nil, err
+	}
+
+	return []BatchItem(is), nil
+}