lbsadmin/Beego
1
0
Fork 0
mirror of https://github.com/astaxie/beego.git synced 2025-06-28 10:30:19 +00:00
Code Issues Releases Activity

add vendor

Browse Source
This commit is contained in:
astaxie
2018-07-30 12:05:51 +08:00
parent d55f54a8ab
commit 48acfa08be
496 changed files with 327583 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
vendor/github.com/couchbase/gomemcached/LICENSE generated vendored Normal file
View File

@ -0,0 +1,19 @@
Copyright (c) 2013 Dustin Sallings
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

32
vendor/github.com/couchbase/gomemcached/README.markdown generated vendored Normal file
View File

@ -0,0 +1,32 @@
# gomemcached
This is a memcached binary protocol toolkit in [go][go].
It provides client and server functionality as well as a little sample
server showing how I might make a server if I valued purity over
performance.
## Server Design
<div>
<img src="http://dustin.github.com/images/gomemcached.png"
alt="overview" style="float: right"/>
</div>
The basic design can be seen in [gocache]. A [storage
server][storage] is run as a goroutine that receives a `MCRequest` on
a channel, and then issues an `MCResponse` to a channel contained
within the request.
Each connection is a separate goroutine, of course, and is responsible
for all IO for that connection until the connection drops or the
`dataServer` decides it's stupid and sends a fatal response back over
the channel.
There is currently no work at all in making the thing perform (there
are specific areas I know need work). This is just my attempt to
learn the language somewhat.
[go]: http://golang.org/
[gocache]: gomemcached/blob/master/gocache/gocache.go
[storage]: gomemcached/blob/master/gocache/mc_storage.go

1146
vendor/github.com/couchbase/gomemcached/client/mc.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

333
vendor/github.com/couchbase/gomemcached/client/tap_feed.go generated vendored Normal file
View File

@ -0,0 +1,333 @@
package memcached
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"math"
"github.com/couchbase/gomemcached"
"github.com/couchbase/goutils/logging"
)
// TAP protocol docs: <http://www.couchbase.com/wiki/display/couchbase/TAP+Protocol>
// TapOpcode is the tap operation type (found in TapEvent)
type TapOpcode uint8
// Tap opcode values.
const (
TapBeginBackfill = TapOpcode(iota)
TapEndBackfill
TapMutation
TapDeletion
TapCheckpointStart
TapCheckpointEnd
tapEndStream
)
const tapMutationExtraLen = 16
var tapOpcodeNames map[TapOpcode]string
func init() {
tapOpcodeNames = map[TapOpcode]string{
TapBeginBackfill: "BeginBackfill",
TapEndBackfill: "EndBackfill",
TapMutation: "Mutation",
TapDeletion: "Deletion",
TapCheckpointStart: "TapCheckpointStart",
TapCheckpointEnd: "TapCheckpointEnd",
tapEndStream: "EndStream",
}
}
func (opcode TapOpcode) String() string {
name := tapOpcodeNames[opcode]
if name == "" {
name = fmt.Sprintf("#%d", opcode)
}
return name
}
// TapEvent is a TAP notification of an operation on the server.
type TapEvent struct {
Opcode TapOpcode // Type of event
VBucket uint16 // VBucket this event applies to
Flags uint32 // Item flags
Expiry uint32 // Item expiration time
Key, Value []byte // Item key/value
Cas uint64
}
func makeTapEvent(req gomemcached.MCRequest) *TapEvent {
event := TapEvent{
VBucket: req.VBucket,
}
switch req.Opcode {
case gomemcached.TAP_MUTATION:
event.Opcode = TapMutation
event.Key = req.Key
event.Value = req.Body
event.Cas = req.Cas
case gomemcached.TAP_DELETE:
event.Opcode = TapDeletion
event.Key = req.Key
event.Cas = req.Cas
case gomemcached.TAP_CHECKPOINT_START:
event.Opcode = TapCheckpointStart
case gomemcached.TAP_CHECKPOINT_END:
event.Opcode = TapCheckpointEnd
case gomemcached.TAP_OPAQUE:
if len(req.Extras) < 8+4 {
return nil
}
switch op := int(binary.BigEndian.Uint32(req.Extras[8:])); op {
case gomemcached.TAP_OPAQUE_INITIAL_VBUCKET_STREAM:
event.Opcode = TapBeginBackfill
case gomemcached.TAP_OPAQUE_CLOSE_BACKFILL:
event.Opcode = TapEndBackfill
case gomemcached.TAP_OPAQUE_CLOSE_TAP_STREAM:
event.Opcode = tapEndStream
case gomemcached.TAP_OPAQUE_ENABLE_AUTO_NACK:
return nil
case gomemcached.TAP_OPAQUE_ENABLE_CHECKPOINT_SYNC:
return nil
default:
logging.Infof("TapFeed: Ignoring TAP_OPAQUE/%d", op)
return nil // unknown opaque event
}
case gomemcached.NOOP:
return nil // ignore
default:
logging.Infof("TapFeed: Ignoring %s", req.Opcode)
return nil // unknown event
}
if len(req.Extras) >= tapMutationExtraLen &&
(event.Opcode == TapMutation || event.Opcode == TapDeletion) {
event.Flags = binary.BigEndian.Uint32(req.Extras[8:])
event.Expiry = binary.BigEndian.Uint32(req.Extras[12:])
}
return &event
}
func (event TapEvent) String() string {
switch event.Opcode {
case TapBeginBackfill, TapEndBackfill, TapCheckpointStart, TapCheckpointEnd:
return fmt.Sprintf("<TapEvent %s, vbucket=%d>",
event.Opcode, event.VBucket)
default:
return fmt.Sprintf("<TapEvent %s, key=%q (%d bytes) flags=%x, exp=%d>",
event.Opcode, event.Key, len(event.Value),
event.Flags, event.Expiry)
}
}
// TapArguments are parameters for requesting a TAP feed.
//
// Call DefaultTapArguments to get a default one.
type TapArguments struct {
// Timestamp of oldest item to send.
//
// Use TapNoBackfill to suppress all past items.
Backfill uint64
// If set, server will disconnect after sending existing items.
Dump bool
// The indices of the vbuckets to watch; empty/nil to watch all.
VBuckets []uint16
// Transfers ownership of vbuckets during cluster rebalance.
Takeover bool
// If true, server will wait for client ACK after every notification.
SupportAck bool
// If true, client doesn't want values so server shouldn't send them.
KeysOnly bool
// If true, client wants the server to send checkpoint events.
Checkpoint bool
// Optional identifier to use for this client, to allow reconnects
ClientName string
// Registers this client (by name) till explicitly deregistered.
RegisteredClient bool
}
// Value for TapArguments.Backfill denoting that no past events at all
// should be sent.
const TapNoBackfill = math.MaxUint64
// DefaultTapArguments returns a default set of parameter values to
// pass to StartTapFeed.
func DefaultTapArguments() TapArguments {
return TapArguments{
Backfill: TapNoBackfill,
}
}
func (args *TapArguments) flags() []byte {
var flags gomemcached.TapConnectFlag
if args.Backfill != 0 {
flags |= gomemcached.BACKFILL
}
if args.Dump {
flags |= gomemcached.DUMP
}
if len(args.VBuckets) > 0 {
flags |= gomemcached.LIST_VBUCKETS
}
if args.Takeover {
flags |= gomemcached.TAKEOVER_VBUCKETS
}
if args.SupportAck {
flags |= gomemcached.SUPPORT_ACK
}
if args.KeysOnly {
flags |= gomemcached.REQUEST_KEYS_ONLY
}
if args.Checkpoint {
flags |= gomemcached.CHECKPOINT
}
if args.RegisteredClient {
flags |= gomemcached.REGISTERED_CLIENT
}
encoded := make([]byte, 4)
binary.BigEndian.PutUint32(encoded, uint32(flags))
return encoded
}
func must(err error) {
if err != nil {
panic(err)
}
}
func (args *TapArguments) bytes() (rv []byte) {
buf := bytes.NewBuffer([]byte{})
if args.Backfill > 0 {
must(binary.Write(buf, binary.BigEndian, uint64(args.Backfill)))
}
if len(args.VBuckets) > 0 {
must(binary.Write(buf, binary.BigEndian, uint16(len(args.VBuckets))))
for i := 0; i < len(args.VBuckets); i++ {
must(binary.Write(buf, binary.BigEndian, uint16(args.VBuckets[i])))
}
}
return buf.Bytes()
}
// TapFeed represents a stream of events from a server.
type TapFeed struct {
C <-chan TapEvent
Error error
closer chan bool
}
// StartTapFeed starts a TAP feed on a client connection.
//
// The events can be read from the returned channel. The connection
// can no longer be used for other purposes; it's now reserved for
// receiving the TAP messages. To stop receiving events, close the
// client connection.
func (mc *Client) StartTapFeed(args TapArguments) (*TapFeed, error) {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.TAP_CONNECT,
Key: []byte(args.ClientName),
Extras: args.flags(),
Body: args.bytes()}
err := mc.Transmit(rq)
if err != nil {
return nil, err
}
ch := make(chan TapEvent)
feed := &TapFeed{
C: ch,
closer: make(chan bool),
}
go mc.runFeed(ch, feed)
return feed, nil
}
// TapRecvHook is called after every incoming tap packet is received.
var TapRecvHook func(*gomemcached.MCRequest, int, error)
// Internal goroutine that reads from the socket and writes events to
// the channel
func (mc *Client) runFeed(ch chan TapEvent, feed *TapFeed) {
defer close(ch)
var headerBuf [gomemcached.HDR_LEN]byte
loop:
for {
// Read the next request from the server.
//
// (Can't call mc.Receive() because it reads a
// _response_ not a request.)
var pkt gomemcached.MCRequest
n, err := pkt.Receive(mc.conn, headerBuf[:])
if TapRecvHook != nil {
TapRecvHook(&pkt, n, err)
}
if err != nil {
if err != io.EOF {
feed.Error = err
}
break loop
}
//logging.Infof("** TapFeed received %#v : %q", pkt, pkt.Body)
if pkt.Opcode == gomemcached.TAP_CONNECT {
// This is not an event from the server; it's
// an error response to my connect request.
feed.Error = fmt.Errorf("tap connection failed: %s", pkt.Body)
break loop
}
event := makeTapEvent(pkt)
if event != nil {
if event.Opcode == tapEndStream {
break loop
}
select {
case ch <- *event:
case <-feed.closer:
break loop
}
}
if len(pkt.Extras) >= 4 {
reqFlags := binary.BigEndian.Uint16(pkt.Extras[2:])
if reqFlags&gomemcached.TAP_ACK != 0 {
if _, err := mc.sendAck(&pkt); err != nil {
feed.Error = err
break loop
}
}
}
}
if err := mc.Close(); err != nil {
logging.Errorf("Error closing memcached client: %v", err)
}
}
func (mc *Client) sendAck(pkt *gomemcached.MCRequest) (int, error) {
res := gomemcached.MCResponse{
Opcode: pkt.Opcode,
Opaque: pkt.Opaque,
Status: gomemcached.SUCCESS,
}
return res.Transmit(mc.conn)
}
// Close terminates a TapFeed.
//
// Call this if you stop using a TapFeed before its channel ends.
func (feed *TapFeed) Close() {
close(feed.closer)
}

67
vendor/github.com/couchbase/gomemcached/client/transport.go generated vendored Normal file
View File

@ -0,0 +1,67 @@
package memcached
import (
"errors"
"io"
"github.com/couchbase/gomemcached"
)
var errNoConn = errors.New("no connection")
// UnwrapMemcachedError converts memcached errors to normal responses.
//
// If the error is a memcached response, declare the error to be nil
// so a client can handle the status without worrying about whether it
// indicates success or failure.
func UnwrapMemcachedError(rv *gomemcached.MCResponse,
err error) (*gomemcached.MCResponse, error) {
if rv == err {
return rv, nil
}
return rv, err
}
// ReceiveHook is called after every packet is received (or attempted to be)
var ReceiveHook func(*gomemcached.MCResponse, int, error)
func getResponse(s io.Reader, hdrBytes []byte) (rv *gomemcached.MCResponse, n int, err error) {
if s == nil {
return nil, 0, errNoConn
}
rv = &gomemcached.MCResponse{}
n, err = rv.Receive(s, hdrBytes)
if ReceiveHook != nil {
ReceiveHook(rv, n, err)
}
if err == nil && (rv.Status != gomemcached.SUCCESS && rv.Status != gomemcached.AUTH_CONTINUE) {
err = rv
}
return rv, n, err
}
// TransmitHook is called after each packet is transmitted.
var TransmitHook func(*gomemcached.MCRequest, int, error)
func transmitRequest(o io.Writer, req *gomemcached.MCRequest) (int, error) {
if o == nil {
return 0, errNoConn
}
n, err := req.Transmit(o)
if TransmitHook != nil {
TransmitHook(req, n, err)
}
return n, err
}
func transmitResponse(o io.Writer, res *gomemcached.MCResponse) (int, error) {
if o == nil {
return 0, errNoConn
}
n, err := res.Transmit(o)
return n, err
}

800
vendor/github.com/couchbase/gomemcached/client/upr_feed.go generated vendored Normal file
View File

@ -0,0 +1,800 @@
// go implementation of upr client.
// See https://github.com/couchbaselabs/cbupr/blob/master/transport-spec.md
// TODO
// 1. Use a pool allocator to avoid garbage
package memcached
import (
"encoding/binary"
"errors"
"fmt"
"github.com/couchbase/gomemcached"
"github.com/couchbase/goutils/logging"
"strconv"
"sync"
)
const uprMutationExtraLen = 30
const uprDeletetionExtraLen = 18
const uprSnapshotExtraLen = 20
const bufferAckThreshold = 0.2
const opaqueOpen = 0xBEAF0001
const opaqueFailover = 0xDEADBEEF
const uprDefaultNoopInterval = 120
// UprEvent memcached events for UPR streams.
type UprEvent struct {
Opcode gomemcached.CommandCode // Type of event
Status gomemcached.Status // Response status
VBucket uint16 // VBucket this event applies to
DataType uint8 // data type
Opaque uint16 // 16 MSB of opaque
VBuuid uint64 // This field is set by downstream
Flags uint32 // Item flags
Expiry uint32 // Item expiration time
Key, Value []byte // Item key/value
OldValue []byte // TODO: TBD: old document value
Cas uint64 // CAS value of the item
Seqno uint64 // sequence number of the mutation
RevSeqno uint64 // rev sequence number : deletions
LockTime uint32 // Lock time
MetadataSize uint16 // Metadata size
SnapstartSeq uint64 // start sequence number of this snapshot
SnapendSeq uint64 // End sequence number of the snapshot
SnapshotType uint32 // 0: disk 1: memory
FailoverLog *FailoverLog // Failover log containing vvuid and sequnce number
Error error // Error value in case of a failure
ExtMeta []byte
AckSize uint32 // The number of bytes that can be Acked to DCP
}
// UprStream is per stream data structure over an UPR Connection.
type UprStream struct {
Vbucket uint16 // Vbucket id
Vbuuid uint64 // vbucket uuid
StartSeq uint64 // start sequence number
EndSeq uint64 // end sequence number
connected bool
}
// UprFeed represents an UPR feed. A feed contains a connection to a single
// host and multiple vBuckets
type UprFeed struct {
// lock for feed.vbstreams
muVbstreams sync.RWMutex
// lock for feed.closed
muClosed sync.RWMutex
C <-chan *UprEvent // Exported channel for receiving UPR events
vbstreams map[uint16]*UprStream // vb->stream mapping
closer chan bool // closer
conn *Client // connection to UPR producer
Error error // error
bytesRead uint64 // total bytes read on this connection
toAckBytes uint32 // bytes client has read
maxAckBytes uint32 // Max buffer control ack bytes
stats UprStats // Stats for upr client
transmitCh chan *gomemcached.MCRequest // transmit command channel
transmitCl chan bool // closer channel for transmit go-routine
closed bool // flag indicating whether the feed has been closed
// flag indicating whether client of upr feed will send ack to upr feed
// if flag is true, upr feed will use ack from client to determine whether/when to send ack to DCP
// if flag is false, upr feed will track how many bytes it has sent to client
// and use that to determine whether/when to send ack to DCP
ackByClient bool
}
// Exported interface - to allow for mocking
type UprFeedIface interface {
Close()
Closed() bool
CloseStream(vbno, opaqueMSB uint16) error
GetError() error
GetUprStats() *UprStats
IncrementAckBytes(bytes uint32) error
GetUprEventCh() <-chan *UprEvent
StartFeed() error
StartFeedWithConfig(datachan_len int) error
UprOpen(name string, sequence uint32, bufSize uint32) error
UprOpenWithXATTR(name string, sequence uint32, bufSize uint32) error
UprRequestStream(vbno, opaqueMSB uint16, flags uint32, vuuid, startSequence, endSequence, snapStart, snapEnd uint64) error
}
type UprStats struct {
TotalBytes uint64
TotalMutation uint64
TotalBufferAckSent uint64
TotalSnapShot uint64
}
// FailoverLog containing vvuid and sequnce number
type FailoverLog [][2]uint64
// error codes
var ErrorInvalidLog = errors.New("couchbase.errorInvalidLog")
func (flogp *FailoverLog) Latest() (vbuuid, seqno uint64, err error) {
if flogp != nil {
flog := *flogp
latest := flog[len(flog)-1]
return latest[0], latest[1], nil
}
return vbuuid, seqno, ErrorInvalidLog
}
func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream) *UprEvent {
event := &UprEvent{
Opcode: rq.Opcode,
VBucket: stream.Vbucket,
VBuuid: stream.Vbuuid,
Key: rq.Key,
Value: rq.Body,
Cas: rq.Cas,
ExtMeta: rq.ExtMeta,
DataType: rq.DataType,
AckSize: uint32(rq.Size()),
}
// 16 LSBits are used by client library to encode vbucket number.
// 16 MSBits are left for application to multiplex on opaque value.
event.Opaque = appOpaque(rq.Opaque)
if len(rq.Extras) >= uprMutationExtraLen &&
event.Opcode == gomemcached.UPR_MUTATION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.Flags = binary.BigEndian.Uint32(rq.Extras[16:20])
event.Expiry = binary.BigEndian.Uint32(rq.Extras[20:24])
event.LockTime = binary.BigEndian.Uint32(rq.Extras[24:28])
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[28:30])
} else if len(rq.Extras) >= uprDeletetionExtraLen &&
event.Opcode == gomemcached.UPR_DELETION ||
event.Opcode == gomemcached.UPR_EXPIRATION {
event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[16:18])
} else if len(rq.Extras) >= uprSnapshotExtraLen &&
event.Opcode == gomemcached.UPR_SNAPSHOT {
event.SnapstartSeq = binary.BigEndian.Uint64(rq.Extras[:8])
event.SnapendSeq = binary.BigEndian.Uint64(rq.Extras[8:16])
event.SnapshotType = binary.BigEndian.Uint32(rq.Extras[16:20])
}
return event
}
func (event *UprEvent) String() string {
name := gomemcached.CommandNames[event.Opcode]
if name == "" {
name = fmt.Sprintf("#%d", event.Opcode)
}
return name
}
func (feed *UprFeed) sendCommands(mc *Client) {
transmitCh := feed.transmitCh
transmitCl := feed.transmitCl
loop:
for {
select {
case command := <-transmitCh:
if err := mc.Transmit(command); err != nil {
logging.Errorf("Failed to transmit command %s. Error %s", command.Opcode.String(), err.Error())
// get feed to close and runFeed routine to exit
feed.Close()
break loop
}
case <-transmitCl:
break loop
}
}
// After sendCommands exits, write to transmitCh will block forever
// when we write to transmitCh, e.g., at CloseStream(), we need to check feed closure to have an exit route
logging.Infof("sendCommands exiting")
}
// NewUprFeed creates a new UPR Feed.
// TODO: Describe side-effects on bucket instance and its connection pool.
func (mc *Client) NewUprFeed() (*UprFeed, error) {
return mc.NewUprFeedWithConfig(false /*ackByClient*/)
}
func (mc *Client) NewUprFeedWithConfig(ackByClient bool) (*UprFeed, error) {
feed := &UprFeed{
conn: mc,
closer: make(chan bool, 1),
vbstreams: make(map[uint16]*UprStream),
transmitCh: make(chan *gomemcached.MCRequest),
transmitCl: make(chan bool),
ackByClient: ackByClient,
}
go feed.sendCommands(mc)
return feed, nil
}
func (mc *Client) NewUprFeedIface() (UprFeedIface, error) {
return mc.NewUprFeed()
}
func (mc *Client) NewUprFeedWithConfigIface(ackByClient bool) (UprFeedIface, error) {
return mc.NewUprFeedWithConfig(ackByClient)
}
func doUprOpen(mc *Client, name string, sequence uint32, enableXATTR bool) error {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_OPEN,
Key: []byte(name),
Opaque: opaqueOpen,
}
rq.Extras = make([]byte, 8)
binary.BigEndian.PutUint32(rq.Extras[:4], sequence)
// opens a producer type connection
flags := gomemcached.DCP_PRODUCER
if enableXATTR {
// set DCP_OPEN_INCLUDE_XATTRS bit in flags
flags = flags | gomemcached.DCP_OPEN_INCLUDE_XATTRS
}
binary.BigEndian.PutUint32(rq.Extras[4:], flags)
if err := mc.Transmit(rq); err != nil {
return err
}
if res, err := mc.Receive(); err != nil {
return err
} else if res.Opcode != gomemcached.UPR_OPEN {
return fmt.Errorf("unexpected #opcode %v", res.Opcode)
} else if rq.Opaque != res.Opaque {
return fmt.Errorf("opaque mismatch, %v over %v", res.Opaque, res.Opaque)
} else if res.Status != gomemcached.SUCCESS {
return fmt.Errorf("error %v", res.Status)
}
return nil
}
// UprOpen to connect with a UPR producer.
// Name: name of te UPR connection
// sequence: sequence number for the connection
// bufsize: max size of the application
func (feed *UprFeed) UprOpen(name string, sequence uint32, bufSize uint32) error {
return feed.uprOpen(name, sequence, bufSize, false /*enableXATTR*/)
}
// UprOpen with XATTR enabled.
func (feed *UprFeed) UprOpenWithXATTR(name string, sequence uint32, bufSize uint32) error {
return feed.uprOpen(name, sequence, bufSize, true /*enableXATTR*/)
}
func (feed *UprFeed) uprOpen(name string, sequence uint32, bufSize uint32, enableXATTR bool) error {
mc := feed.conn
var err error
if err = doUprOpen(mc, name, sequence, enableXATTR); err != nil {
return err
}
// send a UPR control message to set the window size for the this connection
if bufSize > 0 {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_CONTROL,
Key: []byte("connection_buffer_size"),
Body: []byte(strconv.Itoa(int(bufSize))),
}
err = feed.writeToTransmitCh(rq)
if err != nil {
return err
}
feed.maxAckBytes = uint32(bufferAckThreshold * float32(bufSize))
}
// enable noop and set noop interval
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_CONTROL,
Key: []byte("enable_noop"),
Body: []byte("true"),
}
err = feed.writeToTransmitCh(rq)
if err != nil {
return err
}
rq = &gomemcached.MCRequest{
Opcode: gomemcached.UPR_CONTROL,
Key: []byte("set_noop_interval"),
Body: []byte(strconv.Itoa(int(uprDefaultNoopInterval))),
}
err = feed.writeToTransmitCh(rq)
if err != nil {
return err
}
return nil
}
// UprGetFailoverLog for given list of vbuckets.
func (mc *Client) UprGetFailoverLog(
vb []uint16) (map[uint16]*FailoverLog, error) {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_FAILOVERLOG,
Opaque: opaqueFailover,
}
if err := doUprOpen(mc, "FailoverLog", 0, false); err != nil {
return nil, fmt.Errorf("UPR_OPEN Failed %s", err.Error())
}
failoverLogs := make(map[uint16]*FailoverLog)
for _, vBucket := range vb {
rq.VBucket = vBucket
if err := mc.Transmit(rq); err != nil {
return nil, err
}
res, err := mc.Receive()
if err != nil {
return nil, fmt.Errorf("failed to receive %s", err.Error())
} else if res.Opcode != gomemcached.UPR_FAILOVERLOG || res.Status != gomemcached.SUCCESS {
return nil, fmt.Errorf("unexpected #opcode %v", res.Opcode)
}
flog, err := parseFailoverLog(res.Body)
if err != nil {
return nil, fmt.Errorf("unable to parse failover logs for vb %d", vb)
}
failoverLogs[vBucket] = flog
}
return failoverLogs, nil
}
// UprRequestStream for a single vbucket.
func (feed *UprFeed) UprRequestStream(vbno, opaqueMSB uint16, flags uint32,
vuuid, startSequence, endSequence, snapStart, snapEnd uint64) error {
rq := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_STREAMREQ,
VBucket: vbno,
Opaque: composeOpaque(vbno, opaqueMSB),
}
rq.Extras = make([]byte, 48) // #Extras
binary.BigEndian.PutUint32(rq.Extras[:4], flags)
binary.BigEndian.PutUint32(rq.Extras[4:8], uint32(0))
binary.BigEndian.PutUint64(rq.Extras[8:16], startSequence)
binary.BigEndian.PutUint64(rq.Extras[16:24], endSequence)
binary.BigEndian.PutUint64(rq.Extras[24:32], vuuid)
binary.BigEndian.PutUint64(rq.Extras[32:40], snapStart)
binary.BigEndian.PutUint64(rq.Extras[40:48], snapEnd)
stream := &UprStream{
Vbucket: vbno,
Vbuuid: vuuid,
StartSeq: startSequence,
EndSeq: endSequence,
}
feed.muVbstreams.Lock()
// Any client that has ever called this method, regardless of return code,
// should expect a potential UPR_CLOSESTREAM message due to this new map entry prior to Transmit.
feed.vbstreams[vbno] = stream
feed.muVbstreams.Unlock()
if err := feed.conn.Transmit(rq); err != nil {
logging.Errorf("Error in StreamRequest %s", err.Error())
// If an error occurs during transmit, then the UPRFeed will keep the stream
// in the vbstreams map. This is to prevent nil lookup from any previously
// sent stream requests.
return err
}
return nil
}
// CloseStream for specified vbucket.
func (feed *UprFeed) CloseStream(vbno, opaqueMSB uint16) error {
err := feed.validateCloseStream(vbno)
if err != nil {
logging.Infof("CloseStream for %v has been skipped because of error %v", vbno, err)
return err
}
closeStream := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_CLOSESTREAM,
VBucket: vbno,
Opaque: composeOpaque(vbno, opaqueMSB),
}
feed.writeToTransmitCh(closeStream)
return nil
}
func (feed *UprFeed) GetUprEventCh() <-chan *UprEvent {
return feed.C
}
func (feed *UprFeed) GetError() error {
return feed.Error
}
func (feed *UprFeed) validateCloseStream(vbno uint16) error {
feed.muVbstreams.RLock()
defer feed.muVbstreams.RUnlock()
if feed.vbstreams[vbno] == nil {
return fmt.Errorf("Stream for vb %d has not been requested", vbno)
}
return nil
}
func (feed *UprFeed) writeToTransmitCh(rq *gomemcached.MCRequest) error {
// write to transmitCh may block forever if sendCommands has exited
// check for feed closure to have an exit route in this case
select {
case <-feed.closer:
errMsg := fmt.Sprintf("Abort sending request to transmitCh because feed has been closed. request=%v", rq)
logging.Infof(errMsg)
return errors.New(errMsg)
case feed.transmitCh <- rq:
}
return nil
}
// StartFeed to start the upper feed.
func (feed *UprFeed) StartFeed() error {
return feed.StartFeedWithConfig(10)
}
func (feed *UprFeed) StartFeedWithConfig(datachan_len int) error {
ch := make(chan *UprEvent, datachan_len)
feed.C = ch
go feed.runFeed(ch)
return nil
}
func parseFailoverLog(body []byte) (*FailoverLog, error) {
if len(body)%16 != 0 {
err := fmt.Errorf("invalid body length %v, in failover-log", len(body))
return nil, err
}
log := make(FailoverLog, len(body)/16)
for i, j := 0, 0; i < len(body); i += 16 {
vuuid := binary.BigEndian.Uint64(body[i : i+8])
seqno := binary.BigEndian.Uint64(body[i+8 : i+16])
log[j] = [2]uint64{vuuid, seqno}
j++
}
return &log, nil
}
func handleStreamRequest(
res *gomemcached.MCResponse,
headerBuf []byte,
) (gomemcached.Status, uint64, *FailoverLog, error) {
var rollback uint64
var err error
switch {
case res.Status == gomemcached.ROLLBACK:
logging.Infof("Rollback response. body=%v, headerBuf=%v\n", res.Body, headerBuf)
rollback = binary.BigEndian.Uint64(res.Body)
logging.Infof("Rollback %v for vb %v\n", rollback, res.Opaque)
return res.Status, rollback, nil, nil
case res.Status != gomemcached.SUCCESS:
err = fmt.Errorf("unexpected status %v, for %v", res.Status, res.Opaque)
return res.Status, 0, nil, err
}
flog, err := parseFailoverLog(res.Body[:])
return res.Status, rollback, flog, err
}
// generate stream end responses for all active vb streams
func (feed *UprFeed) doStreamClose(ch chan *UprEvent) {
feed.muVbstreams.RLock()
uprEvents := make([]*UprEvent, len(feed.vbstreams))
index := 0
for vbno, stream := range feed.vbstreams {
uprEvent := &UprEvent{
VBucket: vbno,
VBuuid: stream.Vbuuid,
Opcode: gomemcached.UPR_STREAMEND,
}
uprEvents[index] = uprEvent
index++
}
// release the lock before sending uprEvents to ch, which may block
feed.muVbstreams.RUnlock()
loop:
for _, uprEvent := range uprEvents {
select {
case ch <- uprEvent:
case <-feed.closer:
logging.Infof("Feed has been closed. Aborting doStreamClose.")
break loop
}
}
}
func (feed *UprFeed) runFeed(ch chan *UprEvent) {
defer close(ch)
var headerBuf [gomemcached.HDR_LEN]byte
var pkt gomemcached.MCRequest
var event *UprEvent
mc := feed.conn.Hijack()
uprStats := &feed.stats
loop:
for {
select {
case <-feed.closer:
logging.Infof("Feed has been closed. Exiting.")
break loop
default:
sendAck := false
bytes, err := pkt.Receive(mc, headerBuf[:])
if err != nil {
logging.Errorf("Error in receive %s", err.Error())
feed.Error = err
// send all the stream close messages to the client
feed.doStreamClose(ch)
break loop
} else {
event = nil
res := &gomemcached.MCResponse{
Opcode: pkt.Opcode,
Cas: pkt.Cas,
Opaque: pkt.Opaque,
Status: gomemcached.Status(pkt.VBucket),
Extras: pkt.Extras,
Key: pkt.Key,
Body: pkt.Body,
}
vb := vbOpaque(pkt.Opaque)
uprStats.TotalBytes = uint64(bytes)
feed.muVbstreams.RLock()
stream := feed.vbstreams[vb]
feed.muVbstreams.RUnlock()
switch pkt.Opcode {
case gomemcached.UPR_STREAMREQ:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
status, rb, flog, err := handleStreamRequest(res, headerBuf[:])
if status == gomemcached.ROLLBACK {
event = makeUprEvent(pkt, stream)
event.Status = status
// rollback stream
logging.Infof("UPR_STREAMREQ with rollback %d for vb %d Failed: %v", rb, vb, err)
// delete the stream from the vbmap for the feed
feed.muVbstreams.Lock()
delete(feed.vbstreams, vb)
feed.muVbstreams.Unlock()
} else if status == gomemcached.SUCCESS {
event = makeUprEvent(pkt, stream)
event.Seqno = stream.StartSeq
event.FailoverLog = flog
event.Status = status
stream.connected = true
logging.Infof("UPR_STREAMREQ for vb %d successful", vb)
} else if err != nil {
logging.Errorf("UPR_STREAMREQ for vbucket %d erro %s", vb, err.Error())
event = &UprEvent{
Opcode: gomemcached.UPR_STREAMREQ,
Status: status,
VBucket: vb,
Error: err,
}
// delete the stream
feed.muVbstreams.Lock()
delete(feed.vbstreams, vb)
feed.muVbstreams.Unlock()
}
case gomemcached.UPR_MUTATION,
gomemcached.UPR_DELETION,
gomemcached.UPR_EXPIRATION:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
event = makeUprEvent(pkt, stream)
uprStats.TotalMutation++
sendAck = true
case gomemcached.UPR_STREAMEND:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
//stream has ended
event = makeUprEvent(pkt, stream)
logging.Infof("Stream Ended for vb %d", vb)
sendAck = true
feed.muVbstreams.Lock()
delete(feed.vbstreams, vb)
feed.muVbstreams.Unlock()
case gomemcached.UPR_SNAPSHOT:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
// snapshot marker
event = makeUprEvent(pkt, stream)
uprStats.TotalSnapShot++
sendAck = true
case gomemcached.UPR_FLUSH:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
// special processing for flush ?
event = makeUprEvent(pkt, stream)
case gomemcached.UPR_CLOSESTREAM:
if stream == nil {
logging.Infof("Stream not found for vb %d: %#v", vb, pkt)
break loop
}
event = makeUprEvent(pkt, stream)
event.Opcode = gomemcached.UPR_STREAMEND // opcode re-write !!
logging.Infof("Stream Closed for vb %d StreamEnd simulated", vb)
sendAck = true
feed.muVbstreams.Lock()
delete(feed.vbstreams, vb)
feed.muVbstreams.Unlock()
case gomemcached.UPR_ADDSTREAM:
logging.Infof("Opcode %v not implemented", pkt.Opcode)
case gomemcached.UPR_CONTROL, gomemcached.UPR_BUFFERACK:
if res.Status != gomemcached.SUCCESS {
logging.Infof("Opcode %v received status %d", pkt.Opcode.String(), res.Status)
}
case gomemcached.UPR_NOOP:
// send a NOOP back
noop := &gomemcached.MCResponse{
Opcode: gomemcached.UPR_NOOP,
Opaque: pkt.Opaque,
}
if err := feed.conn.TransmitResponse(noop); err != nil {
logging.Warnf("failed to transmit command %s. Error %s", noop.Opcode.String(), err.Error())
}
default:
logging.Infof("Recived an unknown response for vbucket %d", vb)
}
}
if event != nil {
select {
case ch <- event:
case <-feed.closer:
logging.Infof("Feed has been closed. Skip sending events. Exiting.")
break loop
}
feed.muVbstreams.RLock()
l := len(feed.vbstreams)
feed.muVbstreams.RUnlock()
if event.Opcode == gomemcached.UPR_CLOSESTREAM && l == 0 {
logging.Infof("No more streams")
}
}
if !feed.ackByClient {
// if client does not ack, use the size of data sent to client to determine if ack to dcp is needed
feed.sendBufferAckIfNeeded(sendAck, uint32(bytes))
}
}
}
// make sure that feed is closed before we signal transmitCl and exit runFeed
feed.Close()
close(feed.transmitCl)
logging.Infof("runFeed exiting")
}
// Client, after setting ackByClient flag to true in NewUprFeedWithConfig() call,
// can call this API to notify gomemcached that the client has completed processing
// of a number of bytes
// This API is not thread safe. Caller should NOT have more than one go rountine calling this API
func (feed *UprFeed) IncrementAckBytes(bytes uint32) error {
if !feed.ackByClient {
return errors.New("Upr feed does not have ackByclient flag set")
}
feed.sendBufferAckIfNeeded(true, bytes)
return nil
}
// send buffer ack if enough ack bytes have been accumulated
func (feed *UprFeed) sendBufferAckIfNeeded(sendAck bool, bytes uint32) {
if sendAck {
totalBytes := feed.toAckBytes + bytes
if totalBytes > feed.maxAckBytes {
feed.toAckBytes = 0
feed.sendBufferAck(totalBytes)
} else {
feed.toAckBytes = totalBytes
}
}
}
// send buffer ack to dcp
func (feed *UprFeed) sendBufferAck(sendSize uint32) {
bufferAck := &gomemcached.MCRequest{
Opcode: gomemcached.UPR_BUFFERACK,
}
bufferAck.Extras = make([]byte, 4)
binary.BigEndian.PutUint32(bufferAck.Extras[:4], uint32(sendSize))
feed.writeToTransmitCh(bufferAck)
feed.stats.TotalBufferAckSent++
}
func (feed *UprFeed) GetUprStats() *UprStats {
return &feed.stats
}
func composeOpaque(vbno, opaqueMSB uint16) uint32 {
return (uint32(opaqueMSB) << 16) | uint32(vbno)
}
func appOpaque(opq32 uint32) uint16 {
return uint16((opq32 & 0xFFFF0000) >> 16)
}
func vbOpaque(opq32 uint32) uint16 {
return uint16(opq32 & 0xFFFF)
}
// Close this UprFeed.
func (feed *UprFeed) Close() {
feed.muClosed.Lock()
defer feed.muClosed.Unlock()
if !feed.closed {
close(feed.closer)
feed.closed = true
}
}
// check if the UprFeed has been closed
func (feed *UprFeed) Closed() bool {
feed.muClosed.RLock()
defer feed.muClosed.RUnlock()
return feed.closed
}

286
vendor/github.com/couchbase/gomemcached/mc_constants.go generated vendored Normal file
View File

@ -0,0 +1,286 @@
// Package gomemcached is binary protocol packet formats and constants.
package gomemcached
import (
"fmt"
)
const (
REQ_MAGIC = 0x80
RES_MAGIC = 0x81
)
// CommandCode for memcached packets.
type CommandCode uint8
const (
GET = CommandCode(0x00)
SET = CommandCode(0x01)
ADD = CommandCode(0x02)
REPLACE = CommandCode(0x03)
DELETE = CommandCode(0x04)
INCREMENT = CommandCode(0x05)
DECREMENT = CommandCode(0x06)
QUIT = CommandCode(0x07)
FLUSH = CommandCode(0x08)
GETQ = CommandCode(0x09)
NOOP = CommandCode(0x0a)
VERSION = CommandCode(0x0b)
GETK = CommandCode(0x0c)
GETKQ = CommandCode(0x0d)
APPEND = CommandCode(0x0e)
PREPEND = CommandCode(0x0f)
STAT = CommandCode(0x10)
SETQ = CommandCode(0x11)
ADDQ = CommandCode(0x12)
REPLACEQ = CommandCode(0x13)
DELETEQ = CommandCode(0x14)
INCREMENTQ = CommandCode(0x15)
DECREMENTQ = CommandCode(0x16)
QUITQ = CommandCode(0x17)
FLUSHQ = CommandCode(0x18)
APPENDQ = CommandCode(0x19)
PREPENDQ = CommandCode(0x1a)
GAT = CommandCode(0x1d)
HELLO = CommandCode(0x1f)
RGET = CommandCode(0x30)
RSET = CommandCode(0x31)
RSETQ = CommandCode(0x32)
RAPPEND = CommandCode(0x33)
RAPPENDQ = CommandCode(0x34)
RPREPEND = CommandCode(0x35)
RPREPENDQ = CommandCode(0x36)
RDELETE = CommandCode(0x37)
RDELETEQ = CommandCode(0x38)
RINCR = CommandCode(0x39)
RINCRQ = CommandCode(0x3a)
RDECR = CommandCode(0x3b)
RDECRQ = CommandCode(0x3c)
SASL_LIST_MECHS = CommandCode(0x20)
SASL_AUTH = CommandCode(0x21)
SASL_STEP = CommandCode(0x22)
TAP_CONNECT = CommandCode(0x40) // Client-sent request to initiate Tap feed
TAP_MUTATION = CommandCode(0x41) // Notification of a SET/ADD/REPLACE/etc. on the server
TAP_DELETE = CommandCode(0x42) // Notification of a DELETE on the server
TAP_FLUSH = CommandCode(0x43) // Replicates a flush_all command
TAP_OPAQUE = CommandCode(0x44) // Opaque control data from the engine
TAP_VBUCKET_SET = CommandCode(0x45) // Sets state of vbucket in receiver (used in takeover)
TAP_CHECKPOINT_START = CommandCode(0x46) // Notifies start of new checkpoint
TAP_CHECKPOINT_END = CommandCode(0x47) // Notifies end of checkpoint
UPR_OPEN = CommandCode(0x50) // Open a UPR connection with a name
UPR_ADDSTREAM = CommandCode(0x51) // Sent by ebucketMigrator to UPR Consumer
UPR_CLOSESTREAM = CommandCode(0x52) // Sent by eBucketMigrator to UPR Consumer
UPR_FAILOVERLOG = CommandCode(0x54) // Request failover logs
UPR_STREAMREQ = CommandCode(0x53) // Stream request from consumer to producer
UPR_STREAMEND = CommandCode(0x55) // Sent by producer when it has no more messages to stream
UPR_SNAPSHOT = CommandCode(0x56) // Start of a new snapshot
UPR_MUTATION = CommandCode(0x57) // Key mutation
UPR_DELETION = CommandCode(0x58) // Key deletion
UPR_EXPIRATION = CommandCode(0x59) // Key expiration
UPR_FLUSH = CommandCode(0x5a) // Delete all the data for a vbucket
UPR_NOOP = CommandCode(0x5c) // UPR NOOP
UPR_BUFFERACK = CommandCode(0x5d) // UPR Buffer Acknowledgement
UPR_CONTROL = CommandCode(0x5e) // Set flow control params
SELECT_BUCKET = CommandCode(0x89) // Select bucket
OBSERVE_SEQNO = CommandCode(0x91) // Sequence Number based Observe
OBSERVE = CommandCode(0x92)
GET_META = CommandCode(0xA0) // Get meta. returns with expiry, flags, cas etc
)
// Status field for memcached response.
type Status uint16
const (
SUCCESS = Status(0x00)
KEY_ENOENT = Status(0x01)
KEY_EEXISTS = Status(0x02)
E2BIG = Status(0x03)
EINVAL = Status(0x04)
NOT_STORED = Status(0x05)
DELTA_BADVAL = Status(0x06)
NOT_MY_VBUCKET = Status(0x07)
NO_BUCKET = Status(0x08)
AUTH_STALE = Status(0x1f)
AUTH_ERROR = Status(0x20)
AUTH_CONTINUE = Status(0x21)
ERANGE = Status(0x22)
ROLLBACK = Status(0x23)
EACCESS = Status(0x24)
NOT_INITIALIZED = Status(0x25)
UNKNOWN_COMMAND = Status(0x81)
ENOMEM = Status(0x82)
NOT_SUPPORTED = Status(0x83)
EINTERNAL = Status(0x84)
EBUSY = Status(0x85)
TMPFAIL = Status(0x86)
)
// the producer/consumer bit in dcp flags
var DCP_PRODUCER uint32 = 0x01
// the include XATTRS bit in dcp flags
var DCP_OPEN_INCLUDE_XATTRS uint32 = 0x04
// MCItem is an internal representation of an item.
type MCItem struct {
Cas uint64
Flags, Expiration uint32
Data []byte
}
// Number of bytes in a binary protocol header.
const HDR_LEN = 24
// Mapping of CommandCode -> name of command (not exhaustive)
var CommandNames map[CommandCode]string
// StatusNames human readable names for memcached response.
var StatusNames map[Status]string
func init() {
CommandNames = make(map[CommandCode]string)
CommandNames[GET] = "GET"
CommandNames[SET] = "SET"
CommandNames[ADD] = "ADD"
CommandNames[REPLACE] = "REPLACE"
CommandNames[DELETE] = "DELETE"
CommandNames[INCREMENT] = "INCREMENT"
CommandNames[DECREMENT] = "DECREMENT"
CommandNames[QUIT] = "QUIT"
CommandNames[FLUSH] = "FLUSH"
CommandNames[GETQ] = "GETQ"
CommandNames[NOOP] = "NOOP"
CommandNames[VERSION] = "VERSION"
CommandNames[GETK] = "GETK"
CommandNames[GETKQ] = "GETKQ"
CommandNames[APPEND] = "APPEND"
CommandNames[PREPEND] = "PREPEND"
CommandNames[STAT] = "STAT"
CommandNames[SETQ] = "SETQ"
CommandNames[ADDQ] = "ADDQ"
CommandNames[REPLACEQ] = "REPLACEQ"
CommandNames[DELETEQ] = "DELETEQ"
CommandNames[INCREMENTQ] = "INCREMENTQ"
CommandNames[DECREMENTQ] = "DECREMENTQ"
CommandNames[QUITQ] = "QUITQ"
CommandNames[FLUSHQ] = "FLUSHQ"
CommandNames[APPENDQ] = "APPENDQ"
CommandNames[PREPENDQ] = "PREPENDQ"
CommandNames[RGET] = "RGET"
CommandNames[RSET] = "RSET"
CommandNames[RSETQ] = "RSETQ"
CommandNames[RAPPEND] = "RAPPEND"
CommandNames[RAPPENDQ] = "RAPPENDQ"
CommandNames[RPREPEND] = "RPREPEND"
CommandNames[RPREPENDQ] = "RPREPENDQ"
CommandNames[RDELETE] = "RDELETE"
CommandNames[RDELETEQ] = "RDELETEQ"
CommandNames[RINCR] = "RINCR"
CommandNames[RINCRQ] = "RINCRQ"
CommandNames[RDECR] = "RDECR"
CommandNames[RDECRQ] = "RDECRQ"
CommandNames[SASL_LIST_MECHS] = "SASL_LIST_MECHS"
CommandNames[SASL_AUTH] = "SASL_AUTH"
CommandNames[SASL_STEP] = "SASL_STEP"
CommandNames[TAP_CONNECT] = "TAP_CONNECT"
CommandNames[TAP_MUTATION] = "TAP_MUTATION"
CommandNames[TAP_DELETE] = "TAP_DELETE"
CommandNames[TAP_FLUSH] = "TAP_FLUSH"
CommandNames[TAP_OPAQUE] = "TAP_OPAQUE"
CommandNames[TAP_VBUCKET_SET] = "TAP_VBUCKET_SET"
CommandNames[TAP_CHECKPOINT_START] = "TAP_CHECKPOINT_START"
CommandNames[TAP_CHECKPOINT_END] = "TAP_CHECKPOINT_END"
CommandNames[UPR_OPEN] = "UPR_OPEN"
CommandNames[UPR_ADDSTREAM] = "UPR_ADDSTREAM"
CommandNames[UPR_CLOSESTREAM] = "UPR_CLOSESTREAM"
CommandNames[UPR_FAILOVERLOG] = "UPR_FAILOVERLOG"
CommandNames[UPR_STREAMREQ] = "UPR_STREAMREQ"
CommandNames[UPR_STREAMEND] = "UPR_STREAMEND"
CommandNames[UPR_SNAPSHOT] = "UPR_SNAPSHOT"
CommandNames[UPR_MUTATION] = "UPR_MUTATION"
CommandNames[UPR_DELETION] = "UPR_DELETION"
CommandNames[UPR_EXPIRATION] = "UPR_EXPIRATION"
CommandNames[UPR_FLUSH] = "UPR_FLUSH"
CommandNames[UPR_NOOP] = "UPR_NOOP"
CommandNames[UPR_BUFFERACK] = "UPR_BUFFERACK"
CommandNames[UPR_CONTROL] = "UPR_CONTROL"
StatusNames = make(map[Status]string)
StatusNames[SUCCESS] = "SUCCESS"
StatusNames[KEY_ENOENT] = "KEY_ENOENT"
StatusNames[KEY_EEXISTS] = "KEY_EEXISTS"
StatusNames[E2BIG] = "E2BIG"
StatusNames[EINVAL] = "EINVAL"
StatusNames[NOT_STORED] = "NOT_STORED"
StatusNames[DELTA_BADVAL] = "DELTA_BADVAL"
StatusNames[NOT_MY_VBUCKET] = "NOT_MY_VBUCKET"
StatusNames[NO_BUCKET] = "NO_BUCKET"
StatusNames[AUTH_STALE] = "AUTH_STALE"
StatusNames[AUTH_ERROR] = "AUTH_ERROR"
StatusNames[AUTH_CONTINUE] = "AUTH_CONTINUE"
StatusNames[ERANGE] = "ERANGE"
StatusNames[ROLLBACK] = "ROLLBACK"
StatusNames[EACCESS] = "EACCESS"
StatusNames[NOT_INITIALIZED] = "NOT_INITIALIZED"
StatusNames[UNKNOWN_COMMAND] = "UNKNOWN_COMMAND"
StatusNames[ENOMEM] = "ENOMEM"
StatusNames[NOT_SUPPORTED] = "NOT_SUPPORTED"
StatusNames[EINTERNAL] = "EINTERNAL"
StatusNames[EBUSY] = "EBUSY"
StatusNames[TMPFAIL] = "TMPFAIL"
}
// String an op code.
func (o CommandCode) String() (rv string) {
rv = CommandNames[o]
if rv == "" {
rv = fmt.Sprintf("0x%02x", int(o))
}
return rv
}
// String an op code.
func (s Status) String() (rv string) {
rv = StatusNames[s]
if rv == "" {
rv = fmt.Sprintf("0x%02x", int(s))
}
return rv
}
// IsQuiet will return true if a command is a "quiet" command.
func (o CommandCode) IsQuiet() bool {
switch o {
case GETQ,
GETKQ,
SETQ,
ADDQ,
REPLACEQ,
DELETEQ,
INCREMENTQ,
DECREMENTQ,
QUITQ,
FLUSHQ,
APPENDQ,
PREPENDQ,
RSETQ,
RAPPENDQ,
RPREPENDQ,
RDELETEQ,
RINCRQ,
RDECRQ:
return true
}
return false
}

197
vendor/github.com/couchbase/gomemcached/mc_req.go generated vendored Normal file
View File

@ -0,0 +1,197 @@
package gomemcached
import (
"encoding/binary"
"fmt"
"io"
)
// The maximum reasonable body length to expect.
// Anything larger than this will result in an error.
// The current limit, 20MB, is the size limit supported by ep-engine.
var MaxBodyLen = int(20 * 1024 * 1024)
// MCRequest is memcached Request
type MCRequest struct {
// The command being issued
Opcode CommandCode
// The CAS (if applicable, or 0)
Cas uint64
// An opaque value to be returned with this request
Opaque uint32
// The vbucket to which this command belongs
VBucket uint16
// Command extras, key, and body
Extras, Key, Body, ExtMeta []byte
// Datatype identifier
DataType uint8
}
// Size gives the number of bytes this request requires.
func (req *MCRequest) Size() int {
return HDR_LEN + len(req.Extras) + len(req.Key) + len(req.Body) + len(req.ExtMeta)
}
// A debugging string representation of this request
func (req MCRequest) String() string {
return fmt.Sprintf("{MCRequest opcode=%s, bodylen=%d, key='%s'}",
req.Opcode, len(req.Body), req.Key)
}
func (req *MCRequest) fillHeaderBytes(data []byte) int {
pos := 0
data[pos] = REQ_MAGIC
pos++
data[pos] = byte(req.Opcode)
pos++
binary.BigEndian.PutUint16(data[pos:pos+2],
uint16(len(req.Key)))
pos += 2
// 4
data[pos] = byte(len(req.Extras))
pos++
// Data type
if req.DataType != 0 {
data[pos] = byte(req.DataType)
}
pos++
binary.BigEndian.PutUint16(data[pos:pos+2], req.VBucket)
pos += 2
// 8
binary.BigEndian.PutUint32(data[pos:pos+4],
uint32(len(req.Body)+len(req.Key)+len(req.Extras)+len(req.ExtMeta)))
pos += 4
// 12
binary.BigEndian.PutUint32(data[pos:pos+4], req.Opaque)
pos += 4
// 16
if req.Cas != 0 {
binary.BigEndian.PutUint64(data[pos:pos+8], req.Cas)
}
pos += 8
if len(req.Extras) > 0 {
copy(data[pos:pos+len(req.Extras)], req.Extras)
pos += len(req.Extras)
}
if len(req.Key) > 0 {
copy(data[pos:pos+len(req.Key)], req.Key)
pos += len(req.Key)
}
return pos
}
// HeaderBytes will return the wire representation of the request header
// (with the extras and key).
func (req *MCRequest) HeaderBytes() []byte {
data := make([]byte, HDR_LEN+len(req.Extras)+len(req.Key))
req.fillHeaderBytes(data)
return data
}
// Bytes will return the wire representation of this request.
func (req *MCRequest) Bytes() []byte {
data := make([]byte, req.Size())
pos := req.fillHeaderBytes(data)
if len(req.Body) > 0 {
copy(data[pos:pos+len(req.Body)], req.Body)
}
if len(req.ExtMeta) > 0 {
copy(data[pos+len(req.Body):pos+len(req.Body)+len(req.ExtMeta)], req.ExtMeta)
}
return data
}
// Transmit will send this request message across a writer.
func (req *MCRequest) Transmit(w io.Writer) (n int, err error) {
if len(req.Body) < 128 {
n, err = w.Write(req.Bytes())
} else {
n, err = w.Write(req.HeaderBytes())
if err == nil {
m := 0
m, err = w.Write(req.Body)
n += m
}
}
return
}
// Receive will fill this MCRequest with the data from a reader.
func (req *MCRequest) Receive(r io.Reader, hdrBytes []byte) (int, error) {
if len(hdrBytes) < HDR_LEN {
hdrBytes = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0}
}
n, err := io.ReadFull(r, hdrBytes)
if err != nil {
return n, err
}
if hdrBytes[0] != RES_MAGIC && hdrBytes[0] != REQ_MAGIC {
return n, fmt.Errorf("bad magic: 0x%02x", hdrBytes[0])
}
klen := int(binary.BigEndian.Uint16(hdrBytes[2:]))
elen := int(hdrBytes[4])
// Data type at 5
req.DataType = uint8(hdrBytes[5])
req.Opcode = CommandCode(hdrBytes[1])
// Vbucket at 6:7
req.VBucket = binary.BigEndian.Uint16(hdrBytes[6:])
totalBodyLen := int(binary.BigEndian.Uint32(hdrBytes[8:]))
req.Opaque = binary.BigEndian.Uint32(hdrBytes[12:])
req.Cas = binary.BigEndian.Uint64(hdrBytes[16:])
if totalBodyLen > 0 {
buf := make([]byte, totalBodyLen)
m, err := io.ReadFull(r, buf)
n += m
if err == nil {
if req.Opcode >= TAP_MUTATION &&
req.Opcode <= TAP_CHECKPOINT_END &&
len(buf) > 1 {
// In these commands there is "engine private"
// data at the end of the extras. The first 2
// bytes of extra data give its length.
elen += int(binary.BigEndian.Uint16(buf))
}
req.Extras = buf[0:elen]
req.Key = buf[elen : klen+elen]
// get the length of extended metadata
extMetaLen := 0
if elen > 29 {
extMetaLen = int(binary.BigEndian.Uint16(req.Extras[28:30]))
}
bodyLen := totalBodyLen - klen - elen - extMetaLen
if bodyLen > MaxBodyLen {
return n, fmt.Errorf("%d is too big (max %d)",
bodyLen, MaxBodyLen)
}
req.Body = buf[klen+elen : klen+elen+bodyLen]
req.ExtMeta = buf[klen+elen+bodyLen:]
}
}
return n, err
}

267
vendor/github.com/couchbase/gomemcached/mc_res.go generated vendored Normal file
View File

@ -0,0 +1,267 @@
package gomemcached
import (
"encoding/binary"
"fmt"
"io"
"sync"
)
// MCResponse is memcached response
type MCResponse struct {
// The command opcode of the command that sent the request
Opcode CommandCode
// The status of the response
Status Status
// The opaque sent in the request
Opaque uint32
// The CAS identifier (if applicable)
Cas uint64
// Extras, key, and body for this response
Extras, Key, Body []byte
// If true, this represents a fatal condition and we should hang up
Fatal bool
// Datatype identifier
DataType uint8
}
// A debugging string representation of this response
func (res MCResponse) String() string {
return fmt.Sprintf("{MCResponse status=%v keylen=%d, extralen=%d, bodylen=%d}",
res.Status, len(res.Key), len(res.Extras), len(res.Body))
}
// Response as an error.
func (res *MCResponse) Error() string {
return fmt.Sprintf("MCResponse status=%v, opcode=%v, opaque=%v, msg: %s",
res.Status, res.Opcode, res.Opaque, string(res.Body))
}
func errStatus(e error) Status {
status := Status(0xffff)
if res, ok := e.(*MCResponse); ok {
status = res.Status
}
return status
}
// IsNotFound is true if this error represents a "not found" response.
func IsNotFound(e error) bool {
return errStatus(e) == KEY_ENOENT
}
// IsFatal is false if this error isn't believed to be fatal to a connection.
func IsFatal(e error) bool {
if e == nil {
return false
}
switch errStatus(e) {
case KEY_ENOENT, KEY_EEXISTS, NOT_STORED, TMPFAIL:
return false
}
return true
}
// Size is number of bytes this response consumes on the wire.
func (res *MCResponse) Size() int {
return HDR_LEN + len(res.Extras) + len(res.Key) + len(res.Body)
}
func (res *MCResponse) fillHeaderBytes(data []byte) int {
pos := 0
data[pos] = RES_MAGIC
pos++
data[pos] = byte(res.Opcode)
pos++
binary.BigEndian.PutUint16(data[pos:pos+2],
uint16(len(res.Key)))
pos += 2
// 4
data[pos] = byte(len(res.Extras))
pos++
// Data type
if res.DataType != 0 {
data[pos] = byte(res.DataType)
} else {
data[pos] = 0
}
pos++
binary.BigEndian.PutUint16(data[pos:pos+2], uint16(res.Status))
pos += 2
// 8
binary.BigEndian.PutUint32(data[pos:pos+4],
uint32(len(res.Body)+len(res.Key)+len(res.Extras)))
pos += 4
// 12
binary.BigEndian.PutUint32(data[pos:pos+4], res.Opaque)
pos += 4
// 16
binary.BigEndian.PutUint64(data[pos:pos+8], res.Cas)
pos += 8
if len(res.Extras) > 0 {
copy(data[pos:pos+len(res.Extras)], res.Extras)
pos += len(res.Extras)
}
if len(res.Key) > 0 {
copy(data[pos:pos+len(res.Key)], res.Key)
pos += len(res.Key)
}
return pos
}
// HeaderBytes will get just the header bytes for this response.
func (res *MCResponse) HeaderBytes() []byte {
data := make([]byte, HDR_LEN+len(res.Extras)+len(res.Key))
res.fillHeaderBytes(data)
return data
}
// Bytes will return the actual bytes transmitted for this response.
func (res *MCResponse) Bytes() []byte {
data := make([]byte, res.Size())
pos := res.fillHeaderBytes(data)
copy(data[pos:pos+len(res.Body)], res.Body)
return data
}
// Transmit will send this response message across a writer.
func (res *MCResponse) Transmit(w io.Writer) (n int, err error) {
if len(res.Body) < 128 {
n, err = w.Write(res.Bytes())
} else {
n, err = w.Write(res.HeaderBytes())
if err == nil {
m := 0
m, err = w.Write(res.Body)
m += n
}
}
return
}
// Receive will fill this MCResponse with the data from this reader.
func (res *MCResponse) Receive(r io.Reader, hdrBytes []byte) (n int, err error) {
if len(hdrBytes) < HDR_LEN {
hdrBytes = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0}
}
n, err = io.ReadFull(r, hdrBytes)
if err != nil {
return n, err
}
if hdrBytes[0] != RES_MAGIC && hdrBytes[0] != REQ_MAGIC {
return n, fmt.Errorf("bad magic: 0x%02x", hdrBytes[0])
}
klen := int(binary.BigEndian.Uint16(hdrBytes[2:4]))
elen := int(hdrBytes[4])
res.Opcode = CommandCode(hdrBytes[1])
res.DataType = uint8(hdrBytes[5])
res.Status = Status(binary.BigEndian.Uint16(hdrBytes[6:8]))
res.Opaque = binary.BigEndian.Uint32(hdrBytes[12:16])
res.Cas = binary.BigEndian.Uint64(hdrBytes[16:24])
bodyLen := int(binary.BigEndian.Uint32(hdrBytes[8:12])) - (klen + elen)
//defer function to debug the panic seen with MB-15557
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf(`Panic in Receive. Response %v \n
key len %v extra len %v bodylen %v`, res, klen, elen, bodyLen)
}
}()
buf := make([]byte, klen+elen+bodyLen)
m, err := io.ReadFull(r, buf)
if err == nil {
res.Extras = buf[0:elen]
res.Key = buf[elen : klen+elen]
res.Body = buf[klen+elen:]
}
return n + m, err
}
type MCResponsePool struct {
pool *sync.Pool
}
func NewMCResponsePool() *MCResponsePool {
rv := &MCResponsePool{
pool: &sync.Pool{
New: func() interface{} {
return &MCResponse{}
},
},
}
return rv
}
func (this *MCResponsePool) Get() *MCResponse {
return this.pool.Get().(*MCResponse)
}
func (this *MCResponsePool) Put(r *MCResponse) {
if r == nil {
return
}
r.Extras = nil
r.Key = nil
r.Body = nil
r.Fatal = false
this.pool.Put(r)
}
type StringMCResponsePool struct {
pool *sync.Pool
size int
}
func NewStringMCResponsePool(size int) *StringMCResponsePool {
rv := &StringMCResponsePool{
pool: &sync.Pool{
New: func() interface{} {
return make(map[string]*MCResponse, size)
},
},
size: size,
}
return rv
}
func (this *StringMCResponsePool) Get() map[string]*MCResponse {
return this.pool.Get().(map[string]*MCResponse)
}
func (this *StringMCResponsePool) Put(m map[string]*MCResponse) {
if m == nil || len(m) > 2*this.size {
return
}
for k := range m {
m[k] = nil
delete(m, k)
}
this.pool.Put(m)
}

168
vendor/github.com/couchbase/gomemcached/tap.go generated vendored Normal file
View File

@ -0,0 +1,168 @@
package gomemcached
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"io/ioutil"
"strings"
)
type TapConnectFlag uint32
// Tap connect option flags
const (
BACKFILL = TapConnectFlag(0x01)
DUMP = TapConnectFlag(0x02)
LIST_VBUCKETS = TapConnectFlag(0x04)
TAKEOVER_VBUCKETS = TapConnectFlag(0x08)
SUPPORT_ACK = TapConnectFlag(0x10)
REQUEST_KEYS_ONLY = TapConnectFlag(0x20)
CHECKPOINT = TapConnectFlag(0x40)
REGISTERED_CLIENT = TapConnectFlag(0x80)
FIX_FLAG_BYTEORDER = TapConnectFlag(0x100)
)
// Tap opaque event subtypes
const (
TAP_OPAQUE_ENABLE_AUTO_NACK = 0
TAP_OPAQUE_INITIAL_VBUCKET_STREAM = 1
TAP_OPAQUE_ENABLE_CHECKPOINT_SYNC = 2
TAP_OPAQUE_CLOSE_TAP_STREAM = 7
TAP_OPAQUE_CLOSE_BACKFILL = 8
)
// Tap item flags
const (
TAP_ACK = 1
TAP_NO_VALUE = 2
TAP_FLAG_NETWORK_BYTE_ORDER = 4
)
// TapConnectFlagNames for TapConnectFlag
var TapConnectFlagNames = map[TapConnectFlag]string{
BACKFILL: "BACKFILL",
DUMP: "DUMP",
LIST_VBUCKETS: "LIST_VBUCKETS",
TAKEOVER_VBUCKETS: "TAKEOVER_VBUCKETS",
SUPPORT_ACK: "SUPPORT_ACK",
REQUEST_KEYS_ONLY: "REQUEST_KEYS_ONLY",
CHECKPOINT: "CHECKPOINT",
REGISTERED_CLIENT: "REGISTERED_CLIENT",
FIX_FLAG_BYTEORDER: "FIX_FLAG_BYTEORDER",
}
// TapItemParser is a function to parse a single tap extra.
type TapItemParser func(io.Reader) (interface{}, error)
// TapParseUint64 is a function to parse a single tap uint64.
func TapParseUint64(r io.Reader) (interface{}, error) {
var rv uint64
err := binary.Read(r, binary.BigEndian, &rv)
return rv, err
}
// TapParseUint16 is a function to parse a single tap uint16.
func TapParseUint16(r io.Reader) (interface{}, error) {
var rv uint16
err := binary.Read(r, binary.BigEndian, &rv)
return rv, err
}
// TapParseBool is a function to parse a single tap boolean.
func TapParseBool(r io.Reader) (interface{}, error) {
return true, nil
}
// TapParseVBList parses a list of vBucket numbers as []uint16.
func TapParseVBList(r io.Reader) (interface{}, error) {
num, err := TapParseUint16(r)
if err != nil {
return nil, err
}
n := int(num.(uint16))
rv := make([]uint16, n)
for i := 0; i < n; i++ {
x, err := TapParseUint16(r)
if err != nil {
return nil, err
}
rv[i] = x.(uint16)
}
return rv, err
}
// TapFlagParsers parser functions for TAP fields.
var TapFlagParsers = map[TapConnectFlag]TapItemParser{
BACKFILL: TapParseUint64,
LIST_VBUCKETS: TapParseVBList,
}
// SplitFlags will split the ORed flags into the individual bit flags.
func (f TapConnectFlag) SplitFlags() []TapConnectFlag {
rv := []TapConnectFlag{}
for i := uint32(1); f != 0; i = i << 1 {
if uint32(f)&i == i {
rv = append(rv, TapConnectFlag(i))
}
f = TapConnectFlag(uint32(f) & (^i))
}
return rv
}
func (f TapConnectFlag) String() string {
parts := []string{}
for _, x := range f.SplitFlags() {
p := TapConnectFlagNames[x]
if p == "" {
p = fmt.Sprintf("0x%x", int(x))
}
parts = append(parts, p)
}
return strings.Join(parts, "|")
}
type TapConnect struct {
Flags map[TapConnectFlag]interface{}
RemainingBody []byte
Name string
}
// ParseTapCommands parse the tap request into the interesting bits we may
// need to do something with.
func (req *MCRequest) ParseTapCommands() (TapConnect, error) {
rv := TapConnect{
Flags: map[TapConnectFlag]interface{}{},
Name: string(req.Key),
}
if len(req.Extras) < 4 {
return rv, fmt.Errorf("not enough extra bytes: %x", req.Extras)
}
flags := TapConnectFlag(binary.BigEndian.Uint32(req.Extras))
r := bytes.NewReader(req.Body)
for _, f := range flags.SplitFlags() {
fun := TapFlagParsers[f]
if fun == nil {
fun = TapParseBool
}
val, err := fun(r)
if err != nil {
return rv, err
}
rv.Flags[f] = val
}
var err error
rv.RemainingBody, err = ioutil.ReadAll(r)
return rv, err
}