add module

This commit is contained in:
zhangkun 2024-12-14 17:38:07 +08:00
parent 042009bfc0
commit d7f5dc9bfe
479 changed files with 1 additions and 236364 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
vendor/

View File

@ -1,10 +0,0 @@
language: go
go:
- 1.0.3
- 1.1.2
- 1.2
- tip
install:
- go get github.com/bmizerany/assert
notifications:
email: false

View File

@ -1,17 +0,0 @@
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@ -1,13 +0,0 @@
### go-simplejson
a Go package to interact with arbitrary JSON
[![Build Status](https://secure.travis-ci.org/bitly/go-simplejson.png)](http://travis-ci.org/bitly/go-simplejson)
### Importing
import github.com/bitly/go-simplejson
### Documentation
Visit the docs on [gopkgdoc](http://godoc.org/github.com/bitly/go-simplejson)

View File

@ -1,446 +0,0 @@
package simplejson
import (
"encoding/json"
"errors"
"log"
)
// returns the current implementation version
func Version() string {
return "0.5.0"
}
type Json struct {
data interface{}
}
// NewJson returns a pointer to a new `Json` object
// after unmarshaling `body` bytes
func NewJson(body []byte) (*Json, error) {
j := new(Json)
err := j.UnmarshalJSON(body)
if err != nil {
return nil, err
}
return j, nil
}
// New returns a pointer to a new, empty `Json` object
func New() *Json {
return &Json{
data: make(map[string]interface{}),
}
}
// Interface returns the underlying data
func (j *Json) Interface() interface{} {
return j.data
}
// Encode returns its marshaled data as `[]byte`
func (j *Json) Encode() ([]byte, error) {
return j.MarshalJSON()
}
// EncodePretty returns its marshaled data as `[]byte` with indentation
func (j *Json) EncodePretty() ([]byte, error) {
return json.MarshalIndent(&j.data, "", " ")
}
// Implements the json.Marshaler interface.
func (j *Json) MarshalJSON() ([]byte, error) {
return json.Marshal(&j.data)
}
// Set modifies `Json` map by `key` and `value`
// Useful for changing single key/value in a `Json` object easily.
func (j *Json) Set(key string, val interface{}) {
m, err := j.Map()
if err != nil {
return
}
m[key] = val
}
// SetPath modifies `Json`, recursively checking/creating map keys for the supplied path,
// and then finally writing in the value
func (j *Json) SetPath(branch []string, val interface{}) {
if len(branch) == 0 {
j.data = val
return
}
// in order to insert our branch, we need map[string]interface{}
if _, ok := (j.data).(map[string]interface{}); !ok {
// have to replace with something suitable
j.data = make(map[string]interface{})
}
curr := j.data.(map[string]interface{})
for i := 0; i < len(branch)-1; i++ {
b := branch[i]
// key exists?
if _, ok := curr[b]; !ok {
n := make(map[string]interface{})
curr[b] = n
curr = n
continue
}
// make sure the value is the right sort of thing
if _, ok := curr[b].(map[string]interface{}); !ok {
// have to replace with something suitable
n := make(map[string]interface{})
curr[b] = n
}
curr = curr[b].(map[string]interface{})
}
// add remaining k/v
curr[branch[len(branch)-1]] = val
}
// Del modifies `Json` map by deleting `key` if it is present.
func (j *Json) Del(key string) {
m, err := j.Map()
if err != nil {
return
}
delete(m, key)
}
// Get returns a pointer to a new `Json` object
// for `key` in its `map` representation
//
// useful for chaining operations (to traverse a nested JSON):
// js.Get("top_level").Get("dict").Get("value").Int()
func (j *Json) Get(key string) *Json {
m, err := j.Map()
if err == nil {
if val, ok := m[key]; ok {
return &Json{val}
}
}
return &Json{nil}
}
// GetPath searches for the item as specified by the branch
// without the need to deep dive using Get()'s.
//
// js.GetPath("top_level", "dict")
func (j *Json) GetPath(branch ...string) *Json {
jin := j
for _, p := range branch {
jin = jin.Get(p)
}
return jin
}
// GetIndex returns a pointer to a new `Json` object
// for `index` in its `array` representation
//
// this is the analog to Get when accessing elements of
// a json array instead of a json object:
// js.Get("top_level").Get("array").GetIndex(1).Get("key").Int()
func (j *Json) GetIndex(index int) *Json {
a, err := j.Array()
if err == nil {
if len(a) > index {
return &Json{a[index]}
}
}
return &Json{nil}
}
// CheckGet returns a pointer to a new `Json` object and
// a `bool` identifying success or failure
//
// useful for chained operations when success is important:
// if data, ok := js.Get("top_level").CheckGet("inner"); ok {
// log.Println(data)
// }
func (j *Json) CheckGet(key string) (*Json, bool) {
m, err := j.Map()
if err == nil {
if val, ok := m[key]; ok {
return &Json{val}, true
}
}
return nil, false
}
// Map type asserts to `map`
func (j *Json) Map() (map[string]interface{}, error) {
if m, ok := (j.data).(map[string]interface{}); ok {
return m, nil
}
return nil, errors.New("type assertion to map[string]interface{} failed")
}
// Array type asserts to an `array`
func (j *Json) Array() ([]interface{}, error) {
if a, ok := (j.data).([]interface{}); ok {
return a, nil
}
return nil, errors.New("type assertion to []interface{} failed")
}
// Bool type asserts to `bool`
func (j *Json) Bool() (bool, error) {
if s, ok := (j.data).(bool); ok {
return s, nil
}
return false, errors.New("type assertion to bool failed")
}
// String type asserts to `string`
func (j *Json) String() (string, error) {
if s, ok := (j.data).(string); ok {
return s, nil
}
return "", errors.New("type assertion to string failed")
}
// Bytes type asserts to `[]byte`
func (j *Json) Bytes() ([]byte, error) {
if s, ok := (j.data).(string); ok {
return []byte(s), nil
}
return nil, errors.New("type assertion to []byte failed")
}
// StringArray type asserts to an `array` of `string`
func (j *Json) StringArray() ([]string, error) {
arr, err := j.Array()
if err != nil {
return nil, err
}
retArr := make([]string, 0, len(arr))
for _, a := range arr {
if a == nil {
retArr = append(retArr, "")
continue
}
s, ok := a.(string)
if !ok {
return nil, err
}
retArr = append(retArr, s)
}
return retArr, nil
}
// MustArray guarantees the return of a `[]interface{}` (with optional default)
//
// useful when you want to interate over array values in a succinct manner:
// for i, v := range js.Get("results").MustArray() {
// fmt.Println(i, v)
// }
func (j *Json) MustArray(args ...[]interface{}) []interface{} {
var def []interface{}
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustArray() received too many arguments %d", len(args))
}
a, err := j.Array()
if err == nil {
return a
}
return def
}
// MustMap guarantees the return of a `map[string]interface{}` (with optional default)
//
// useful when you want to interate over map values in a succinct manner:
// for k, v := range js.Get("dictionary").MustMap() {
// fmt.Println(k, v)
// }
func (j *Json) MustMap(args ...map[string]interface{}) map[string]interface{} {
var def map[string]interface{}
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustMap() received too many arguments %d", len(args))
}
a, err := j.Map()
if err == nil {
return a
}
return def
}
// MustString guarantees the return of a `string` (with optional default)
//
// useful when you explicitly want a `string` in a single value return context:
// myFunc(js.Get("param1").MustString(), js.Get("optional_param").MustString("my_default"))
func (j *Json) MustString(args ...string) string {
var def string
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustString() received too many arguments %d", len(args))
}
s, err := j.String()
if err == nil {
return s
}
return def
}
// MustStringArray guarantees the return of a `[]string` (with optional default)
//
// useful when you want to interate over array values in a succinct manner:
// for i, s := range js.Get("results").MustStringArray() {
// fmt.Println(i, s)
// }
func (j *Json) MustStringArray(args ...[]string) []string {
var def []string
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustStringArray() received too many arguments %d", len(args))
}
a, err := j.StringArray()
if err == nil {
return a
}
return def
}
// MustInt guarantees the return of an `int` (with optional default)
//
// useful when you explicitly want an `int` in a single value return context:
// myFunc(js.Get("param1").MustInt(), js.Get("optional_param").MustInt(5150))
func (j *Json) MustInt(args ...int) int {
var def int
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustInt() received too many arguments %d", len(args))
}
i, err := j.Int()
if err == nil {
return i
}
return def
}
// MustFloat64 guarantees the return of a `float64` (with optional default)
//
// useful when you explicitly want a `float64` in a single value return context:
// myFunc(js.Get("param1").MustFloat64(), js.Get("optional_param").MustFloat64(5.150))
func (j *Json) MustFloat64(args ...float64) float64 {
var def float64
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustFloat64() received too many arguments %d", len(args))
}
f, err := j.Float64()
if err == nil {
return f
}
return def
}
// MustBool guarantees the return of a `bool` (with optional default)
//
// useful when you explicitly want a `bool` in a single value return context:
// myFunc(js.Get("param1").MustBool(), js.Get("optional_param").MustBool(true))
func (j *Json) MustBool(args ...bool) bool {
var def bool
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustBool() received too many arguments %d", len(args))
}
b, err := j.Bool()
if err == nil {
return b
}
return def
}
// MustInt64 guarantees the return of an `int64` (with optional default)
//
// useful when you explicitly want an `int64` in a single value return context:
// myFunc(js.Get("param1").MustInt64(), js.Get("optional_param").MustInt64(5150))
func (j *Json) MustInt64(args ...int64) int64 {
var def int64
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustInt64() received too many arguments %d", len(args))
}
i, err := j.Int64()
if err == nil {
return i
}
return def
}
// MustUInt64 guarantees the return of an `uint64` (with optional default)
//
// useful when you explicitly want an `uint64` in a single value return context:
// myFunc(js.Get("param1").MustUint64(), js.Get("optional_param").MustUint64(5150))
func (j *Json) MustUint64(args ...uint64) uint64 {
var def uint64
switch len(args) {
case 0:
case 1:
def = args[0]
default:
log.Panicf("MustUint64() received too many arguments %d", len(args))
}
i, err := j.Uint64()
if err == nil {
return i
}
return def
}

View File

@ -1,75 +0,0 @@
// +build !go1.1
package simplejson
import (
"encoding/json"
"errors"
"io"
"reflect"
)
// NewFromReader returns a *Json by decoding from an io.Reader
func NewFromReader(r io.Reader) (*Json, error) {
j := new(Json)
dec := json.NewDecoder(r)
err := dec.Decode(&j.data)
return j, err
}
// Implements the json.Unmarshaler interface.
func (j *Json) UnmarshalJSON(p []byte) error {
return json.Unmarshal(p, &j.data)
}
// Float64 coerces into a float64
func (j *Json) Float64() (float64, error) {
switch j.data.(type) {
case float32, float64:
return reflect.ValueOf(j.data).Float(), nil
case int, int8, int16, int32, int64:
return float64(reflect.ValueOf(j.data).Int()), nil
case uint, uint8, uint16, uint32, uint64:
return float64(reflect.ValueOf(j.data).Uint()), nil
}
return 0, errors.New("invalid value type")
}
// Int coerces into an int
func (j *Json) Int() (int, error) {
switch j.data.(type) {
case float32, float64:
return int(reflect.ValueOf(j.data).Float()), nil
case int, int8, int16, int32, int64:
return int(reflect.ValueOf(j.data).Int()), nil
case uint, uint8, uint16, uint32, uint64:
return int(reflect.ValueOf(j.data).Uint()), nil
}
return 0, errors.New("invalid value type")
}
// Int64 coerces into an int64
func (j *Json) Int64() (int64, error) {
switch j.data.(type) {
case float32, float64:
return int64(reflect.ValueOf(j.data).Float()), nil
case int, int8, int16, int32, int64:
return reflect.ValueOf(j.data).Int(), nil
case uint, uint8, uint16, uint32, uint64:
return int64(reflect.ValueOf(j.data).Uint()), nil
}
return 0, errors.New("invalid value type")
}
// Uint64 coerces into an uint64
func (j *Json) Uint64() (uint64, error) {
switch j.data.(type) {
case float32, float64:
return uint64(reflect.ValueOf(j.data).Float()), nil
case int, int8, int16, int32, int64:
return uint64(reflect.ValueOf(j.data).Int()), nil
case uint, uint8, uint16, uint32, uint64:
return reflect.ValueOf(j.data).Uint(), nil
}
return 0, errors.New("invalid value type")
}

View File

@ -1,89 +0,0 @@
// +build go1.1
package simplejson
import (
"bytes"
"encoding/json"
"errors"
"io"
"reflect"
"strconv"
)
// Implements the json.Unmarshaler interface.
func (j *Json) UnmarshalJSON(p []byte) error {
dec := json.NewDecoder(bytes.NewBuffer(p))
dec.UseNumber()
return dec.Decode(&j.data)
}
// NewFromReader returns a *Json by decoding from an io.Reader
func NewFromReader(r io.Reader) (*Json, error) {
j := new(Json)
dec := json.NewDecoder(r)
dec.UseNumber()
err := dec.Decode(&j.data)
return j, err
}
// Float64 coerces into a float64
func (j *Json) Float64() (float64, error) {
switch j.data.(type) {
case json.Number:
return j.data.(json.Number).Float64()
case float32, float64:
return reflect.ValueOf(j.data).Float(), nil
case int, int8, int16, int32, int64:
return float64(reflect.ValueOf(j.data).Int()), nil
case uint, uint8, uint16, uint32, uint64:
return float64(reflect.ValueOf(j.data).Uint()), nil
}
return 0, errors.New("invalid value type")
}
// Int coerces into an int
func (j *Json) Int() (int, error) {
switch j.data.(type) {
case json.Number:
i, err := j.data.(json.Number).Int64()
return int(i), err
case float32, float64:
return int(reflect.ValueOf(j.data).Float()), nil
case int, int8, int16, int32, int64:
return int(reflect.ValueOf(j.data).Int()), nil
case uint, uint8, uint16, uint32, uint64:
return int(reflect.ValueOf(j.data).Uint()), nil
}
return 0, errors.New("invalid value type")
}
// Int64 coerces into an int64
func (j *Json) Int64() (int64, error) {
switch j.data.(type) {
case json.Number:
return j.data.(json.Number).Int64()
case float32, float64:
return int64(reflect.ValueOf(j.data).Float()), nil
case int, int8, int16, int32, int64:
return reflect.ValueOf(j.data).Int(), nil
case uint, uint8, uint16, uint32, uint64:
return int64(reflect.ValueOf(j.data).Uint()), nil
}
return 0, errors.New("invalid value type")
}
// Uint64 coerces into an uint64
func (j *Json) Uint64() (uint64, error) {
switch j.data.(type) {
case json.Number:
return strconv.ParseUint(j.data.(json.Number).String(), 10, 64)
case float32, float64:
return uint64(reflect.ValueOf(j.data).Float()), nil
case int, int8, int16, int32, int64:
return uint64(reflect.ValueOf(j.data).Int()), nil
case uint, uint8, uint16, uint32, uint64:
return reflect.ValueOf(j.data).Uint(), nil
}
return 0, errors.New("invalid value type")
}

View File

@ -1,2 +0,0 @@
*.rdb
testdata/*/

View File

@ -1,20 +0,0 @@
sudo: false
language: go
services:
- redis-server
go:
- 1.9.x
- 1.10.x
- 1.11.x
- 1.12.x
- tip
matrix:
allow_failures:
- go: tip
install:
- go get github.com/onsi/ginkgo
- go get github.com/onsi/gomega

View File

@ -1,25 +0,0 @@
# Changelog
## Unreleased
- Cluster and Ring pipelines process commands for each node in its own goroutine.
## 6.14
- Added Options.MinIdleConns.
- Added Options.MaxConnAge.
- PoolStats.FreeConns is renamed to PoolStats.IdleConns.
- Add Client.Do to simplify creating custom commands.
- Add Cmd.String, Cmd.Int, Cmd.Int64, Cmd.Uint64, Cmd.Float64, and Cmd.Bool helpers.
- Lower memory usage.
## v6.13
- Ring got new options called `HashReplicas` and `Hash`. It is recommended to set `HashReplicas = 1000` for better keys distribution between shards.
- Cluster client was optimized to use much less memory when reloading cluster state.
- PubSub.ReceiveMessage is re-worked to not use ReceiveTimeout so it does not lose data when timeout occurres. In most cases it is recommended to use PubSub.Channel instead.
- Dialer.KeepAlive is set to 5 minutes by default.
## v6.12
- ClusterClient got new option called `ClusterSlots` which allows to build cluster of normal Redis Servers that don't have cluster mode enabled. See https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup

View File

@ -1,25 +0,0 @@
Copyright (c) 2013 The github.com/go-redis/redis Authors.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,22 +0,0 @@
all: testdeps
go test ./...
go test ./... -short -race
env GOOS=linux GOARCH=386 go test ./...
go vet
go get github.com/gordonklaus/ineffassign
ineffassign .
testdeps: testdata/redis/src/redis-server
bench: testdeps
go test ./... -test.run=NONE -test.bench=. -test.benchmem
.PHONY: all test testdeps bench
testdata/redis:
mkdir -p $@
wget -qO- https://github.com/antirez/redis/archive/5.0.tar.gz | tar xvz --strip-components=1 -C $@
testdata/redis/src/redis-server: testdata/redis
sed -i.bak 's/libjemalloc.a/libjemalloc.a -lrt/g' $</src/Makefile
cd $< && make all

View File

@ -1,146 +0,0 @@
# Redis client for Golang
[![Build Status](https://travis-ci.org/go-redis/redis.png?branch=master)](https://travis-ci.org/go-redis/redis)
[![GoDoc](https://godoc.org/github.com/go-redis/redis?status.svg)](https://godoc.org/github.com/go-redis/redis)
[![Airbrake](https://img.shields.io/badge/kudos-airbrake.io-orange.svg)](https://airbrake.io)
Supports:
- Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC.
- Automatic connection pooling with [circuit breaker](https://en.wikipedia.org/wiki/Circuit_breaker_design_pattern) support.
- [Pub/Sub](https://godoc.org/github.com/go-redis/redis#PubSub).
- [Transactions](https://godoc.org/github.com/go-redis/redis#example-Client-TxPipeline).
- [Pipeline](https://godoc.org/github.com/go-redis/redis#example-Client-Pipeline) and [TxPipeline](https://godoc.org/github.com/go-redis/redis#example-Client-TxPipeline).
- [Scripting](https://godoc.org/github.com/go-redis/redis#Script).
- [Timeouts](https://godoc.org/github.com/go-redis/redis#Options).
- [Redis Sentinel](https://godoc.org/github.com/go-redis/redis#NewFailoverClient).
- [Redis Cluster](https://godoc.org/github.com/go-redis/redis#NewClusterClient).
- [Cluster of Redis Servers](https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup) without using cluster mode and Redis Sentinel.
- [Ring](https://godoc.org/github.com/go-redis/redis#NewRing).
- [Instrumentation](https://godoc.org/github.com/go-redis/redis#ex-package--Instrumentation).
- [Cache friendly](https://github.com/go-redis/cache).
- [Rate limiting](https://github.com/go-redis/redis_rate).
- [Distributed Locks](https://github.com/bsm/redis-lock).
API docs: https://godoc.org/github.com/go-redis/redis.
Examples: https://godoc.org/github.com/go-redis/redis#pkg-examples.
## Installation
Install:
```shell
go get -u github.com/go-redis/redis
```
Import:
```go
import "github.com/go-redis/redis"
```
## Quickstart
```go
func ExampleNewClient() {
client := redis.NewClient(&redis.Options{
Addr: "localhost:6379",
Password: "", // no password set
DB: 0, // use default DB
})
pong, err := client.Ping().Result()
fmt.Println(pong, err)
// Output: PONG <nil>
}
func ExampleClient() {
err := client.Set("key", "value", 0).Err()
if err != nil {
panic(err)
}
val, err := client.Get("key").Result()
if err != nil {
panic(err)
}
fmt.Println("key", val)
val2, err := client.Get("key2").Result()
if err == redis.Nil {
fmt.Println("key2 does not exist")
} else if err != nil {
panic(err)
} else {
fmt.Println("key2", val2)
}
// Output: key value
// key2 does not exist
}
```
## Howto
Please go through [examples](https://godoc.org/github.com/go-redis/redis#pkg-examples) to get an idea how to use this package.
## Look and feel
Some corner cases:
```go
// SET key value EX 10 NX
set, err := client.SetNX("key", "value", 10*time.Second).Result()
// SORT list LIMIT 0 2 ASC
vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result()
// ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2
vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{
Min: "-inf",
Max: "+inf",
Offset: 0,
Count: 2,
}).Result()
// ZINTERSTORE out 2 zset1 zset2 WEIGHTS 2 3 AGGREGATE SUM
vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result()
// EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello"
vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result()
```
## Benchmark
go-redis vs redigo:
```
BenchmarkSetGoRedis10Conns64Bytes-4 200000 7621 ns/op 210 B/op 6 allocs/op
BenchmarkSetGoRedis100Conns64Bytes-4 200000 7554 ns/op 210 B/op 6 allocs/op
BenchmarkSetGoRedis10Conns1KB-4 200000 7697 ns/op 210 B/op 6 allocs/op
BenchmarkSetGoRedis100Conns1KB-4 200000 7688 ns/op 210 B/op 6 allocs/op
BenchmarkSetGoRedis10Conns10KB-4 200000 9214 ns/op 210 B/op 6 allocs/op
BenchmarkSetGoRedis100Conns10KB-4 200000 9181 ns/op 210 B/op 6 allocs/op
BenchmarkSetGoRedis10Conns1MB-4 2000 583242 ns/op 2337 B/op 6 allocs/op
BenchmarkSetGoRedis100Conns1MB-4 2000 583089 ns/op 2338 B/op 6 allocs/op
BenchmarkSetRedigo10Conns64Bytes-4 200000 7576 ns/op 208 B/op 7 allocs/op
BenchmarkSetRedigo100Conns64Bytes-4 200000 7782 ns/op 208 B/op 7 allocs/op
BenchmarkSetRedigo10Conns1KB-4 200000 7958 ns/op 208 B/op 7 allocs/op
BenchmarkSetRedigo100Conns1KB-4 200000 7725 ns/op 208 B/op 7 allocs/op
BenchmarkSetRedigo10Conns10KB-4 100000 18442 ns/op 208 B/op 7 allocs/op
BenchmarkSetRedigo100Conns10KB-4 100000 18818 ns/op 208 B/op 7 allocs/op
BenchmarkSetRedigo10Conns1MB-4 2000 668829 ns/op 226 B/op 7 allocs/op
BenchmarkSetRedigo100Conns1MB-4 2000 679542 ns/op 226 B/op 7 allocs/op
```
Redis Cluster:
```
BenchmarkRedisPing-4 200000 6983 ns/op 116 B/op 4 allocs/op
BenchmarkRedisClusterPing-4 100000 11535 ns/op 117 B/op 4 allocs/op
```
## See also
- [Golang PostgreSQL ORM](https://github.com/go-pg/pg)
- [Golang msgpack](https://github.com/vmihailenco/msgpack)
- [Golang message task queue](https://github.com/vmihailenco/taskq)

File diff suppressed because it is too large Load Diff

View File

@ -1,22 +0,0 @@
package redis
import "sync/atomic"
func (c *ClusterClient) DBSize() *IntCmd {
cmd := NewIntCmd("dbsize")
var size int64
err := c.ForEachMaster(func(master *Client) error {
n, err := master.DBSize().Result()
if err != nil {
return err
}
atomic.AddInt64(&size, n)
return nil
})
if err != nil {
cmd.setErr(err)
return cmd
}
cmd.val = size
return cmd
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,4 +0,0 @@
/*
Package redis implements a Redis client.
*/
package redis

View File

@ -1,81 +0,0 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package consistenthash provides an implementation of a ring hash.
package consistenthash
import (
"hash/crc32"
"sort"
"strconv"
)
type Hash func(data []byte) uint32
type Map struct {
hash Hash
replicas int
keys []int // Sorted
hashMap map[int]string
}
func New(replicas int, fn Hash) *Map {
m := &Map{
replicas: replicas,
hash: fn,
hashMap: make(map[int]string),
}
if m.hash == nil {
m.hash = crc32.ChecksumIEEE
}
return m
}
// Returns true if there are no items available.
func (m *Map) IsEmpty() bool {
return len(m.keys) == 0
}
// Adds some keys to the hash.
func (m *Map) Add(keys ...string) {
for _, key := range keys {
for i := 0; i < m.replicas; i++ {
hash := int(m.hash([]byte(strconv.Itoa(i) + key)))
m.keys = append(m.keys, hash)
m.hashMap[hash] = key
}
}
sort.Ints(m.keys)
}
// Gets the closest item in the hash to the provided key.
func (m *Map) Get(key string) string {
if m.IsEmpty() {
return ""
}
hash := int(m.hash([]byte(key)))
// Binary search for appropriate replica.
idx := sort.Search(len(m.keys), func(i int) bool { return m.keys[i] >= hash })
// Means we have cycled back to the first replica.
if idx == len(m.keys) {
idx = 0
}
return m.hashMap[m.keys[idx]]
}

View File

@ -1,89 +0,0 @@
package internal
import (
"io"
"net"
"strings"
"github.com/go-redis/redis/internal/proto"
)
func IsRetryableError(err error, retryTimeout bool) bool {
if err == nil {
return false
}
if err == io.EOF {
return true
}
if netErr, ok := err.(net.Error); ok {
if netErr.Timeout() {
return retryTimeout
}
return true
}
s := err.Error()
if s == "ERR max number of clients reached" {
return true
}
if strings.HasPrefix(s, "LOADING ") {
return true
}
if strings.HasPrefix(s, "READONLY ") {
return true
}
if strings.HasPrefix(s, "CLUSTERDOWN ") {
return true
}
return false
}
func IsRedisError(err error) bool {
_, ok := err.(proto.RedisError)
return ok
}
func IsBadConn(err error, allowTimeout bool) bool {
if err == nil {
return false
}
if IsRedisError(err) {
// #790
return IsReadOnlyError(err)
}
if allowTimeout {
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
return false
}
}
return true
}
func IsMovedError(err error) (moved bool, ask bool, addr string) {
if !IsRedisError(err) {
return
}
s := err.Error()
if strings.HasPrefix(s, "MOVED ") {
moved = true
} else if strings.HasPrefix(s, "ASK ") {
ask = true
} else {
return
}
ind := strings.LastIndex(s, " ")
if ind == -1 {
return false, false, ""
}
addr = s[ind+1:]
return
}
func IsLoadingError(err error) bool {
return strings.HasPrefix(err.Error(), "LOADING ")
}
func IsReadOnlyError(err error) bool {
return strings.HasPrefix(err.Error(), "READONLY ")
}

View File

@ -1,77 +0,0 @@
package hashtag
import (
"math/rand"
"strings"
)
const slotNumber = 16384
// CRC16 implementation according to CCITT standards.
// Copyright 2001-2010 Georges Menie (www.menie.org)
// Copyright 2013 The Go Authors. All rights reserved.
// http://redis.io/topics/cluster-spec#appendix-a-crc16-reference-implementation-in-ansi-c
var crc16tab = [256]uint16{
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
}
func Key(key string) string {
if s := strings.IndexByte(key, '{'); s > -1 {
if e := strings.IndexByte(key[s+1:], '}'); e > 0 {
return key[s+1 : s+e+1]
}
}
return key
}
func RandomSlot() int {
return rand.Intn(slotNumber)
}
// hashSlot returns a consistent slot number between 0 and 16383
// for any given string key.
func Slot(key string) int {
if key == "" {
return RandomSlot()
}
key = Key(key)
return int(crc16sum(key)) % slotNumber
}
func crc16sum(key string) (crc uint16) {
for i := 0; i < len(key); i++ {
crc = (crc << 8) ^ crc16tab[(byte(crc>>8)^key[i])&0x00ff]
}
return
}

View File

@ -1,24 +0,0 @@
package internal
import (
"math/rand"
"time"
)
// Retry backoff with jitter sleep to prevent overloaded conditions during intervals
// https://www.awsarchitectureblog.com/2015/03/backoff.html
func RetryBackoff(retry int, minBackoff, maxBackoff time.Duration) time.Duration {
if retry < 0 {
retry = 0
}
backoff := minBackoff << uint(retry)
if backoff > maxBackoff || backoff < minBackoff {
backoff = maxBackoff
}
if backoff == 0 {
return 0
}
return time.Duration(rand.Int63n(int64(backoff)))
}

View File

@ -1,15 +0,0 @@
package internal
import (
"fmt"
"log"
)
var Logger *log.Logger
func Logf(s string, args ...interface{}) {
if Logger == nil {
return
}
Logger.Output(2, fmt.Sprintf(s, args...))
}

View File

@ -1,60 +0,0 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"sync"
"sync/atomic"
)
// A Once will perform a successful action exactly once.
//
// Unlike a sync.Once, this Once's func returns an error
// and is re-armed on failure.
type Once struct {
m sync.Mutex
done uint32
}
// Do calls the function f if and only if Do has not been invoked
// without error for this instance of Once. In other words, given
// var once Once
// if once.Do(f) is called multiple times, only the first call will
// invoke f, even if f has a different value in each invocation unless
// f returns an error. A new instance of Once is required for each
// function to execute.
//
// Do is intended for initialization that must be run exactly once. Since f
// is niladic, it may be necessary to use a function literal to capture the
// arguments to a function to be invoked by Do:
// err := config.once.Do(func() error { return config.init(filename) })
func (o *Once) Do(f func() error) error {
if atomic.LoadUint32(&o.done) == 1 {
return nil
}
// Slow-path.
o.m.Lock()
defer o.m.Unlock()
var err error
if o.done == 0 {
err = f()
if err == nil {
atomic.StoreUint32(&o.done, 1)
}
}
return err
}

View File

@ -1,95 +0,0 @@
package pool
import (
"net"
"sync/atomic"
"time"
"github.com/go-redis/redis/internal/proto"
)
var noDeadline = time.Time{}
type Conn struct {
netConn net.Conn
rd *proto.Reader
rdLocked bool
wr *proto.Writer
Inited bool
pooled bool
createdAt time.Time
usedAt atomic.Value
}
func NewConn(netConn net.Conn) *Conn {
cn := &Conn{
netConn: netConn,
createdAt: time.Now(),
}
cn.rd = proto.NewReader(netConn)
cn.wr = proto.NewWriter(netConn)
cn.SetUsedAt(time.Now())
return cn
}
func (cn *Conn) UsedAt() time.Time {
return cn.usedAt.Load().(time.Time)
}
func (cn *Conn) SetUsedAt(tm time.Time) {
cn.usedAt.Store(tm)
}
func (cn *Conn) SetNetConn(netConn net.Conn) {
cn.netConn = netConn
cn.rd.Reset(netConn)
cn.wr.Reset(netConn)
}
func (cn *Conn) setReadTimeout(timeout time.Duration) error {
now := time.Now()
cn.SetUsedAt(now)
if timeout > 0 {
return cn.netConn.SetReadDeadline(now.Add(timeout))
}
return cn.netConn.SetReadDeadline(noDeadline)
}
func (cn *Conn) setWriteTimeout(timeout time.Duration) error {
now := time.Now()
cn.SetUsedAt(now)
if timeout > 0 {
return cn.netConn.SetWriteDeadline(now.Add(timeout))
}
return cn.netConn.SetWriteDeadline(noDeadline)
}
func (cn *Conn) Write(b []byte) (int, error) {
return cn.netConn.Write(b)
}
func (cn *Conn) RemoteAddr() net.Addr {
return cn.netConn.RemoteAddr()
}
func (cn *Conn) WithReader(timeout time.Duration, fn func(rd *proto.Reader) error) error {
_ = cn.setReadTimeout(timeout)
return fn(cn.rd)
}
func (cn *Conn) WithWriter(timeout time.Duration, fn func(wr *proto.Writer) error) error {
_ = cn.setWriteTimeout(timeout)
firstErr := fn(cn.wr)
err := cn.wr.Flush()
if err != nil && firstErr == nil {
firstErr = err
}
return firstErr
}
func (cn *Conn) Close() error {
return cn.netConn.Close()
}

View File

@ -1,476 +0,0 @@
package pool
import (
"errors"
"net"
"sync"
"sync/atomic"
"time"
"github.com/go-redis/redis/internal"
)
var ErrClosed = errors.New("redis: client is closed")
var ErrPoolTimeout = errors.New("redis: connection pool timeout")
var timers = sync.Pool{
New: func() interface{} {
t := time.NewTimer(time.Hour)
t.Stop()
return t
},
}
// Stats contains pool state information and accumulated stats.
type Stats struct {
Hits uint32 // number of times free connection was found in the pool
Misses uint32 // number of times free connection was NOT found in the pool
Timeouts uint32 // number of times a wait timeout occurred
TotalConns uint32 // number of total connections in the pool
IdleConns uint32 // number of idle connections in the pool
StaleConns uint32 // number of stale connections removed from the pool
}
type Pooler interface {
NewConn() (*Conn, error)
CloseConn(*Conn) error
Get() (*Conn, error)
Put(*Conn)
Remove(*Conn, error)
Len() int
IdleLen() int
Stats() *Stats
Close() error
}
type Options struct {
Dialer func() (net.Conn, error)
OnClose func(*Conn) error
PoolSize int
MinIdleConns int
MaxConnAge time.Duration
PoolTimeout time.Duration
IdleTimeout time.Duration
IdleCheckFrequency time.Duration
}
type ConnPool struct {
opt *Options
dialErrorsNum uint32 // atomic
lastDialErrorMu sync.RWMutex
lastDialError error
queue chan struct{}
connsMu sync.Mutex
conns []*Conn
idleConns []*Conn
poolSize int
idleConnsLen int
stats Stats
_closed uint32 // atomic
}
var _ Pooler = (*ConnPool)(nil)
func NewConnPool(opt *Options) *ConnPool {
p := &ConnPool{
opt: opt,
queue: make(chan struct{}, opt.PoolSize),
conns: make([]*Conn, 0, opt.PoolSize),
idleConns: make([]*Conn, 0, opt.PoolSize),
}
for i := 0; i < opt.MinIdleConns; i++ {
p.checkMinIdleConns()
}
if opt.IdleTimeout > 0 && opt.IdleCheckFrequency > 0 {
go p.reaper(opt.IdleCheckFrequency)
}
return p
}
func (p *ConnPool) checkMinIdleConns() {
if p.opt.MinIdleConns == 0 {
return
}
if p.poolSize < p.opt.PoolSize && p.idleConnsLen < p.opt.MinIdleConns {
p.poolSize++
p.idleConnsLen++
go p.addIdleConn()
}
}
func (p *ConnPool) addIdleConn() {
cn, err := p.newConn(true)
if err != nil {
return
}
p.connsMu.Lock()
p.conns = append(p.conns, cn)
p.idleConns = append(p.idleConns, cn)
p.connsMu.Unlock()
}
func (p *ConnPool) NewConn() (*Conn, error) {
return p._NewConn(false)
}
func (p *ConnPool) _NewConn(pooled bool) (*Conn, error) {
cn, err := p.newConn(pooled)
if err != nil {
return nil, err
}
p.connsMu.Lock()
p.conns = append(p.conns, cn)
if pooled {
if p.poolSize < p.opt.PoolSize {
p.poolSize++
} else {
cn.pooled = false
}
}
p.connsMu.Unlock()
return cn, nil
}
func (p *ConnPool) newConn(pooled bool) (*Conn, error) {
if p.closed() {
return nil, ErrClosed
}
if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.opt.PoolSize) {
return nil, p.getLastDialError()
}
netConn, err := p.opt.Dialer()
if err != nil {
p.setLastDialError(err)
if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.opt.PoolSize) {
go p.tryDial()
}
return nil, err
}
cn := NewConn(netConn)
cn.pooled = pooled
return cn, nil
}
func (p *ConnPool) tryDial() {
for {
if p.closed() {
return
}
conn, err := p.opt.Dialer()
if err != nil {
p.setLastDialError(err)
time.Sleep(time.Second)
continue
}
atomic.StoreUint32(&p.dialErrorsNum, 0)
_ = conn.Close()
return
}
}
func (p *ConnPool) setLastDialError(err error) {
p.lastDialErrorMu.Lock()
p.lastDialError = err
p.lastDialErrorMu.Unlock()
}
func (p *ConnPool) getLastDialError() error {
p.lastDialErrorMu.RLock()
err := p.lastDialError
p.lastDialErrorMu.RUnlock()
return err
}
// Get returns existed connection from the pool or creates a new one.
func (p *ConnPool) Get() (*Conn, error) {
if p.closed() {
return nil, ErrClosed
}
err := p.waitTurn()
if err != nil {
return nil, err
}
for {
p.connsMu.Lock()
cn := p.popIdle()
p.connsMu.Unlock()
if cn == nil {
break
}
if p.isStaleConn(cn) {
_ = p.CloseConn(cn)
continue
}
atomic.AddUint32(&p.stats.Hits, 1)
return cn, nil
}
atomic.AddUint32(&p.stats.Misses, 1)
newcn, err := p._NewConn(true)
if err != nil {
p.freeTurn()
return nil, err
}
return newcn, nil
}
func (p *ConnPool) getTurn() {
p.queue <- struct{}{}
}
func (p *ConnPool) waitTurn() error {
select {
case p.queue <- struct{}{}:
return nil
default:
timer := timers.Get().(*time.Timer)
timer.Reset(p.opt.PoolTimeout)
select {
case p.queue <- struct{}{}:
if !timer.Stop() {
<-timer.C
}
timers.Put(timer)
return nil
case <-timer.C:
timers.Put(timer)
atomic.AddUint32(&p.stats.Timeouts, 1)
return ErrPoolTimeout
}
}
}
func (p *ConnPool) freeTurn() {
<-p.queue
}
func (p *ConnPool) popIdle() *Conn {
if len(p.idleConns) == 0 {
return nil
}
idx := len(p.idleConns) - 1
cn := p.idleConns[idx]
p.idleConns = p.idleConns[:idx]
p.idleConnsLen--
p.checkMinIdleConns()
return cn
}
func (p *ConnPool) Put(cn *Conn) {
if !cn.pooled {
p.Remove(cn, nil)
return
}
p.connsMu.Lock()
p.idleConns = append(p.idleConns, cn)
p.idleConnsLen++
p.connsMu.Unlock()
p.freeTurn()
}
func (p *ConnPool) Remove(cn *Conn, reason error) {
p.removeConn(cn)
p.freeTurn()
_ = p.closeConn(cn)
}
func (p *ConnPool) CloseConn(cn *Conn) error {
p.removeConn(cn)
return p.closeConn(cn)
}
func (p *ConnPool) removeConn(cn *Conn) {
p.connsMu.Lock()
for i, c := range p.conns {
if c == cn {
p.conns = append(p.conns[:i], p.conns[i+1:]...)
if cn.pooled {
p.poolSize--
p.checkMinIdleConns()
}
break
}
}
p.connsMu.Unlock()
}
func (p *ConnPool) closeConn(cn *Conn) error {
if p.opt.OnClose != nil {
_ = p.opt.OnClose(cn)
}
return cn.Close()
}
// Len returns total number of connections.
func (p *ConnPool) Len() int {
p.connsMu.Lock()
n := len(p.conns)
p.connsMu.Unlock()
return n
}
// IdleLen returns number of idle connections.
func (p *ConnPool) IdleLen() int {
p.connsMu.Lock()
n := p.idleConnsLen
p.connsMu.Unlock()
return n
}
func (p *ConnPool) Stats() *Stats {
idleLen := p.IdleLen()
return &Stats{
Hits: atomic.LoadUint32(&p.stats.Hits),
Misses: atomic.LoadUint32(&p.stats.Misses),
Timeouts: atomic.LoadUint32(&p.stats.Timeouts),
TotalConns: uint32(p.Len()),
IdleConns: uint32(idleLen),
StaleConns: atomic.LoadUint32(&p.stats.StaleConns),
}
}
func (p *ConnPool) closed() bool {
return atomic.LoadUint32(&p._closed) == 1
}
func (p *ConnPool) Filter(fn func(*Conn) bool) error {
var firstErr error
p.connsMu.Lock()
for _, cn := range p.conns {
if fn(cn) {
if err := p.closeConn(cn); err != nil && firstErr == nil {
firstErr = err
}
}
}
p.connsMu.Unlock()
return firstErr
}
func (p *ConnPool) Close() error {
if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) {
return ErrClosed
}
var firstErr error
p.connsMu.Lock()
for _, cn := range p.conns {
if err := p.closeConn(cn); err != nil && firstErr == nil {
firstErr = err
}
}
p.conns = nil
p.poolSize = 0
p.idleConns = nil
p.idleConnsLen = 0
p.connsMu.Unlock()
return firstErr
}
func (p *ConnPool) reapStaleConn() *Conn {
if len(p.idleConns) == 0 {
return nil
}
cn := p.idleConns[0]
if !p.isStaleConn(cn) {
return nil
}
p.idleConns = append(p.idleConns[:0], p.idleConns[1:]...)
p.idleConnsLen--
return cn
}
func (p *ConnPool) ReapStaleConns() (int, error) {
var n int
for {
p.getTurn()
p.connsMu.Lock()
cn := p.reapStaleConn()
p.connsMu.Unlock()
if cn != nil {
p.removeConn(cn)
}
p.freeTurn()
if cn != nil {
p.closeConn(cn)
n++
} else {
break
}
}
return n, nil
}
func (p *ConnPool) reaper(frequency time.Duration) {
ticker := time.NewTicker(frequency)
defer ticker.Stop()
for range ticker.C {
if p.closed() {
break
}
n, err := p.ReapStaleConns()
if err != nil {
internal.Logf("ReapStaleConns failed: %s", err)
continue
}
atomic.AddUint32(&p.stats.StaleConns, uint32(n))
}
}
func (p *ConnPool) isStaleConn(cn *Conn) bool {
if p.opt.IdleTimeout == 0 && p.opt.MaxConnAge == 0 {
return false
}
now := time.Now()
if p.opt.IdleTimeout > 0 && now.Sub(cn.UsedAt()) >= p.opt.IdleTimeout {
return true
}
if p.opt.MaxConnAge > 0 && now.Sub(cn.createdAt) >= p.opt.MaxConnAge {
return true
}
return false
}

View File

@ -1,203 +0,0 @@
package pool
import (
"fmt"
"sync/atomic"
)
const (
stateDefault = 0
stateInited = 1
stateClosed = 2
)
type BadConnError struct {
wrapped error
}
var _ error = (*BadConnError)(nil)
func (e BadConnError) Error() string {
return "pg: Conn is in a bad state"
}
func (e BadConnError) Unwrap() error {
return e.wrapped
}
type SingleConnPool struct {
pool Pooler
level int32 // atomic
state uint32 // atomic
ch chan *Conn
_badConnError atomic.Value
}
var _ Pooler = (*SingleConnPool)(nil)
func NewSingleConnPool(pool Pooler) *SingleConnPool {
p, ok := pool.(*SingleConnPool)
if !ok {
p = &SingleConnPool{
pool: pool,
ch: make(chan *Conn, 1),
}
}
atomic.AddInt32(&p.level, 1)
return p
}
func (p *SingleConnPool) SetConn(cn *Conn) {
if atomic.CompareAndSwapUint32(&p.state, stateDefault, stateInited) {
p.ch <- cn
} else {
panic("not reached")
}
}
func (p *SingleConnPool) NewConn() (*Conn, error) {
return p.pool.NewConn()
}
func (p *SingleConnPool) CloseConn(cn *Conn) error {
return p.pool.CloseConn(cn)
}
func (p *SingleConnPool) Get() (*Conn, error) {
// In worst case this races with Close which is not a very common operation.
for i := 0; i < 1000; i++ {
switch atomic.LoadUint32(&p.state) {
case stateDefault:
cn, err := p.pool.Get()
if err != nil {
return nil, err
}
if atomic.CompareAndSwapUint32(&p.state, stateDefault, stateInited) {
return cn, nil
}
p.pool.Remove(cn, ErrClosed)
case stateInited:
if err := p.badConnError(); err != nil {
return nil, err
}
cn, ok := <-p.ch
if !ok {
return nil, ErrClosed
}
return cn, nil
case stateClosed:
return nil, ErrClosed
default:
panic("not reached")
}
}
return nil, fmt.Errorf("pg: SingleConnPool.Get: infinite loop")
}
func (p *SingleConnPool) Put(cn *Conn) {
defer func() {
if recover() != nil {
p.freeConn(cn)
}
}()
p.ch <- cn
}
func (p *SingleConnPool) freeConn(cn *Conn) {
if err := p.badConnError(); err != nil {
p.pool.Remove(cn, err)
} else {
p.pool.Put(cn)
}
}
func (p *SingleConnPool) Remove(cn *Conn, reason error) {
defer func() {
if recover() != nil {
p.pool.Remove(cn, ErrClosed)
}
}()
p._badConnError.Store(BadConnError{wrapped: reason})
p.ch <- cn
}
func (p *SingleConnPool) Len() int {
switch atomic.LoadUint32(&p.state) {
case stateDefault:
return 0
case stateInited:
return 1
case stateClosed:
return 0
default:
panic("not reached")
}
}
func (p *SingleConnPool) IdleLen() int {
return len(p.ch)
}
func (p *SingleConnPool) Stats() *Stats {
return &Stats{}
}
func (p *SingleConnPool) Close() error {
level := atomic.AddInt32(&p.level, -1)
if level > 0 {
return nil
}
for i := 0; i < 1000; i++ {
state := atomic.LoadUint32(&p.state)
if state == stateClosed {
return ErrClosed
}
if atomic.CompareAndSwapUint32(&p.state, state, stateClosed) {
close(p.ch)
cn, ok := <-p.ch
if ok {
p.freeConn(cn)
}
return nil
}
}
return fmt.Errorf("pg: SingleConnPool.Close: infinite loop")
}
func (p *SingleConnPool) Reset() error {
if p.badConnError() == nil {
return nil
}
select {
case cn, ok := <-p.ch:
if !ok {
return ErrClosed
}
p.pool.Remove(cn, ErrClosed)
p._badConnError.Store(BadConnError{wrapped: nil})
default:
return fmt.Errorf("pg: SingleConnPool does not have a Conn")
}
if !atomic.CompareAndSwapUint32(&p.state, stateInited, stateDefault) {
state := atomic.LoadUint32(&p.state)
return fmt.Errorf("pg: invalid SingleConnPool state: %d", state)
}
return nil
}
func (p *SingleConnPool) badConnError() error {
if v := p._badConnError.Load(); v != nil {
err := v.(BadConnError)
if err.wrapped != nil {
return err
}
}
return nil
}

View File

@ -1,109 +0,0 @@
package pool
import "sync"
type StickyConnPool struct {
pool *ConnPool
reusable bool
cn *Conn
closed bool
mu sync.Mutex
}
var _ Pooler = (*StickyConnPool)(nil)
func NewStickyConnPool(pool *ConnPool, reusable bool) *StickyConnPool {
return &StickyConnPool{
pool: pool,
reusable: reusable,
}
}
func (p *StickyConnPool) NewConn() (*Conn, error) {
panic("not implemented")
}
func (p *StickyConnPool) CloseConn(*Conn) error {
panic("not implemented")
}
func (p *StickyConnPool) Get() (*Conn, error) {
p.mu.Lock()
defer p.mu.Unlock()
if p.closed {
return nil, ErrClosed
}
if p.cn != nil {
return p.cn, nil
}
cn, err := p.pool.Get()
if err != nil {
return nil, err
}
p.cn = cn
return cn, nil
}
func (p *StickyConnPool) putUpstream() {
p.pool.Put(p.cn)
p.cn = nil
}
func (p *StickyConnPool) Put(cn *Conn) {}
func (p *StickyConnPool) removeUpstream(reason error) {
p.pool.Remove(p.cn, reason)
p.cn = nil
}
func (p *StickyConnPool) Remove(cn *Conn, reason error) {
p.removeUpstream(reason)
}
func (p *StickyConnPool) Len() int {
p.mu.Lock()
defer p.mu.Unlock()
if p.cn == nil {
return 0
}
return 1
}
func (p *StickyConnPool) IdleLen() int {
p.mu.Lock()
defer p.mu.Unlock()
if p.cn == nil {
return 1
}
return 0
}
func (p *StickyConnPool) Stats() *Stats {
return nil
}
func (p *StickyConnPool) Close() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.closed {
return ErrClosed
}
p.closed = true
if p.cn != nil {
if p.reusable {
p.putUpstream()
} else {
p.removeUpstream(ErrClosed)
}
}
return nil
}

View File

@ -1,290 +0,0 @@
package proto
import (
"bufio"
"fmt"
"io"
"strconv"
"github.com/go-redis/redis/internal/util"
)
const (
ErrorReply = '-'
StatusReply = '+'
IntReply = ':'
StringReply = '$'
ArrayReply = '*'
)
//------------------------------------------------------------------------------
const Nil = RedisError("redis: nil")
type RedisError string
func (e RedisError) Error() string { return string(e) }
//------------------------------------------------------------------------------
type MultiBulkParse func(*Reader, int64) (interface{}, error)
type Reader struct {
rd *bufio.Reader
_buf []byte
}
func NewReader(rd io.Reader) *Reader {
return &Reader{
rd: bufio.NewReader(rd),
_buf: make([]byte, 64),
}
}
func (r *Reader) Reset(rd io.Reader) {
r.rd.Reset(rd)
}
func (r *Reader) ReadLine() ([]byte, error) {
line, isPrefix, err := r.rd.ReadLine()
if err != nil {
return nil, err
}
if isPrefix {
return nil, bufio.ErrBufferFull
}
if len(line) == 0 {
return nil, fmt.Errorf("redis: reply is empty")
}
if isNilReply(line) {
return nil, Nil
}
return line, nil
}
func (r *Reader) ReadReply(m MultiBulkParse) (interface{}, error) {
line, err := r.ReadLine()
if err != nil {
return nil, err
}
switch line[0] {
case ErrorReply:
return nil, ParseErrorReply(line)
case StatusReply:
return string(line[1:]), nil
case IntReply:
return util.ParseInt(line[1:], 10, 64)
case StringReply:
return r.readStringReply(line)
case ArrayReply:
n, err := parseArrayLen(line)
if err != nil {
return nil, err
}
return m(r, n)
}
return nil, fmt.Errorf("redis: can't parse %.100q", line)
}
func (r *Reader) ReadIntReply() (int64, error) {
line, err := r.ReadLine()
if err != nil {
return 0, err
}
switch line[0] {
case ErrorReply:
return 0, ParseErrorReply(line)
case IntReply:
return util.ParseInt(line[1:], 10, 64)
default:
return 0, fmt.Errorf("redis: can't parse int reply: %.100q", line)
}
}
func (r *Reader) ReadString() (string, error) {
line, err := r.ReadLine()
if err != nil {
return "", err
}
switch line[0] {
case ErrorReply:
return "", ParseErrorReply(line)
case StringReply:
return r.readStringReply(line)
case StatusReply:
return string(line[1:]), nil
case IntReply:
return string(line[1:]), nil
default:
return "", fmt.Errorf("redis: can't parse reply=%.100q reading string", line)
}
}
func (r *Reader) readStringReply(line []byte) (string, error) {
if isNilReply(line) {
return "", Nil
}
replyLen, err := strconv.Atoi(string(line[1:]))
if err != nil {
return "", err
}
b := make([]byte, replyLen+2)
_, err = io.ReadFull(r.rd, b)
if err != nil {
return "", err
}
return util.BytesToString(b[:replyLen]), nil
}
func (r *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) {
line, err := r.ReadLine()
if err != nil {
return nil, err
}
switch line[0] {
case ErrorReply:
return nil, ParseErrorReply(line)
case ArrayReply:
n, err := parseArrayLen(line)
if err != nil {
return nil, err
}
return m(r, n)
default:
return nil, fmt.Errorf("redis: can't parse array reply: %.100q", line)
}
}
func (r *Reader) ReadArrayLen() (int64, error) {
line, err := r.ReadLine()
if err != nil {
return 0, err
}
switch line[0] {
case ErrorReply:
return 0, ParseErrorReply(line)
case ArrayReply:
return parseArrayLen(line)
default:
return 0, fmt.Errorf("redis: can't parse array reply: %.100q", line)
}
}
func (r *Reader) ReadScanReply() ([]string, uint64, error) {
n, err := r.ReadArrayLen()
if err != nil {
return nil, 0, err
}
if n != 2 {
return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2", n)
}
cursor, err := r.ReadUint()
if err != nil {
return nil, 0, err
}
n, err = r.ReadArrayLen()
if err != nil {
return nil, 0, err
}
keys := make([]string, n)
for i := int64(0); i < n; i++ {
key, err := r.ReadString()
if err != nil {
return nil, 0, err
}
keys[i] = key
}
return keys, cursor, err
}
func (r *Reader) ReadInt() (int64, error) {
b, err := r.readTmpBytesReply()
if err != nil {
return 0, err
}
return util.ParseInt(b, 10, 64)
}
func (r *Reader) ReadUint() (uint64, error) {
b, err := r.readTmpBytesReply()
if err != nil {
return 0, err
}
return util.ParseUint(b, 10, 64)
}
func (r *Reader) ReadFloatReply() (float64, error) {
b, err := r.readTmpBytesReply()
if err != nil {
return 0, err
}
return util.ParseFloat(b, 64)
}
func (r *Reader) readTmpBytesReply() ([]byte, error) {
line, err := r.ReadLine()
if err != nil {
return nil, err
}
switch line[0] {
case ErrorReply:
return nil, ParseErrorReply(line)
case StringReply:
return r._readTmpBytesReply(line)
case StatusReply:
return line[1:], nil
default:
return nil, fmt.Errorf("redis: can't parse string reply: %.100q", line)
}
}
func (r *Reader) _readTmpBytesReply(line []byte) ([]byte, error) {
if isNilReply(line) {
return nil, Nil
}
replyLen, err := strconv.Atoi(string(line[1:]))
if err != nil {
return nil, err
}
buf := r.buf(replyLen + 2)
_, err = io.ReadFull(r.rd, buf)
if err != nil {
return nil, err
}
return buf[:replyLen], nil
}
func (r *Reader) buf(n int) []byte {
if d := n - cap(r._buf); d > 0 {
r._buf = append(r._buf, make([]byte, d)...)
}
return r._buf[:n]
}
func isNilReply(b []byte) bool {
return len(b) == 3 &&
(b[0] == StringReply || b[0] == ArrayReply) &&
b[1] == '-' && b[2] == '1'
}
func ParseErrorReply(line []byte) error {
return RedisError(string(line[1:]))
}
func parseArrayLen(line []byte) (int64, error) {
if isNilReply(line) {
return 0, Nil
}
return util.ParseInt(line[1:], 10, 64)
}

View File

@ -1,166 +0,0 @@
package proto
import (
"encoding"
"fmt"
"reflect"
"github.com/go-redis/redis/internal/util"
)
func Scan(b []byte, v interface{}) error {
switch v := v.(type) {
case nil:
return fmt.Errorf("redis: Scan(nil)")
case *string:
*v = util.BytesToString(b)
return nil
case *[]byte:
*v = b
return nil
case *int:
var err error
*v, err = util.Atoi(b)
return err
case *int8:
n, err := util.ParseInt(b, 10, 8)
if err != nil {
return err
}
*v = int8(n)
return nil
case *int16:
n, err := util.ParseInt(b, 10, 16)
if err != nil {
return err
}
*v = int16(n)
return nil
case *int32:
n, err := util.ParseInt(b, 10, 32)
if err != nil {
return err
}
*v = int32(n)
return nil
case *int64:
n, err := util.ParseInt(b, 10, 64)
if err != nil {
return err
}
*v = n
return nil
case *uint:
n, err := util.ParseUint(b, 10, 64)
if err != nil {
return err
}
*v = uint(n)
return nil
case *uint8:
n, err := util.ParseUint(b, 10, 8)
if err != nil {
return err
}
*v = uint8(n)
return nil
case *uint16:
n, err := util.ParseUint(b, 10, 16)
if err != nil {
return err
}
*v = uint16(n)
return nil
case *uint32:
n, err := util.ParseUint(b, 10, 32)
if err != nil {
return err
}
*v = uint32(n)
return nil
case *uint64:
n, err := util.ParseUint(b, 10, 64)
if err != nil {
return err
}
*v = n
return nil
case *float32:
n, err := util.ParseFloat(b, 32)
if err != nil {
return err
}
*v = float32(n)
return err
case *float64:
var err error
*v, err = util.ParseFloat(b, 64)
return err
case *bool:
*v = len(b) == 1 && b[0] == '1'
return nil
case encoding.BinaryUnmarshaler:
return v.UnmarshalBinary(b)
default:
return fmt.Errorf(
"redis: can't unmarshal %T (consider implementing BinaryUnmarshaler)", v)
}
}
func ScanSlice(data []string, slice interface{}) error {
v := reflect.ValueOf(slice)
if !v.IsValid() {
return fmt.Errorf("redis: ScanSlice(nil)")
}
if v.Kind() != reflect.Ptr {
return fmt.Errorf("redis: ScanSlice(non-pointer %T)", slice)
}
v = v.Elem()
if v.Kind() != reflect.Slice {
return fmt.Errorf("redis: ScanSlice(non-slice %T)", slice)
}
next := makeSliceNextElemFunc(v)
for i, s := range data {
elem := next()
if err := Scan([]byte(s), elem.Addr().Interface()); err != nil {
err = fmt.Errorf("redis: ScanSlice index=%d value=%q failed: %s", i, s, err)
return err
}
}
return nil
}
func makeSliceNextElemFunc(v reflect.Value) func() reflect.Value {
elemType := v.Type().Elem()
if elemType.Kind() == reflect.Ptr {
elemType = elemType.Elem()
return func() reflect.Value {
if v.Len() < v.Cap() {
v.Set(v.Slice(0, v.Len()+1))
elem := v.Index(v.Len() - 1)
if elem.IsNil() {
elem.Set(reflect.New(elemType))
}
return elem.Elem()
}
elem := reflect.New(elemType)
v.Set(reflect.Append(v, elem))
return elem.Elem()
}
}
zero := reflect.Zero(elemType)
return func() reflect.Value {
if v.Len() < v.Cap() {
v.Set(v.Slice(0, v.Len()+1))
return v.Index(v.Len() - 1)
}
v.Set(reflect.Append(v, zero))
return v.Index(v.Len() - 1)
}
}

View File

@ -1,159 +0,0 @@
package proto
import (
"bufio"
"encoding"
"fmt"
"io"
"strconv"
"github.com/go-redis/redis/internal/util"
)
type Writer struct {
wr *bufio.Writer
lenBuf []byte
numBuf []byte
}
func NewWriter(wr io.Writer) *Writer {
return &Writer{
wr: bufio.NewWriter(wr),
lenBuf: make([]byte, 64),
numBuf: make([]byte, 64),
}
}
func (w *Writer) WriteArgs(args []interface{}) error {
err := w.wr.WriteByte(ArrayReply)
if err != nil {
return err
}
err = w.writeLen(len(args))
if err != nil {
return err
}
for _, arg := range args {
err := w.writeArg(arg)
if err != nil {
return err
}
}
return nil
}
func (w *Writer) writeLen(n int) error {
w.lenBuf = strconv.AppendUint(w.lenBuf[:0], uint64(n), 10)
w.lenBuf = append(w.lenBuf, '\r', '\n')
_, err := w.wr.Write(w.lenBuf)
return err
}
func (w *Writer) writeArg(v interface{}) error {
switch v := v.(type) {
case nil:
return w.string("")
case string:
return w.string(v)
case []byte:
return w.bytes(v)
case int:
return w.int(int64(v))
case int8:
return w.int(int64(v))
case int16:
return w.int(int64(v))
case int32:
return w.int(int64(v))
case int64:
return w.int(v)
case uint:
return w.uint(uint64(v))
case uint8:
return w.uint(uint64(v))
case uint16:
return w.uint(uint64(v))
case uint32:
return w.uint(uint64(v))
case uint64:
return w.uint(v)
case float32:
return w.float(float64(v))
case float64:
return w.float(v)
case bool:
if v {
return w.int(1)
} else {
return w.int(0)
}
case encoding.BinaryMarshaler:
b, err := v.MarshalBinary()
if err != nil {
return err
}
return w.bytes(b)
default:
return fmt.Errorf(
"redis: can't marshal %T (implement encoding.BinaryMarshaler)", v)
}
}
func (w *Writer) bytes(b []byte) error {
err := w.wr.WriteByte(StringReply)
if err != nil {
return err
}
err = w.writeLen(len(b))
if err != nil {
return err
}
_, err = w.wr.Write(b)
if err != nil {
return err
}
return w.crlf()
}
func (w *Writer) string(s string) error {
return w.bytes(util.StringToBytes(s))
}
func (w *Writer) uint(n uint64) error {
w.numBuf = strconv.AppendUint(w.numBuf[:0], n, 10)
return w.bytes(w.numBuf)
}
func (w *Writer) int(n int64) error {
w.numBuf = strconv.AppendInt(w.numBuf[:0], n, 10)
return w.bytes(w.numBuf)
}
func (w *Writer) float(f float64) error {
w.numBuf = strconv.AppendFloat(w.numBuf[:0], f, 'f', -1, 64)
return w.bytes(w.numBuf)
}
func (w *Writer) crlf() error {
err := w.wr.WriteByte('\r')
if err != nil {
return err
}
return w.wr.WriteByte('\n')
}
func (w *Writer) Reset(wr io.Writer) {
w.wr.Reset(wr)
}
func (w *Writer) Flush() error {
return w.wr.Flush()
}

View File

@ -1,39 +0,0 @@
package internal
import "github.com/go-redis/redis/internal/util"
func ToLower(s string) string {
if isLower(s) {
return s
}
b := make([]byte, len(s))
for i := range b {
c := s[i]
if c >= 'A' && c <= 'Z' {
c += 'a' - 'A'
}
b[i] = c
}
return util.BytesToString(b)
}
func isLower(s string) bool {
for i := 0; i < len(s); i++ {
c := s[i]
if c >= 'A' && c <= 'Z' {
return false
}
}
return true
}
func Unwrap(err error) error {
u, ok := err.(interface {
Unwrap() error
})
if !ok {
return nil
}
return u.Unwrap()
}

View File

@ -1,11 +0,0 @@
// +build appengine
package util
func BytesToString(b []byte) string {
return string(b)
}
func StringToBytes(s string) []byte {
return []byte(s)
}

View File

@ -1,19 +0,0 @@
package util
import "strconv"
func Atoi(b []byte) (int, error) {
return strconv.Atoi(BytesToString(b))
}
func ParseInt(b []byte, base int, bitSize int) (int64, error) {
return strconv.ParseInt(BytesToString(b), base, bitSize)
}
func ParseUint(b []byte, base int, bitSize int) (uint64, error) {
return strconv.ParseUint(BytesToString(b), base, bitSize)
}
func ParseFloat(b []byte, bitSize int) (float64, error) {
return strconv.ParseFloat(BytesToString(b), bitSize)
}

View File

@ -1,22 +0,0 @@
// +build !appengine
package util
import (
"unsafe"
)
// BytesToString converts byte slice to string.
func BytesToString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}
// StringToBytes converts string to byte slice.
func StringToBytes(s string) []byte {
return *(*[]byte)(unsafe.Pointer(
&struct {
string
Cap int
}{s, len(s)},
))
}

View File

@ -1,74 +0,0 @@
package redis
import "sync"
// ScanIterator is used to incrementally iterate over a collection of elements.
// It's safe for concurrent use by multiple goroutines.
type ScanIterator struct {
mu sync.Mutex // protects Scanner and pos
cmd *ScanCmd
pos int
}
// Err returns the last iterator error, if any.
func (it *ScanIterator) Err() error {
it.mu.Lock()
err := it.cmd.Err()
it.mu.Unlock()
return err
}
// Next advances the cursor and returns true if more values can be read.
func (it *ScanIterator) Next() bool {
it.mu.Lock()
defer it.mu.Unlock()
// Instantly return on errors.
if it.cmd.Err() != nil {
return false
}
// Advance cursor, check if we are still within range.
if it.pos < len(it.cmd.page) {
it.pos++
return true
}
for {
// Return if there is no more data to fetch.
if it.cmd.cursor == 0 {
return false
}
// Fetch next page.
switch it.cmd._args[0] {
case "scan", "qscan":
it.cmd._args[1] = it.cmd.cursor
default:
it.cmd._args[2] = it.cmd.cursor
}
err := it.cmd.process(it.cmd)
if err != nil {
return false
}
it.pos = 1
// Redis can occasionally return empty page.
if len(it.cmd.page) > 0 {
return true
}
}
}
// Val returns the key/field at the current cursor position.
func (it *ScanIterator) Val() string {
var v string
it.mu.Lock()
if it.cmd.Err() == nil && it.pos > 0 && it.pos <= len(it.cmd.page) {
v = it.cmd.page[it.pos-1]
}
it.mu.Unlock()
return v
}

View File

@ -1,226 +0,0 @@
package redis
import (
"crypto/tls"
"errors"
"fmt"
"net"
"net/url"
"runtime"
"strconv"
"strings"
"time"
"github.com/go-redis/redis/internal/pool"
)
// Limiter is the interface of a rate limiter or a circuit breaker.
type Limiter interface {
// Allow returns a nil if operation is allowed or an error otherwise.
// If operation is allowed client must report the result of operation
// whether is a success or a failure.
Allow() error
// ReportResult reports the result of previously allowed operation.
// nil indicates a success, non-nil error indicates a failure.
ReportResult(result error)
}
type Options struct {
// The network type, either tcp or unix.
// Default is tcp.
Network string
// host:port address.
Addr string
// Dialer creates new network connection and has priority over
// Network and Addr options.
Dialer func() (net.Conn, error)
// Hook that is called when new connection is established.
OnConnect func(*Conn) error
// Optional password. Must match the password specified in the
// requirepass server configuration option.
Password string
// Database to be selected after connecting to the server.
DB int
// Maximum number of retries before giving up.
// Default is to not retry failed commands.
MaxRetries int
// Minimum backoff between each retry.
// Default is 8 milliseconds; -1 disables backoff.
MinRetryBackoff time.Duration
// Maximum backoff between each retry.
// Default is 512 milliseconds; -1 disables backoff.
MaxRetryBackoff time.Duration
// Dial timeout for establishing new connections.
// Default is 5 seconds.
DialTimeout time.Duration
// Timeout for socket reads. If reached, commands will fail
// with a timeout instead of blocking. Use value -1 for no timeout and 0 for default.
// Default is 3 seconds.
ReadTimeout time.Duration
// Timeout for socket writes. If reached, commands will fail
// with a timeout instead of blocking.
// Default is ReadTimeout.
WriteTimeout time.Duration
// Maximum number of socket connections.
// Default is 10 connections per every CPU as reported by runtime.NumCPU.
PoolSize int
// Minimum number of idle connections which is useful when establishing
// new connection is slow.
MinIdleConns int
// Connection age at which client retires (closes) the connection.
// Default is to not close aged connections.
MaxConnAge time.Duration
// Amount of time client waits for connection if all connections
// are busy before returning an error.
// Default is ReadTimeout + 1 second.
PoolTimeout time.Duration
// Amount of time after which client closes idle connections.
// Should be less than server's timeout.
// Default is 5 minutes. -1 disables idle timeout check.
IdleTimeout time.Duration
// Frequency of idle checks made by idle connections reaper.
// Default is 1 minute. -1 disables idle connections reaper,
// but idle connections are still discarded by the client
// if IdleTimeout is set.
IdleCheckFrequency time.Duration
// Enables read only queries on slave nodes.
readOnly bool
// TLS Config to use. When set TLS will be negotiated.
TLSConfig *tls.Config
}
func (opt *Options) init() {
if opt.Network == "" {
opt.Network = "tcp"
}
if opt.Addr == "" {
opt.Addr = "localhost:6379"
}
if opt.Dialer == nil {
opt.Dialer = func() (net.Conn, error) {
netDialer := &net.Dialer{
Timeout: opt.DialTimeout,
KeepAlive: 5 * time.Minute,
}
if opt.TLSConfig == nil {
return netDialer.Dial(opt.Network, opt.Addr)
} else {
return tls.DialWithDialer(netDialer, opt.Network, opt.Addr, opt.TLSConfig)
}
}
}
if opt.PoolSize == 0 {
opt.PoolSize = 10 * runtime.NumCPU()
}
if opt.DialTimeout == 0 {
opt.DialTimeout = 5 * time.Second
}
switch opt.ReadTimeout {
case -1:
opt.ReadTimeout = 0
case 0:
opt.ReadTimeout = 3 * time.Second
}
switch opt.WriteTimeout {
case -1:
opt.WriteTimeout = 0
case 0:
opt.WriteTimeout = opt.ReadTimeout
}
if opt.PoolTimeout == 0 {
opt.PoolTimeout = opt.ReadTimeout + time.Second
}
if opt.IdleTimeout == 0 {
opt.IdleTimeout = 5 * time.Minute
}
if opt.IdleCheckFrequency == 0 {
opt.IdleCheckFrequency = time.Minute
}
switch opt.MinRetryBackoff {
case -1:
opt.MinRetryBackoff = 0
case 0:
opt.MinRetryBackoff = 8 * time.Millisecond
}
switch opt.MaxRetryBackoff {
case -1:
opt.MaxRetryBackoff = 0
case 0:
opt.MaxRetryBackoff = 512 * time.Millisecond
}
}
// ParseURL parses an URL into Options that can be used to connect to Redis.
func ParseURL(redisURL string) (*Options, error) {
o := &Options{Network: "tcp"}
u, err := url.Parse(redisURL)
if err != nil {
return nil, err
}
if u.Scheme != "redis" && u.Scheme != "rediss" {
return nil, errors.New("invalid redis URL scheme: " + u.Scheme)
}
if u.User != nil {
if p, ok := u.User.Password(); ok {
o.Password = p
}
}
if len(u.Query()) > 0 {
return nil, errors.New("no options supported")
}
h, p, err := net.SplitHostPort(u.Host)
if err != nil {
h = u.Host
}
if h == "" {
h = "localhost"
}
if p == "" {
p = "6379"
}
o.Addr = net.JoinHostPort(h, p)
f := strings.FieldsFunc(u.Path, func(r rune) bool {
return r == '/'
})
switch len(f) {
case 0:
o.DB = 0
case 1:
if o.DB, err = strconv.Atoi(f[0]); err != nil {
return nil, fmt.Errorf("invalid redis database number: %q", f[0])
}
default:
return nil, errors.New("invalid redis URL path: " + u.Path)
}
if u.Scheme == "rediss" {
o.TLSConfig = &tls.Config{ServerName: h}
}
return o, nil
}
func newConnPool(opt *Options) *pool.ConnPool {
return pool.NewConnPool(&pool.Options{
Dialer: opt.Dialer,
PoolSize: opt.PoolSize,
MinIdleConns: opt.MinIdleConns,
MaxConnAge: opt.MaxConnAge,
PoolTimeout: opt.PoolTimeout,
IdleTimeout: opt.IdleTimeout,
IdleCheckFrequency: opt.IdleCheckFrequency,
})
}

View File

@ -1,133 +0,0 @@
package redis
import (
"sync"
"github.com/go-redis/redis/internal/pool"
)
type pipelineExecer func([]Cmder) error
// Pipeliner is an mechanism to realise Redis Pipeline technique.
//
// Pipelining is a technique to extremely speed up processing by packing
// operations to batches, send them at once to Redis and read a replies in a
// singe step.
// See https://redis.io/topics/pipelining
//
// Pay attention, that Pipeline is not a transaction, so you can get unexpected
// results in case of big pipelines and small read/write timeouts.
// Redis client has retransmission logic in case of timeouts, pipeline
// can be retransmitted and commands can be executed more then once.
// To avoid this: it is good idea to use reasonable bigger read/write timeouts
// depends of your batch size and/or use TxPipeline.
type Pipeliner interface {
StatefulCmdable
Do(args ...interface{}) *Cmd
Process(cmd Cmder) error
Close() error
Discard() error
Exec() ([]Cmder, error)
}
var _ Pipeliner = (*Pipeline)(nil)
// Pipeline implements pipelining as described in
// http://redis.io/topics/pipelining. It's safe for concurrent use
// by multiple goroutines.
type Pipeline struct {
statefulCmdable
exec pipelineExecer
mu sync.Mutex
cmds []Cmder
closed bool
}
func (c *Pipeline) Do(args ...interface{}) *Cmd {
cmd := NewCmd(args...)
_ = c.Process(cmd)
return cmd
}
// Process queues the cmd for later execution.
func (c *Pipeline) Process(cmd Cmder) error {
c.mu.Lock()
c.cmds = append(c.cmds, cmd)
c.mu.Unlock()
return nil
}
// Close closes the pipeline, releasing any open resources.
func (c *Pipeline) Close() error {
c.mu.Lock()
c.discard()
c.closed = true
c.mu.Unlock()
return nil
}
// Discard resets the pipeline and discards queued commands.
func (c *Pipeline) Discard() error {
c.mu.Lock()
err := c.discard()
c.mu.Unlock()
return err
}
func (c *Pipeline) discard() error {
if c.closed {
return pool.ErrClosed
}
c.cmds = c.cmds[:0]
return nil
}
// Exec executes all previously queued commands using one
// client-server roundtrip.
//
// Exec always returns list of commands and error of the first failed
// command if any.
func (c *Pipeline) Exec() ([]Cmder, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return nil, pool.ErrClosed
}
if len(c.cmds) == 0 {
return nil, nil
}
cmds := c.cmds
c.cmds = nil
return cmds, c.exec(cmds)
}
func (c *Pipeline) pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
if err := fn(c); err != nil {
return nil, err
}
cmds, err := c.Exec()
_ = c.Close()
return cmds, err
}
func (c *Pipeline) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.pipelined(fn)
}
func (c *Pipeline) Pipeline() Pipeliner {
return c
}
func (c *Pipeline) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.pipelined(fn)
}
func (c *Pipeline) TxPipeline() Pipeliner {
return c
}

View File

@ -1,514 +0,0 @@
package redis
import (
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/go-redis/redis/internal"
"github.com/go-redis/redis/internal/pool"
"github.com/go-redis/redis/internal/proto"
)
var errPingTimeout = errors.New("redis: ping timeout")
// PubSub implements Pub/Sub commands as described in
// http://redis.io/topics/pubsub. Message receiving is NOT safe
// for concurrent use by multiple goroutines.
//
// PubSub automatically reconnects to Redis Server and resubscribes
// to the channels in case of network errors.
type PubSub struct {
opt *Options
newConn func([]string) (*pool.Conn, error)
closeConn func(*pool.Conn) error
mu sync.Mutex
cn *pool.Conn
channels map[string]struct{}
patterns map[string]struct{}
closed bool
exit chan struct{}
cmd *Cmd
chOnce sync.Once
ch chan *Message
ping chan struct{}
}
func (c *PubSub) String() string {
channels := mapKeys(c.channels)
channels = append(channels, mapKeys(c.patterns)...)
return fmt.Sprintf("PubSub(%s)", strings.Join(channels, ", "))
}
func (c *PubSub) init() {
c.exit = make(chan struct{})
}
func (c *PubSub) conn() (*pool.Conn, error) {
c.mu.Lock()
cn, err := c._conn(nil)
c.mu.Unlock()
return cn, err
}
func (c *PubSub) _conn(newChannels []string) (*pool.Conn, error) {
if c.closed {
return nil, pool.ErrClosed
}
if c.cn != nil {
return c.cn, nil
}
channels := mapKeys(c.channels)
channels = append(channels, newChannels...)
cn, err := c.newConn(channels)
if err != nil {
return nil, err
}
if err := c.resubscribe(cn); err != nil {
_ = c.closeConn(cn)
return nil, err
}
c.cn = cn
return cn, nil
}
func (c *PubSub) writeCmd(cn *pool.Conn, cmd Cmder) error {
return cn.WithWriter(c.opt.WriteTimeout, func(wr *proto.Writer) error {
return writeCmd(wr, cmd)
})
}
func (c *PubSub) resubscribe(cn *pool.Conn) error {
var firstErr error
if len(c.channels) > 0 {
err := c._subscribe(cn, "subscribe", mapKeys(c.channels))
if err != nil && firstErr == nil {
firstErr = err
}
}
if len(c.patterns) > 0 {
err := c._subscribe(cn, "psubscribe", mapKeys(c.patterns))
if err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func mapKeys(m map[string]struct{}) []string {
s := make([]string, len(m))
i := 0
for k := range m {
s[i] = k
i++
}
return s
}
func (c *PubSub) _subscribe(
cn *pool.Conn, redisCmd string, channels []string,
) error {
args := make([]interface{}, 0, 1+len(channels))
args = append(args, redisCmd)
for _, channel := range channels {
args = append(args, channel)
}
cmd := NewSliceCmd(args...)
return c.writeCmd(cn, cmd)
}
func (c *PubSub) releaseConn(cn *pool.Conn, err error, allowTimeout bool) {
c.mu.Lock()
c._releaseConn(cn, err, allowTimeout)
c.mu.Unlock()
}
func (c *PubSub) _releaseConn(cn *pool.Conn, err error, allowTimeout bool) {
if c.cn != cn {
return
}
if internal.IsBadConn(err, allowTimeout) {
c._reconnect(err)
}
}
func (c *PubSub) _reconnect(reason error) {
_ = c._closeTheCn(reason)
_, _ = c._conn(nil)
}
func (c *PubSub) _closeTheCn(reason error) error {
if c.cn == nil {
return nil
}
if !c.closed {
internal.Logf("redis: discarding bad PubSub connection: %s", reason)
}
err := c.closeConn(c.cn)
c.cn = nil
return err
}
func (c *PubSub) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return pool.ErrClosed
}
c.closed = true
close(c.exit)
err := c._closeTheCn(pool.ErrClosed)
return err
}
// Subscribe the client to the specified channels. It returns
// empty subscription if there are no channels.
func (c *PubSub) Subscribe(channels ...string) error {
c.mu.Lock()
defer c.mu.Unlock()
err := c.subscribe("subscribe", channels...)
if c.channels == nil {
c.channels = make(map[string]struct{})
}
for _, s := range channels {
c.channels[s] = struct{}{}
}
return err
}
// PSubscribe the client to the given patterns. It returns
// empty subscription if there are no patterns.
func (c *PubSub) PSubscribe(patterns ...string) error {
c.mu.Lock()
defer c.mu.Unlock()
err := c.subscribe("psubscribe", patterns...)
if c.patterns == nil {
c.patterns = make(map[string]struct{})
}
for _, s := range patterns {
c.patterns[s] = struct{}{}
}
return err
}
// Unsubscribe the client from the given channels, or from all of
// them if none is given.
func (c *PubSub) Unsubscribe(channels ...string) error {
c.mu.Lock()
defer c.mu.Unlock()
for _, channel := range channels {
delete(c.channels, channel)
}
err := c.subscribe("unsubscribe", channels...)
return err
}
// PUnsubscribe the client from the given patterns, or from all of
// them if none is given.
func (c *PubSub) PUnsubscribe(patterns ...string) error {
c.mu.Lock()
defer c.mu.Unlock()
for _, pattern := range patterns {
delete(c.patterns, pattern)
}
err := c.subscribe("punsubscribe", patterns...)
return err
}
func (c *PubSub) subscribe(redisCmd string, channels ...string) error {
cn, err := c._conn(channels)
if err != nil {
return err
}
err = c._subscribe(cn, redisCmd, channels)
c._releaseConn(cn, err, false)
return err
}
func (c *PubSub) Ping(payload ...string) error {
args := []interface{}{"ping"}
if len(payload) == 1 {
args = append(args, payload[0])
}
cmd := NewCmd(args...)
cn, err := c.conn()
if err != nil {
return err
}
err = c.writeCmd(cn, cmd)
c.releaseConn(cn, err, false)
return err
}
// Subscription received after a successful subscription to channel.
type Subscription struct {
// Can be "subscribe", "unsubscribe", "psubscribe" or "punsubscribe".
Kind string
// Channel name we have subscribed to.
Channel string
// Number of channels we are currently subscribed to.
Count int
}
func (m *Subscription) String() string {
return fmt.Sprintf("%s: %s", m.Kind, m.Channel)
}
// Message received as result of a PUBLISH command issued by another client.
type Message struct {
Channel string
Pattern string
Payload string
}
func (m *Message) String() string {
return fmt.Sprintf("Message<%s: %s>", m.Channel, m.Payload)
}
// Pong received as result of a PING command issued by another client.
type Pong struct {
Payload string
}
func (p *Pong) String() string {
if p.Payload != "" {
return fmt.Sprintf("Pong<%s>", p.Payload)
}
return "Pong"
}
func (c *PubSub) newMessage(reply interface{}) (interface{}, error) {
switch reply := reply.(type) {
case string:
return &Pong{
Payload: reply,
}, nil
case []interface{}:
switch kind := reply[0].(string); kind {
case "subscribe", "unsubscribe", "psubscribe", "punsubscribe":
channel, _ := reply[1].(string)
return &Subscription{
Kind: kind,
Channel: channel,
Count: int(reply[2].(int64)),
}, nil
case "message":
return &Message{
Channel: reply[1].(string),
Payload: reply[2].(string),
}, nil
case "pmessage":
return &Message{
Pattern: reply[1].(string),
Channel: reply[2].(string),
Payload: reply[3].(string),
}, nil
case "pong":
return &Pong{
Payload: reply[1].(string),
}, nil
default:
return nil, fmt.Errorf("redis: unsupported pubsub message: %q", kind)
}
default:
return nil, fmt.Errorf("redis: unsupported pubsub message: %#v", reply)
}
}
// ReceiveTimeout acts like Receive but returns an error if message
// is not received in time. This is low-level API and in most cases
// Channel should be used instead.
func (c *PubSub) ReceiveTimeout(timeout time.Duration) (interface{}, error) {
if c.cmd == nil {
c.cmd = NewCmd()
}
cn, err := c.conn()
if err != nil {
return nil, err
}
err = cn.WithReader(timeout, func(rd *proto.Reader) error {
return c.cmd.readReply(rd)
})
c.releaseConn(cn, err, timeout > 0)
if err != nil {
return nil, err
}
return c.newMessage(c.cmd.Val())
}
// Receive returns a message as a Subscription, Message, Pong or error.
// See PubSub example for details. This is low-level API and in most cases
// Channel should be used instead.
func (c *PubSub) Receive() (interface{}, error) {
return c.ReceiveTimeout(0)
}
// ReceiveMessage returns a Message or error ignoring Subscription and Pong
// messages. This is low-level API and in most cases Channel should be used
// instead.
func (c *PubSub) ReceiveMessage() (*Message, error) {
for {
msg, err := c.Receive()
if err != nil {
return nil, err
}
switch msg := msg.(type) {
case *Subscription:
// Ignore.
case *Pong:
// Ignore.
case *Message:
return msg, nil
default:
err := fmt.Errorf("redis: unknown message: %T", msg)
return nil, err
}
}
}
// Channel returns a Go channel for concurrently receiving messages.
// It periodically sends Ping messages to test connection health.
// The channel is closed with PubSub. Receive* APIs can not be used
// after channel is created.
//
// If the Go channel is full for 30 seconds the message is dropped.
func (c *PubSub) Channel() <-chan *Message {
return c.channel(100)
}
// ChannelSize is like Channel, but creates a Go channel
// with specified buffer size.
func (c *PubSub) ChannelSize(size int) <-chan *Message {
return c.channel(size)
}
func (c *PubSub) channel(size int) <-chan *Message {
c.chOnce.Do(func() {
c.initChannel(size)
})
if cap(c.ch) != size {
err := fmt.Errorf("redis: PubSub.Channel is called with different buffer size")
panic(err)
}
return c.ch
}
func (c *PubSub) initChannel(size int) {
const timeout = 30 * time.Second
c.ch = make(chan *Message, size)
c.ping = make(chan struct{}, 1)
go func() {
timer := time.NewTimer(timeout)
timer.Stop()
var errCount int
for {
msg, err := c.Receive()
if err != nil {
if err == pool.ErrClosed {
close(c.ch)
return
}
if errCount > 0 {
time.Sleep(c.retryBackoff(errCount))
}
errCount++
continue
}
errCount = 0
// Any message is as good as a ping.
select {
case c.ping <- struct{}{}:
default:
}
switch msg := msg.(type) {
case *Subscription:
// Ignore.
case *Pong:
// Ignore.
case *Message:
timer.Reset(timeout)
select {
case c.ch <- msg:
if !timer.Stop() {
<-timer.C
}
case <-timer.C:
internal.Logf(
"redis: %s channel is full for %s (message is dropped)",
c, timeout)
}
default:
internal.Logf("redis: unknown message type: %T", msg)
}
}
}()
go func() {
timer := time.NewTimer(timeout)
timer.Stop()
healthy := true
for {
timer.Reset(timeout)
select {
case <-c.ping:
healthy = true
if !timer.Stop() {
<-timer.C
}
case <-timer.C:
pingErr := c.Ping()
if healthy {
healthy = false
} else {
if pingErr == nil {
pingErr = errPingTimeout
}
c.mu.Lock()
c._reconnect(pingErr)
c.mu.Unlock()
}
case <-c.exit:
return
}
}
}()
}
func (c *PubSub) retryBackoff(attempt int) time.Duration {
return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff)
}

View File

@ -1,583 +0,0 @@
package redis
import (
"context"
"fmt"
"log"
"os"
"time"
"github.com/go-redis/redis/internal"
"github.com/go-redis/redis/internal/pool"
"github.com/go-redis/redis/internal/proto"
)
// Nil reply Redis returns when key does not exist.
const Nil = proto.Nil
func init() {
SetLogger(log.New(os.Stderr, "redis: ", log.LstdFlags|log.Lshortfile))
}
func SetLogger(logger *log.Logger) {
internal.Logger = logger
}
type baseClient struct {
opt *Options
connPool pool.Pooler
limiter Limiter
process func(Cmder) error
processPipeline func([]Cmder) error
processTxPipeline func([]Cmder) error
onClose func() error // hook called when client is closed
}
func (c *baseClient) init() {
c.process = c.defaultProcess
c.processPipeline = c.defaultProcessPipeline
c.processTxPipeline = c.defaultProcessTxPipeline
}
func (c *baseClient) String() string {
return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB)
}
func (c *baseClient) newConn() (*pool.Conn, error) {
cn, err := c.connPool.NewConn()
if err != nil {
return nil, err
}
err = c.initConn(cn)
if err != nil {
_ = c.connPool.CloseConn(cn)
return nil, err
}
return cn, nil
}
func (c *baseClient) getConn() (*pool.Conn, error) {
if c.limiter != nil {
err := c.limiter.Allow()
if err != nil {
return nil, err
}
}
cn, err := c._getConn()
if err != nil {
if c.limiter != nil {
c.limiter.ReportResult(err)
}
return nil, err
}
return cn, nil
}
func (c *baseClient) _getConn() (*pool.Conn, error) {
cn, err := c.connPool.Get()
if err != nil {
return nil, err
}
err = c.initConn(cn)
if err != nil {
c.connPool.Remove(cn, err)
if err := internal.Unwrap(err); err != nil {
return nil, err
}
return nil, err
}
return cn, nil
}
func (c *baseClient) releaseConn(cn *pool.Conn, err error) {
if c.limiter != nil {
c.limiter.ReportResult(err)
}
if internal.IsBadConn(err, false) {
c.connPool.Remove(cn, err)
} else {
c.connPool.Put(cn)
}
}
func (c *baseClient) releaseConnStrict(cn *pool.Conn, err error) {
if c.limiter != nil {
c.limiter.ReportResult(err)
}
if err == nil || internal.IsRedisError(err) {
c.connPool.Put(cn)
} else {
c.connPool.Remove(cn, err)
}
}
func (c *baseClient) initConn(cn *pool.Conn) error {
if cn.Inited {
return nil
}
cn.Inited = true
if c.opt.Password == "" &&
c.opt.DB == 0 &&
!c.opt.readOnly &&
c.opt.OnConnect == nil {
return nil
}
conn := newConn(c.opt, cn)
_, err := conn.Pipelined(func(pipe Pipeliner) error {
if c.opt.Password != "" {
pipe.Auth(c.opt.Password)
}
if c.opt.DB > 0 {
pipe.Select(c.opt.DB)
}
if c.opt.readOnly {
pipe.ReadOnly()
}
return nil
})
if err != nil {
return err
}
if c.opt.OnConnect != nil {
return c.opt.OnConnect(conn)
}
return nil
}
// Do creates a Cmd from the args and processes the cmd.
func (c *baseClient) Do(args ...interface{}) *Cmd {
cmd := NewCmd(args...)
_ = c.Process(cmd)
return cmd
}
// WrapProcess wraps function that processes Redis commands.
func (c *baseClient) WrapProcess(
fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error,
) {
c.process = fn(c.process)
}
func (c *baseClient) Process(cmd Cmder) error {
return c.process(cmd)
}
func (c *baseClient) defaultProcess(cmd Cmder) error {
for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(c.retryBackoff(attempt))
}
cn, err := c.getConn()
if err != nil {
cmd.setErr(err)
if internal.IsRetryableError(err, true) {
continue
}
return err
}
err = cn.WithWriter(c.opt.WriteTimeout, func(wr *proto.Writer) error {
return writeCmd(wr, cmd)
})
if err != nil {
c.releaseConn(cn, err)
cmd.setErr(err)
if internal.IsRetryableError(err, true) {
continue
}
return err
}
err = cn.WithReader(c.cmdTimeout(cmd), cmd.readReply)
c.releaseConn(cn, err)
if err != nil && internal.IsRetryableError(err, cmd.readTimeout() == nil) {
continue
}
return err
}
return cmd.Err()
}
func (c *baseClient) retryBackoff(attempt int) time.Duration {
return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff)
}
func (c *baseClient) cmdTimeout(cmd Cmder) time.Duration {
if timeout := cmd.readTimeout(); timeout != nil {
t := *timeout
if t == 0 {
return 0
}
return t + 10*time.Second
}
return c.opt.ReadTimeout
}
// Close closes the client, releasing any open resources.
//
// It is rare to Close a Client, as the Client is meant to be
// long-lived and shared between many goroutines.
func (c *baseClient) Close() error {
var firstErr error
if c.onClose != nil {
if err := c.onClose(); err != nil {
firstErr = err
}
}
if err := c.connPool.Close(); err != nil && firstErr == nil {
firstErr = err
}
return firstErr
}
func (c *baseClient) getAddr() string {
return c.opt.Addr
}
func (c *baseClient) WrapProcessPipeline(
fn func(oldProcess func([]Cmder) error) func([]Cmder) error,
) {
c.processPipeline = fn(c.processPipeline)
c.processTxPipeline = fn(c.processTxPipeline)
}
func (c *baseClient) defaultProcessPipeline(cmds []Cmder) error {
return c.generalProcessPipeline(cmds, c.pipelineProcessCmds)
}
func (c *baseClient) defaultProcessTxPipeline(cmds []Cmder) error {
return c.generalProcessPipeline(cmds, c.txPipelineProcessCmds)
}
type pipelineProcessor func(*pool.Conn, []Cmder) (bool, error)
func (c *baseClient) generalProcessPipeline(cmds []Cmder, p pipelineProcessor) error {
for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(c.retryBackoff(attempt))
}
cn, err := c.getConn()
if err != nil {
setCmdsErr(cmds, err)
return err
}
canRetry, err := p(cn, cmds)
c.releaseConnStrict(cn, err)
if !canRetry || !internal.IsRetryableError(err, true) {
break
}
}
return cmdsFirstErr(cmds)
}
func (c *baseClient) pipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) {
err := cn.WithWriter(c.opt.WriteTimeout, func(wr *proto.Writer) error {
return writeCmd(wr, cmds...)
})
if err != nil {
setCmdsErr(cmds, err)
return true, err
}
err = cn.WithReader(c.opt.ReadTimeout, func(rd *proto.Reader) error {
return pipelineReadCmds(rd, cmds)
})
return true, err
}
func pipelineReadCmds(rd *proto.Reader, cmds []Cmder) error {
for _, cmd := range cmds {
err := cmd.readReply(rd)
if err != nil && !internal.IsRedisError(err) {
return err
}
}
return nil
}
func (c *baseClient) txPipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) {
err := cn.WithWriter(c.opt.WriteTimeout, func(wr *proto.Writer) error {
return txPipelineWriteMulti(wr, cmds)
})
if err != nil {
setCmdsErr(cmds, err)
return true, err
}
err = cn.WithReader(c.opt.ReadTimeout, func(rd *proto.Reader) error {
err := txPipelineReadQueued(rd, cmds)
if err != nil {
setCmdsErr(cmds, err)
return err
}
return pipelineReadCmds(rd, cmds)
})
return false, err
}
func txPipelineWriteMulti(wr *proto.Writer, cmds []Cmder) error {
multiExec := make([]Cmder, 0, len(cmds)+2)
multiExec = append(multiExec, NewStatusCmd("MULTI"))
multiExec = append(multiExec, cmds...)
multiExec = append(multiExec, NewSliceCmd("EXEC"))
return writeCmd(wr, multiExec...)
}
func txPipelineReadQueued(rd *proto.Reader, cmds []Cmder) error {
// Parse queued replies.
var statusCmd StatusCmd
err := statusCmd.readReply(rd)
if err != nil {
return err
}
for range cmds {
err = statusCmd.readReply(rd)
if err != nil && !internal.IsRedisError(err) {
return err
}
}
// Parse number of replies.
line, err := rd.ReadLine()
if err != nil {
if err == Nil {
err = TxFailedErr
}
return err
}
switch line[0] {
case proto.ErrorReply:
return proto.ParseErrorReply(line)
case proto.ArrayReply:
// ok
default:
err := fmt.Errorf("redis: expected '*', but got line %q", line)
return err
}
return nil
}
//------------------------------------------------------------------------------
// Client is a Redis client representing a pool of zero or more
// underlying connections. It's safe for concurrent use by multiple
// goroutines.
type Client struct {
baseClient
cmdable
ctx context.Context
}
// NewClient returns a client to the Redis Server specified by Options.
func NewClient(opt *Options) *Client {
opt.init()
c := Client{
baseClient: baseClient{
opt: opt,
connPool: newConnPool(opt),
},
}
c.baseClient.init()
c.init()
return &c
}
func (c *Client) init() {
c.cmdable.setProcessor(c.Process)
}
func (c *Client) Context() context.Context {
if c.ctx != nil {
return c.ctx
}
return context.Background()
}
func (c *Client) WithContext(ctx context.Context) *Client {
if ctx == nil {
panic("nil context")
}
c2 := c.clone()
c2.ctx = ctx
return c2
}
func (c *Client) clone() *Client {
cp := *c
cp.init()
return &cp
}
// Options returns read-only Options that were used to create the client.
func (c *Client) Options() *Options {
return c.opt
}
func (c *Client) SetLimiter(l Limiter) *Client {
c.limiter = l
return c
}
type PoolStats pool.Stats
// PoolStats returns connection pool stats.
func (c *Client) PoolStats() *PoolStats {
stats := c.connPool.Stats()
return (*PoolStats)(stats)
}
func (c *Client) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.Pipeline().Pipelined(fn)
}
func (c *Client) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.processPipeline,
}
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
func (c *Client) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.TxPipeline().Pipelined(fn)
}
// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC.
func (c *Client) TxPipeline() Pipeliner {
pipe := Pipeline{
exec: c.processTxPipeline,
}
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
func (c *Client) pubSub() *PubSub {
pubsub := &PubSub{
opt: c.opt,
newConn: func(channels []string) (*pool.Conn, error) {
return c.newConn()
},
closeConn: c.connPool.CloseConn,
}
pubsub.init()
return pubsub
}
// Subscribe subscribes the client to the specified channels.
// Channels can be omitted to create empty subscription.
// Note that this method does not wait on a response from Redis, so the
// subscription may not be active immediately. To force the connection to wait,
// you may call the Receive() method on the returned *PubSub like so:
//
// sub := client.Subscribe(queryResp)
// iface, err := sub.Receive()
// if err != nil {
// // handle error
// }
//
// // Should be *Subscription, but others are possible if other actions have been
// // taken on sub since it was created.
// switch iface.(type) {
// case *Subscription:
// // subscribe succeeded
// case *Message:
// // received first message
// case *Pong:
// // pong received
// default:
// // handle error
// }
//
// ch := sub.Channel()
func (c *Client) Subscribe(channels ...string) *PubSub {
pubsub := c.pubSub()
if len(channels) > 0 {
_ = pubsub.Subscribe(channels...)
}
return pubsub
}
// PSubscribe subscribes the client to the given patterns.
// Patterns can be omitted to create empty subscription.
func (c *Client) PSubscribe(channels ...string) *PubSub {
pubsub := c.pubSub()
if len(channels) > 0 {
_ = pubsub.PSubscribe(channels...)
}
return pubsub
}
//------------------------------------------------------------------------------
// Conn is like Client, but its pool contains single connection.
type Conn struct {
baseClient
statefulCmdable
}
func newConn(opt *Options, cn *pool.Conn) *Conn {
connPool := pool.NewSingleConnPool(nil)
connPool.SetConn(cn)
c := Conn{
baseClient: baseClient{
opt: opt,
connPool: connPool,
},
}
c.baseClient.init()
c.statefulCmdable.setProcessor(c.Process)
return &c
}
func (c *Conn) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.Pipeline().Pipelined(fn)
}
func (c *Conn) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.processPipeline,
}
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
func (c *Conn) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.TxPipeline().Pipelined(fn)
}
// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC.
func (c *Conn) TxPipeline() Pipeliner {
pipe := Pipeline{
exec: c.processTxPipeline,
}
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}

View File

@ -1,140 +0,0 @@
package redis
import "time"
// NewCmdResult returns a Cmd initialised with val and err for testing
func NewCmdResult(val interface{}, err error) *Cmd {
var cmd Cmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewSliceResult returns a SliceCmd initialised with val and err for testing
func NewSliceResult(val []interface{}, err error) *SliceCmd {
var cmd SliceCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewStatusResult returns a StatusCmd initialised with val and err for testing
func NewStatusResult(val string, err error) *StatusCmd {
var cmd StatusCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewIntResult returns an IntCmd initialised with val and err for testing
func NewIntResult(val int64, err error) *IntCmd {
var cmd IntCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewDurationResult returns a DurationCmd initialised with val and err for testing
func NewDurationResult(val time.Duration, err error) *DurationCmd {
var cmd DurationCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewBoolResult returns a BoolCmd initialised with val and err for testing
func NewBoolResult(val bool, err error) *BoolCmd {
var cmd BoolCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewStringResult returns a StringCmd initialised with val and err for testing
func NewStringResult(val string, err error) *StringCmd {
var cmd StringCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewFloatResult returns a FloatCmd initialised with val and err for testing
func NewFloatResult(val float64, err error) *FloatCmd {
var cmd FloatCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewStringSliceResult returns a StringSliceCmd initialised with val and err for testing
func NewStringSliceResult(val []string, err error) *StringSliceCmd {
var cmd StringSliceCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewBoolSliceResult returns a BoolSliceCmd initialised with val and err for testing
func NewBoolSliceResult(val []bool, err error) *BoolSliceCmd {
var cmd BoolSliceCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewStringStringMapResult returns a StringStringMapCmd initialised with val and err for testing
func NewStringStringMapResult(val map[string]string, err error) *StringStringMapCmd {
var cmd StringStringMapCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewStringIntMapCmdResult returns a StringIntMapCmd initialised with val and err for testing
func NewStringIntMapCmdResult(val map[string]int64, err error) *StringIntMapCmd {
var cmd StringIntMapCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewZSliceCmdResult returns a ZSliceCmd initialised with val and err for testing
func NewZSliceCmdResult(val []Z, err error) *ZSliceCmd {
var cmd ZSliceCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewScanCmdResult returns a ScanCmd initialised with val and err for testing
func NewScanCmdResult(keys []string, cursor uint64, err error) *ScanCmd {
var cmd ScanCmd
cmd.page = keys
cmd.cursor = cursor
cmd.setErr(err)
return &cmd
}
// NewClusterSlotsCmdResult returns a ClusterSlotsCmd initialised with val and err for testing
func NewClusterSlotsCmdResult(val []ClusterSlot, err error) *ClusterSlotsCmd {
var cmd ClusterSlotsCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}
// NewGeoLocationCmdResult returns a GeoLocationCmd initialised with val and err for testing
func NewGeoLocationCmdResult(val []GeoLocation, err error) *GeoLocationCmd {
var cmd GeoLocationCmd
cmd.locations = val
cmd.setErr(err)
return &cmd
}
// NewCommandsInfoCmdResult returns a CommandsInfoCmd initialised with val and err for testing
func NewCommandsInfoCmdResult(val map[string]*CommandInfo, err error) *CommandsInfoCmd {
var cmd CommandsInfoCmd
cmd.val = val
cmd.setErr(err)
return &cmd
}

View File

@ -1,702 +0,0 @@
package redis
import (
"context"
"errors"
"fmt"
"math/rand"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/go-redis/redis/internal"
"github.com/go-redis/redis/internal/consistenthash"
"github.com/go-redis/redis/internal/hashtag"
"github.com/go-redis/redis/internal/pool"
)
// Hash is type of hash function used in consistent hash.
type Hash consistenthash.Hash
var errRingShardsDown = errors.New("redis: all ring shards are down")
// RingOptions are used to configure a ring client and should be
// passed to NewRing.
type RingOptions struct {
// Map of name => host:port addresses of ring shards.
Addrs map[string]string
// Frequency of PING commands sent to check shards availability.
// Shard is considered down after 3 subsequent failed checks.
HeartbeatFrequency time.Duration
// Hash function used in consistent hash.
// Default is crc32.ChecksumIEEE.
Hash Hash
// Number of replicas in consistent hash.
// Default is 100 replicas.
//
// Higher number of replicas will provide less deviation, that is keys will be
// distributed to nodes more evenly.
//
// Following is deviation for common nreplicas:
// --------------------------------------------------------
// | nreplicas | standard error | 99% confidence interval |
// | 10 | 0.3152 | (0.37, 1.98) |
// | 100 | 0.0997 | (0.76, 1.28) |
// | 1000 | 0.0316 | (0.92, 1.09) |
// --------------------------------------------------------
//
// See https://arxiv.org/abs/1406.2294 for reference
HashReplicas int
// Following options are copied from Options struct.
OnConnect func(*Conn) error
DB int
Password string
MaxRetries int
MinRetryBackoff time.Duration
MaxRetryBackoff time.Duration
DialTimeout time.Duration
ReadTimeout time.Duration
WriteTimeout time.Duration
PoolSize int
MinIdleConns int
MaxConnAge time.Duration
PoolTimeout time.Duration
IdleTimeout time.Duration
IdleCheckFrequency time.Duration
}
func (opt *RingOptions) init() {
if opt.HeartbeatFrequency == 0 {
opt.HeartbeatFrequency = 500 * time.Millisecond
}
if opt.HashReplicas == 0 {
opt.HashReplicas = 100
}
switch opt.MinRetryBackoff {
case -1:
opt.MinRetryBackoff = 0
case 0:
opt.MinRetryBackoff = 8 * time.Millisecond
}
switch opt.MaxRetryBackoff {
case -1:
opt.MaxRetryBackoff = 0
case 0:
opt.MaxRetryBackoff = 512 * time.Millisecond
}
}
func (opt *RingOptions) clientOptions() *Options {
return &Options{
OnConnect: opt.OnConnect,
DB: opt.DB,
Password: opt.Password,
DialTimeout: opt.DialTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
PoolSize: opt.PoolSize,
MinIdleConns: opt.MinIdleConns,
MaxConnAge: opt.MaxConnAge,
PoolTimeout: opt.PoolTimeout,
IdleTimeout: opt.IdleTimeout,
IdleCheckFrequency: opt.IdleCheckFrequency,
}
}
//------------------------------------------------------------------------------
type ringShard struct {
Client *Client
down int32
}
func (shard *ringShard) String() string {
var state string
if shard.IsUp() {
state = "up"
} else {
state = "down"
}
return fmt.Sprintf("%s is %s", shard.Client, state)
}
func (shard *ringShard) IsDown() bool {
const threshold = 3
return atomic.LoadInt32(&shard.down) >= threshold
}
func (shard *ringShard) IsUp() bool {
return !shard.IsDown()
}
// Vote votes to set shard state and returns true if state was changed.
func (shard *ringShard) Vote(up bool) bool {
if up {
changed := shard.IsDown()
atomic.StoreInt32(&shard.down, 0)
return changed
}
if shard.IsDown() {
return false
}
atomic.AddInt32(&shard.down, 1)
return shard.IsDown()
}
//------------------------------------------------------------------------------
type ringShards struct {
opt *RingOptions
mu sync.RWMutex
hash *consistenthash.Map
shards map[string]*ringShard // read only
list []*ringShard // read only
len int
closed bool
}
func newRingShards(opt *RingOptions) *ringShards {
return &ringShards{
opt: opt,
hash: newConsistentHash(opt),
shards: make(map[string]*ringShard),
}
}
func (c *ringShards) Add(name string, cl *Client) {
shard := &ringShard{Client: cl}
c.hash.Add(name)
c.shards[name] = shard
c.list = append(c.list, shard)
}
func (c *ringShards) List() []*ringShard {
c.mu.RLock()
list := c.list
c.mu.RUnlock()
return list
}
func (c *ringShards) Hash(key string) string {
c.mu.RLock()
hash := c.hash.Get(key)
c.mu.RUnlock()
return hash
}
func (c *ringShards) GetByKey(key string) (*ringShard, error) {
key = hashtag.Key(key)
c.mu.RLock()
if c.closed {
c.mu.RUnlock()
return nil, pool.ErrClosed
}
hash := c.hash.Get(key)
if hash == "" {
c.mu.RUnlock()
return nil, errRingShardsDown
}
shard := c.shards[hash]
c.mu.RUnlock()
return shard, nil
}
func (c *ringShards) GetByHash(name string) (*ringShard, error) {
if name == "" {
return c.Random()
}
c.mu.RLock()
shard := c.shards[name]
c.mu.RUnlock()
return shard, nil
}
func (c *ringShards) Random() (*ringShard, error) {
return c.GetByKey(strconv.Itoa(rand.Int()))
}
// heartbeat monitors state of each shard in the ring.
func (c *ringShards) Heartbeat(frequency time.Duration) {
ticker := time.NewTicker(frequency)
defer ticker.Stop()
for range ticker.C {
var rebalance bool
c.mu.RLock()
if c.closed {
c.mu.RUnlock()
break
}
shards := c.list
c.mu.RUnlock()
for _, shard := range shards {
err := shard.Client.Ping().Err()
if shard.Vote(err == nil || err == pool.ErrPoolTimeout) {
internal.Logf("ring shard state changed: %s", shard)
rebalance = true
}
}
if rebalance {
c.rebalance()
}
}
}
// rebalance removes dead shards from the Ring.
func (c *ringShards) rebalance() {
c.mu.RLock()
shards := c.shards
c.mu.RUnlock()
hash := newConsistentHash(c.opt)
var shardsNum int
for name, shard := range shards {
if shard.IsUp() {
hash.Add(name)
shardsNum++
}
}
c.mu.Lock()
c.hash = hash
c.len = shardsNum
c.mu.Unlock()
}
func (c *ringShards) Len() int {
c.mu.RLock()
l := c.len
c.mu.RUnlock()
return l
}
func (c *ringShards) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return nil
}
c.closed = true
var firstErr error
for _, shard := range c.shards {
if err := shard.Client.Close(); err != nil && firstErr == nil {
firstErr = err
}
}
c.hash = nil
c.shards = nil
c.list = nil
return firstErr
}
//------------------------------------------------------------------------------
// Ring is a Redis client that uses consistent hashing to distribute
// keys across multiple Redis servers (shards). It's safe for
// concurrent use by multiple goroutines.
//
// Ring monitors the state of each shard and removes dead shards from
// the ring. When a shard comes online it is added back to the ring. This
// gives you maximum availability and partition tolerance, but no
// consistency between different shards or even clients. Each client
// uses shards that are available to the client and does not do any
// coordination when shard state is changed.
//
// Ring should be used when you need multiple Redis servers for caching
// and can tolerate losing data when one of the servers dies.
// Otherwise you should use Redis Cluster.
type Ring struct {
cmdable
ctx context.Context
opt *RingOptions
shards *ringShards
cmdsInfoCache *cmdsInfoCache
process func(Cmder) error
processPipeline func([]Cmder) error
}
func NewRing(opt *RingOptions) *Ring {
opt.init()
ring := &Ring{
opt: opt,
shards: newRingShards(opt),
}
ring.cmdsInfoCache = newCmdsInfoCache(ring.cmdsInfo)
ring.process = ring.defaultProcess
ring.processPipeline = ring.defaultProcessPipeline
ring.init()
for name, addr := range opt.Addrs {
clopt := opt.clientOptions()
clopt.Addr = addr
ring.shards.Add(name, NewClient(clopt))
}
go ring.shards.Heartbeat(opt.HeartbeatFrequency)
return ring
}
func (c *Ring) init() {
c.cmdable.setProcessor(c.Process)
}
func (c *Ring) Context() context.Context {
if c.ctx != nil {
return c.ctx
}
return context.Background()
}
func (c *Ring) WithContext(ctx context.Context) *Ring {
if ctx == nil {
panic("nil context")
}
c2 := c.clone()
c2.ctx = ctx
return c2
}
func (c *Ring) clone() *Ring {
cp := *c
cp.init()
return &cp
}
// Options returns read-only Options that were used to create the client.
func (c *Ring) Options() *RingOptions {
return c.opt
}
func (c *Ring) retryBackoff(attempt int) time.Duration {
return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff)
}
// PoolStats returns accumulated connection pool stats.
func (c *Ring) PoolStats() *PoolStats {
shards := c.shards.List()
var acc PoolStats
for _, shard := range shards {
s := shard.Client.connPool.Stats()
acc.Hits += s.Hits
acc.Misses += s.Misses
acc.Timeouts += s.Timeouts
acc.TotalConns += s.TotalConns
acc.IdleConns += s.IdleConns
}
return &acc
}
// Len returns the current number of shards in the ring.
func (c *Ring) Len() int {
return c.shards.Len()
}
// Subscribe subscribes the client to the specified channels.
func (c *Ring) Subscribe(channels ...string) *PubSub {
if len(channels) == 0 {
panic("at least one channel is required")
}
shard, err := c.shards.GetByKey(channels[0])
if err != nil {
// TODO: return PubSub with sticky error
panic(err)
}
return shard.Client.Subscribe(channels...)
}
// PSubscribe subscribes the client to the given patterns.
func (c *Ring) PSubscribe(channels ...string) *PubSub {
if len(channels) == 0 {
panic("at least one channel is required")
}
shard, err := c.shards.GetByKey(channels[0])
if err != nil {
// TODO: return PubSub with sticky error
panic(err)
}
return shard.Client.PSubscribe(channels...)
}
// ForEachShard concurrently calls the fn on each live shard in the ring.
// It returns the first error if any.
func (c *Ring) ForEachShard(fn func(client *Client) error) error {
shards := c.shards.List()
var wg sync.WaitGroup
errCh := make(chan error, 1)
for _, shard := range shards {
if shard.IsDown() {
continue
}
wg.Add(1)
go func(shard *ringShard) {
defer wg.Done()
err := fn(shard.Client)
if err != nil {
select {
case errCh <- err:
default:
}
}
}(shard)
}
wg.Wait()
select {
case err := <-errCh:
return err
default:
return nil
}
}
func (c *Ring) cmdsInfo() (map[string]*CommandInfo, error) {
shards := c.shards.List()
firstErr := errRingShardsDown
for _, shard := range shards {
cmdsInfo, err := shard.Client.Command().Result()
if err == nil {
return cmdsInfo, nil
}
if firstErr == nil {
firstErr = err
}
}
return nil, firstErr
}
func (c *Ring) cmdInfo(name string) *CommandInfo {
cmdsInfo, err := c.cmdsInfoCache.Get()
if err != nil {
return nil
}
info := cmdsInfo[name]
if info == nil {
internal.Logf("info for cmd=%s not found", name)
}
return info
}
func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) {
cmdInfo := c.cmdInfo(cmd.Name())
pos := cmdFirstKeyPos(cmd, cmdInfo)
if pos == 0 {
return c.shards.Random()
}
firstKey := cmd.stringArg(pos)
return c.shards.GetByKey(firstKey)
}
// Do creates a Cmd from the args and processes the cmd.
func (c *Ring) Do(args ...interface{}) *Cmd {
cmd := NewCmd(args...)
c.Process(cmd)
return cmd
}
func (c *Ring) WrapProcess(
fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error,
) {
c.process = fn(c.process)
}
func (c *Ring) Process(cmd Cmder) error {
return c.process(cmd)
}
func (c *Ring) defaultProcess(cmd Cmder) error {
for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(c.retryBackoff(attempt))
}
shard, err := c.cmdShard(cmd)
if err != nil {
cmd.setErr(err)
return err
}
err = shard.Client.Process(cmd)
if err == nil {
return nil
}
if !internal.IsRetryableError(err, cmd.readTimeout() == nil) {
return err
}
}
return cmd.Err()
}
func (c *Ring) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.processPipeline,
}
pipe.cmdable.setProcessor(pipe.Process)
return &pipe
}
func (c *Ring) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.Pipeline().Pipelined(fn)
}
func (c *Ring) WrapProcessPipeline(
fn func(oldProcess func([]Cmder) error) func([]Cmder) error,
) {
c.processPipeline = fn(c.processPipeline)
}
func (c *Ring) defaultProcessPipeline(cmds []Cmder) error {
cmdsMap := make(map[string][]Cmder)
for _, cmd := range cmds {
cmdInfo := c.cmdInfo(cmd.Name())
hash := cmd.stringArg(cmdFirstKeyPos(cmd, cmdInfo))
if hash != "" {
hash = c.shards.Hash(hashtag.Key(hash))
}
cmdsMap[hash] = append(cmdsMap[hash], cmd)
}
for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(c.retryBackoff(attempt))
}
var mu sync.Mutex
var failedCmdsMap map[string][]Cmder
var wg sync.WaitGroup
for hash, cmds := range cmdsMap {
wg.Add(1)
go func(hash string, cmds []Cmder) {
defer wg.Done()
shard, err := c.shards.GetByHash(hash)
if err != nil {
setCmdsErr(cmds, err)
return
}
cn, err := shard.Client.getConn()
if err != nil {
setCmdsErr(cmds, err)
return
}
canRetry, err := shard.Client.pipelineProcessCmds(cn, cmds)
shard.Client.releaseConnStrict(cn, err)
if canRetry && internal.IsRetryableError(err, true) {
mu.Lock()
if failedCmdsMap == nil {
failedCmdsMap = make(map[string][]Cmder)
}
failedCmdsMap[hash] = cmds
mu.Unlock()
}
}(hash, cmds)
}
wg.Wait()
if len(failedCmdsMap) == 0 {
break
}
cmdsMap = failedCmdsMap
}
return cmdsFirstErr(cmds)
}
func (c *Ring) TxPipeline() Pipeliner {
panic("not implemented")
}
func (c *Ring) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
panic("not implemented")
}
// Close closes the ring client, releasing any open resources.
//
// It is rare to Close a Ring, as the Ring is meant to be long-lived
// and shared between many goroutines.
func (c *Ring) Close() error {
return c.shards.Close()
}
func (c *Ring) Watch(fn func(*Tx) error, keys ...string) error {
if len(keys) == 0 {
return fmt.Errorf("redis: Watch requires at least one key")
}
var shards []*ringShard
for _, key := range keys {
if key != "" {
shard, err := c.shards.GetByKey(hashtag.Key(key))
if err != nil {
return err
}
shards = append(shards, shard)
}
}
if len(shards) == 0 {
return fmt.Errorf("redis: Watch requires at least one shard")
}
if len(shards) > 1 {
for _, shard := range shards[1:] {
if shard.Client != shards[0].Client {
err := fmt.Errorf("redis: Watch requires all keys to be in the same shard")
return err
}
}
}
return shards[0].Client.Watch(fn, keys...)
}
func newConsistentHash(opt *RingOptions) *consistenthash.Map {
return consistenthash.New(opt.HashReplicas, consistenthash.Hash(opt.Hash))
}

View File

@ -1,62 +0,0 @@
package redis
import (
"crypto/sha1"
"encoding/hex"
"io"
"strings"
)
type scripter interface {
Eval(script string, keys []string, args ...interface{}) *Cmd
EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd
ScriptExists(hashes ...string) *BoolSliceCmd
ScriptLoad(script string) *StringCmd
}
var _ scripter = (*Client)(nil)
var _ scripter = (*Ring)(nil)
var _ scripter = (*ClusterClient)(nil)
type Script struct {
src, hash string
}
func NewScript(src string) *Script {
h := sha1.New()
io.WriteString(h, src)
return &Script{
src: src,
hash: hex.EncodeToString(h.Sum(nil)),
}
}
func (s *Script) Hash() string {
return s.hash
}
func (s *Script) Load(c scripter) *StringCmd {
return c.ScriptLoad(s.src)
}
func (s *Script) Exists(c scripter) *BoolSliceCmd {
return c.ScriptExists(s.hash)
}
func (s *Script) Eval(c scripter, keys []string, args ...interface{}) *Cmd {
return c.Eval(s.src, keys, args...)
}
func (s *Script) EvalSha(c scripter, keys []string, args ...interface{}) *Cmd {
return c.EvalSha(s.hash, keys, args...)
}
// Run optimistically uses EVALSHA to run the script. If script does not exist
// it is retried using EVAL.
func (s *Script) Run(c scripter, keys []string, args ...interface{}) *Cmd {
r := s.EvalSha(c, keys, args...)
if err := r.Err(); err != nil && strings.HasPrefix(err.Error(), "NOSCRIPT ") {
return s.Eval(c, keys, args...)
}
return r
}

View File

@ -1,415 +0,0 @@
package redis
import (
"crypto/tls"
"errors"
"net"
"strings"
"sync"
"time"
"github.com/go-redis/redis/internal"
"github.com/go-redis/redis/internal/pool"
)
//------------------------------------------------------------------------------
// FailoverOptions are used to configure a failover client and should
// be passed to NewFailoverClient.
type FailoverOptions struct {
// The master name.
MasterName string
// A seed list of host:port addresses of sentinel nodes.
SentinelAddrs []string
// Following options are copied from Options struct.
OnConnect func(*Conn) error
Password string
DB int
MaxRetries int
MinRetryBackoff time.Duration
MaxRetryBackoff time.Duration
DialTimeout time.Duration
ReadTimeout time.Duration
WriteTimeout time.Duration
PoolSize int
MinIdleConns int
MaxConnAge time.Duration
PoolTimeout time.Duration
IdleTimeout time.Duration
IdleCheckFrequency time.Duration
TLSConfig *tls.Config
}
func (opt *FailoverOptions) options() *Options {
return &Options{
Addr: "FailoverClient",
OnConnect: opt.OnConnect,
DB: opt.DB,
Password: opt.Password,
MaxRetries: opt.MaxRetries,
DialTimeout: opt.DialTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
PoolSize: opt.PoolSize,
PoolTimeout: opt.PoolTimeout,
IdleTimeout: opt.IdleTimeout,
IdleCheckFrequency: opt.IdleCheckFrequency,
TLSConfig: opt.TLSConfig,
}
}
// NewFailoverClient returns a Redis client that uses Redis Sentinel
// for automatic failover. It's safe for concurrent use by multiple
// goroutines.
func NewFailoverClient(failoverOpt *FailoverOptions) *Client {
opt := failoverOpt.options()
opt.init()
failover := &sentinelFailover{
masterName: failoverOpt.MasterName,
sentinelAddrs: failoverOpt.SentinelAddrs,
opt: opt,
}
c := Client{
baseClient: baseClient{
opt: opt,
connPool: failover.Pool(),
onClose: failover.Close,
},
}
c.baseClient.init()
c.cmdable.setProcessor(c.Process)
return &c
}
//------------------------------------------------------------------------------
type SentinelClient struct {
baseClient
}
func NewSentinelClient(opt *Options) *SentinelClient {
opt.init()
c := &SentinelClient{
baseClient: baseClient{
opt: opt,
connPool: newConnPool(opt),
},
}
c.baseClient.init()
return c
}
func (c *SentinelClient) pubSub() *PubSub {
pubsub := &PubSub{
opt: c.opt,
newConn: func(channels []string) (*pool.Conn, error) {
return c.newConn()
},
closeConn: c.connPool.CloseConn,
}
pubsub.init()
return pubsub
}
// Subscribe subscribes the client to the specified channels.
// Channels can be omitted to create empty subscription.
func (c *SentinelClient) Subscribe(channels ...string) *PubSub {
pubsub := c.pubSub()
if len(channels) > 0 {
_ = pubsub.Subscribe(channels...)
}
return pubsub
}
// PSubscribe subscribes the client to the given patterns.
// Patterns can be omitted to create empty subscription.
func (c *SentinelClient) PSubscribe(channels ...string) *PubSub {
pubsub := c.pubSub()
if len(channels) > 0 {
_ = pubsub.PSubscribe(channels...)
}
return pubsub
}
func (c *SentinelClient) GetMasterAddrByName(name string) *StringSliceCmd {
cmd := NewStringSliceCmd("sentinel", "get-master-addr-by-name", name)
c.Process(cmd)
return cmd
}
func (c *SentinelClient) Sentinels(name string) *SliceCmd {
cmd := NewSliceCmd("sentinel", "sentinels", name)
c.Process(cmd)
return cmd
}
// Failover forces a failover as if the master was not reachable, and without
// asking for agreement to other Sentinels.
func (c *SentinelClient) Failover(name string) *StatusCmd {
cmd := NewStatusCmd("sentinel", "failover", name)
c.Process(cmd)
return cmd
}
// Reset resets all the masters with matching name. The pattern argument is a
// glob-style pattern. The reset process clears any previous state in a master
// (including a failover in progress), and removes every slave and sentinel
// already discovered and associated with the master.
func (c *SentinelClient) Reset(pattern string) *IntCmd {
cmd := NewIntCmd("sentinel", "reset", pattern)
c.Process(cmd)
return cmd
}
// FlushConfig forces Sentinel to rewrite its configuration on disk, including
// the current Sentinel state.
func (c *SentinelClient) FlushConfig() *StatusCmd {
cmd := NewStatusCmd("sentinel", "flushconfig")
c.Process(cmd)
return cmd
}
// Master shows the state and info of the specified master.
func (c *SentinelClient) Master(name string) *StringStringMapCmd {
cmd := NewStringStringMapCmd("sentinel", "master", name)
c.Process(cmd)
return cmd
}
type sentinelFailover struct {
sentinelAddrs []string
opt *Options
pool *pool.ConnPool
poolOnce sync.Once
mu sync.RWMutex
masterName string
_masterAddr string
sentinel *SentinelClient
pubsub *PubSub
}
func (c *sentinelFailover) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.sentinel != nil {
return c.closeSentinel()
}
return nil
}
func (c *sentinelFailover) Pool() *pool.ConnPool {
c.poolOnce.Do(func() {
c.opt.Dialer = c.dial
c.pool = newConnPool(c.opt)
})
return c.pool
}
func (c *sentinelFailover) dial() (net.Conn, error) {
addr, err := c.MasterAddr()
if err != nil {
return nil, err
}
return net.DialTimeout("tcp", addr, c.opt.DialTimeout)
}
func (c *sentinelFailover) MasterAddr() (string, error) {
addr, err := c.masterAddr()
if err != nil {
return "", err
}
c.switchMaster(addr)
return addr, nil
}
func (c *sentinelFailover) masterAddr() (string, error) {
c.mu.RLock()
addr := c.getMasterAddr()
c.mu.RUnlock()
if addr != "" {
return addr, nil
}
c.mu.Lock()
defer c.mu.Unlock()
addr = c.getMasterAddr()
if addr != "" {
return addr, nil
}
if c.sentinel != nil {
c.closeSentinel()
}
for i, sentinelAddr := range c.sentinelAddrs {
sentinel := NewSentinelClient(&Options{
Addr: sentinelAddr,
MaxRetries: c.opt.MaxRetries,
DialTimeout: c.opt.DialTimeout,
ReadTimeout: c.opt.ReadTimeout,
WriteTimeout: c.opt.WriteTimeout,
PoolSize: c.opt.PoolSize,
PoolTimeout: c.opt.PoolTimeout,
IdleTimeout: c.opt.IdleTimeout,
IdleCheckFrequency: c.opt.IdleCheckFrequency,
TLSConfig: c.opt.TLSConfig,
})
masterAddr, err := sentinel.GetMasterAddrByName(c.masterName).Result()
if err != nil {
internal.Logf("sentinel: GetMasterAddrByName master=%q failed: %s",
c.masterName, err)
_ = sentinel.Close()
continue
}
// Push working sentinel to the top.
c.sentinelAddrs[0], c.sentinelAddrs[i] = c.sentinelAddrs[i], c.sentinelAddrs[0]
c.setSentinel(sentinel)
addr := net.JoinHostPort(masterAddr[0], masterAddr[1])
return addr, nil
}
return "", errors.New("redis: all sentinels are unreachable")
}
func (c *sentinelFailover) getMasterAddr() string {
sentinel := c.sentinel
if sentinel == nil {
return ""
}
addr, err := sentinel.GetMasterAddrByName(c.masterName).Result()
if err != nil {
internal.Logf("sentinel: GetMasterAddrByName name=%q failed: %s",
c.masterName, err)
return ""
}
return net.JoinHostPort(addr[0], addr[1])
}
func (c *sentinelFailover) switchMaster(addr string) {
c.mu.RLock()
masterAddr := c._masterAddr
c.mu.RUnlock()
if masterAddr == addr {
return
}
c.mu.Lock()
defer c.mu.Unlock()
internal.Logf("sentinel: new master=%q addr=%q",
c.masterName, addr)
_ = c.Pool().Filter(func(cn *pool.Conn) bool {
return cn.RemoteAddr().String() != addr
})
c._masterAddr = addr
}
func (c *sentinelFailover) setSentinel(sentinel *SentinelClient) {
c.discoverSentinels(sentinel)
c.sentinel = sentinel
c.pubsub = sentinel.Subscribe("+switch-master")
go c.listen(c.pubsub)
}
func (c *sentinelFailover) closeSentinel() error {
var firstErr error
err := c.pubsub.Close()
if err != nil && firstErr == err {
firstErr = err
}
c.pubsub = nil
err = c.sentinel.Close()
if err != nil && firstErr == err {
firstErr = err
}
c.sentinel = nil
return firstErr
}
func (c *sentinelFailover) discoverSentinels(sentinel *SentinelClient) {
sentinels, err := sentinel.Sentinels(c.masterName).Result()
if err != nil {
internal.Logf("sentinel: Sentinels master=%q failed: %s", c.masterName, err)
return
}
for _, sentinel := range sentinels {
vals := sentinel.([]interface{})
for i := 0; i < len(vals); i += 2 {
key := vals[i].(string)
if key == "name" {
sentinelAddr := vals[i+1].(string)
if !contains(c.sentinelAddrs, sentinelAddr) {
internal.Logf("sentinel: discovered new sentinel=%q for master=%q",
sentinelAddr, c.masterName)
c.sentinelAddrs = append(c.sentinelAddrs, sentinelAddr)
}
}
}
}
}
func (c *sentinelFailover) listen(pubsub *PubSub) {
ch := pubsub.Channel()
for {
msg, ok := <-ch
if !ok {
break
}
if msg.Channel == "+switch-master" {
parts := strings.Split(msg.Payload, " ")
if parts[0] != c.masterName {
internal.Logf("sentinel: ignore addr for master=%q", parts[0])
continue
}
addr := net.JoinHostPort(parts[3], parts[4])
c.switchMaster(addr)
}
}
}
func contains(slice []string, str string) bool {
for _, s := range slice {
if s == str {
return true
}
}
return false
}

View File

@ -1,110 +0,0 @@
package redis
import (
"github.com/go-redis/redis/internal/pool"
"github.com/go-redis/redis/internal/proto"
)
// TxFailedErr transaction redis failed.
const TxFailedErr = proto.RedisError("redis: transaction failed")
// Tx implements Redis transactions as described in
// http://redis.io/topics/transactions. It's NOT safe for concurrent use
// by multiple goroutines, because Exec resets list of watched keys.
// If you don't need WATCH it is better to use Pipeline.
type Tx struct {
statefulCmdable
baseClient
}
func (c *Client) newTx() *Tx {
tx := Tx{
baseClient: baseClient{
opt: c.opt,
connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), true),
},
}
tx.baseClient.init()
tx.statefulCmdable.setProcessor(tx.Process)
return &tx
}
// Watch prepares a transaction and marks the keys to be watched
// for conditional execution if there are any keys.
//
// The transaction is automatically closed when fn exits.
func (c *Client) Watch(fn func(*Tx) error, keys ...string) error {
tx := c.newTx()
if len(keys) > 0 {
if err := tx.Watch(keys...).Err(); err != nil {
_ = tx.Close()
return err
}
}
err := fn(tx)
_ = tx.Close()
return err
}
// Close closes the transaction, releasing any open resources.
func (c *Tx) Close() error {
_ = c.Unwatch().Err()
return c.baseClient.Close()
}
// Watch marks the keys to be watched for conditional execution
// of a transaction.
func (c *Tx) Watch(keys ...string) *StatusCmd {
args := make([]interface{}, 1+len(keys))
args[0] = "watch"
for i, key := range keys {
args[1+i] = key
}
cmd := NewStatusCmd(args...)
c.Process(cmd)
return cmd
}
// Unwatch flushes all the previously watched keys for a transaction.
func (c *Tx) Unwatch(keys ...string) *StatusCmd {
args := make([]interface{}, 1+len(keys))
args[0] = "unwatch"
for i, key := range keys {
args[1+i] = key
}
cmd := NewStatusCmd(args...)
c.Process(cmd)
return cmd
}
// Pipeline creates a new pipeline. It is more convenient to use Pipelined.
func (c *Tx) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.processTxPipeline,
}
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
// Pipelined executes commands queued in the fn in a transaction.
//
// When using WATCH, EXEC will execute commands only if the watched keys
// were not modified, allowing for a check-and-set mechanism.
//
// Exec always returns list of commands. If transaction fails
// TxFailedErr is returned. Otherwise Exec returns an error of the first
// failed command or nil.
func (c *Tx) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.Pipeline().Pipelined(fn)
}
// TxPipelined is an alias for Pipelined.
func (c *Tx) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.Pipelined(fn)
}
// TxPipeline is an alias for Pipeline.
func (c *Tx) TxPipeline() Pipeliner {
return c.Pipeline()
}

View File

@ -1,180 +0,0 @@
package redis
import (
"crypto/tls"
"time"
)
// UniversalOptions information is required by UniversalClient to establish
// connections.
type UniversalOptions struct {
// Either a single address or a seed list of host:port addresses
// of cluster/sentinel nodes.
Addrs []string
// Database to be selected after connecting to the server.
// Only single-node and failover clients.
DB int
// Common options.
OnConnect func(*Conn) error
Password string
MaxRetries int
MinRetryBackoff time.Duration
MaxRetryBackoff time.Duration
DialTimeout time.Duration
ReadTimeout time.Duration
WriteTimeout time.Duration
PoolSize int
MinIdleConns int
MaxConnAge time.Duration
PoolTimeout time.Duration
IdleTimeout time.Duration
IdleCheckFrequency time.Duration
TLSConfig *tls.Config
// Only cluster clients.
MaxRedirects int
ReadOnly bool
RouteByLatency bool
RouteRandomly bool
// The sentinel master name.
// Only failover clients.
MasterName string
}
func (o *UniversalOptions) cluster() *ClusterOptions {
if len(o.Addrs) == 0 {
o.Addrs = []string{"127.0.0.1:6379"}
}
return &ClusterOptions{
Addrs: o.Addrs,
OnConnect: o.OnConnect,
Password: o.Password,
MaxRedirects: o.MaxRedirects,
ReadOnly: o.ReadOnly,
RouteByLatency: o.RouteByLatency,
RouteRandomly: o.RouteRandomly,
MaxRetries: o.MaxRetries,
MinRetryBackoff: o.MinRetryBackoff,
MaxRetryBackoff: o.MaxRetryBackoff,
DialTimeout: o.DialTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
PoolSize: o.PoolSize,
MinIdleConns: o.MinIdleConns,
MaxConnAge: o.MaxConnAge,
PoolTimeout: o.PoolTimeout,
IdleTimeout: o.IdleTimeout,
IdleCheckFrequency: o.IdleCheckFrequency,
TLSConfig: o.TLSConfig,
}
}
func (o *UniversalOptions) failover() *FailoverOptions {
if len(o.Addrs) == 0 {
o.Addrs = []string{"127.0.0.1:26379"}
}
return &FailoverOptions{
SentinelAddrs: o.Addrs,
MasterName: o.MasterName,
OnConnect: o.OnConnect,
DB: o.DB,
Password: o.Password,
MaxRetries: o.MaxRetries,
MinRetryBackoff: o.MinRetryBackoff,
MaxRetryBackoff: o.MaxRetryBackoff,
DialTimeout: o.DialTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
PoolSize: o.PoolSize,
MinIdleConns: o.MinIdleConns,
MaxConnAge: o.MaxConnAge,
PoolTimeout: o.PoolTimeout,
IdleTimeout: o.IdleTimeout,
IdleCheckFrequency: o.IdleCheckFrequency,
TLSConfig: o.TLSConfig,
}
}
func (o *UniversalOptions) simple() *Options {
addr := "127.0.0.1:6379"
if len(o.Addrs) > 0 {
addr = o.Addrs[0]
}
return &Options{
Addr: addr,
OnConnect: o.OnConnect,
DB: o.DB,
Password: o.Password,
MaxRetries: o.MaxRetries,
MinRetryBackoff: o.MinRetryBackoff,
MaxRetryBackoff: o.MaxRetryBackoff,
DialTimeout: o.DialTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
PoolSize: o.PoolSize,
MinIdleConns: o.MinIdleConns,
MaxConnAge: o.MaxConnAge,
PoolTimeout: o.PoolTimeout,
IdleTimeout: o.IdleTimeout,
IdleCheckFrequency: o.IdleCheckFrequency,
TLSConfig: o.TLSConfig,
}
}
// --------------------------------------------------------------------
// UniversalClient is an abstract client which - based on the provided options -
// can connect to either clusters, or sentinel-backed failover instances or simple
// single-instance servers. This can be useful for testing cluster-specific
// applications locally.
type UniversalClient interface {
Cmdable
Watch(fn func(*Tx) error, keys ...string) error
Process(cmd Cmder) error
WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error)
WrapProcessPipeline(fn func(oldProcess func([]Cmder) error) func([]Cmder) error)
Subscribe(channels ...string) *PubSub
PSubscribe(channels ...string) *PubSub
Close() error
}
var _ UniversalClient = (*Client)(nil)
var _ UniversalClient = (*ClusterClient)(nil)
// NewUniversalClient returns a new multi client. The type of client returned depends
// on the following three conditions:
//
// 1. if a MasterName is passed a sentinel-backed FailoverClient will be returned
// 2. if the number of Addrs is two or more, a ClusterClient will be returned
// 3. otherwise, a single-node redis Client will be returned.
func NewUniversalClient(opts *UniversalOptions) UniversalClient {
if opts.MasterName != "" {
return NewFailoverClient(opts.failover())
} else if len(opts.Addrs) > 1 {
return NewClusterClient(opts.cluster())
}
return NewClient(opts.simple())
}

View File

@ -1,20 +0,0 @@
; https://editorconfig.org/
root = true
[*]
insert_final_newline = true
charset = utf-8
trim_trailing_whitespace = true
indent_style = space
indent_size = 2
[{Makefile,go.mod,go.sum,*.go,.gitmodules}]
indent_style = tab
indent_size = 4
[*.md]
indent_size = 4
trim_trailing_whitespace = false
eclint_indent_style = unset

View File

@ -1 +0,0 @@
coverage.coverprofile

View File

@ -1,3 +0,0 @@
run:
skip-dirs:
- examples/*.go

View File

@ -1,27 +0,0 @@
Copyright (c) 2023 The Gorilla Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,34 +0,0 @@
GO_LINT=$(shell which golangci-lint 2> /dev/null || echo '')
GO_LINT_URI=github.com/golangci/golangci-lint/cmd/golangci-lint@latest
GO_SEC=$(shell which gosec 2> /dev/null || echo '')
GO_SEC_URI=github.com/securego/gosec/v2/cmd/gosec@latest
GO_VULNCHECK=$(shell which govulncheck 2> /dev/null || echo '')
GO_VULNCHECK_URI=golang.org/x/vuln/cmd/govulncheck@latest
.PHONY: golangci-lint
golangci-lint:
$(if $(GO_LINT), ,go install $(GO_LINT_URI))
@echo "##### Running golangci-lint"
golangci-lint run -v
.PHONY: gosec
gosec:
$(if $(GO_SEC), ,go install $(GO_SEC_URI))
@echo "##### Running gosec"
gosec -exclude-dir examples ./...
.PHONY: govulncheck
govulncheck:
$(if $(GO_VULNCHECK), ,go install $(GO_VULNCHECK_URI))
@echo "##### Running govulncheck"
govulncheck ./...
.PHONY: verify
verify: golangci-lint gosec govulncheck
.PHONY: test
test:
@echo "##### Running tests"
go test -race -cover -coverprofile=coverage.coverprofile -covermode=atomic -v ./...

View File

@ -1,36 +0,0 @@
# gorilla/websocket
![testing](https://github.com/gorilla/websocket/actions/workflows/test.yml/badge.svg)
[![codecov](https://codecov.io/github/gorilla/websocket/branch/main/graph/badge.svg)](https://codecov.io/github/gorilla/websocket)
[![godoc](https://godoc.org/github.com/gorilla/websocket?status.svg)](https://godoc.org/github.com/gorilla/websocket)
[![sourcegraph](https://sourcegraph.com/github.com/gorilla/websocket/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/websocket?badge)
Gorilla WebSocket is a [Go](http://golang.org/) implementation of the [WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol.
![Gorilla Logo](https://github.com/gorilla/.github/assets/53367916/d92caabf-98e0-473e-bfbf-ab554ba435e5)
### Documentation
* [API Reference](https://pkg.go.dev/github.com/gorilla/websocket?tab=doc)
* [Chat example](https://github.com/gorilla/websocket/tree/master/examples/chat)
* [Command example](https://github.com/gorilla/websocket/tree/master/examples/command)
* [Client and server example](https://github.com/gorilla/websocket/tree/master/examples/echo)
* [File watch example](https://github.com/gorilla/websocket/tree/master/examples/filewatch)
* [Write buffer pool example](https://github.com/gorilla/websocket/tree/master/examples/bufferpool)
### Status
The Gorilla WebSocket package provides a complete and tested implementation of
the [WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol. The
package API is stable.
### Installation
go get github.com/gorilla/websocket
### Protocol Compliance
The Gorilla WebSocket package passes the server tests in the [Autobahn Test
Suite](https://github.com/crossbario/autobahn-testsuite) using the application in the [examples/autobahn
subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn).

View File

@ -1,444 +0,0 @@
// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/http/httptrace"
"net/url"
"strings"
"time"
"golang.org/x/net/proxy"
)
// ErrBadHandshake is returned when the server response to opening handshake is
// invalid.
var ErrBadHandshake = errors.New("websocket: bad handshake")
var errInvalidCompression = errors.New("websocket: invalid compression negotiation")
// NewClient creates a new client connection using the given net connection.
// The URL u specifies the host and request URI. Use requestHeader to specify
// the origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies
// (Cookie). Use the response.Header to get the selected subprotocol
// (Sec-WebSocket-Protocol) and cookies (Set-Cookie).
//
// If the WebSocket handshake fails, ErrBadHandshake is returned along with a
// non-nil *http.Response so that callers can handle redirects, authentication,
// etc.
//
// Deprecated: Use Dialer instead.
func NewClient(netConn net.Conn, u *url.URL, requestHeader http.Header, readBufSize, writeBufSize int) (c *Conn, response *http.Response, err error) {
d := Dialer{
ReadBufferSize: readBufSize,
WriteBufferSize: writeBufSize,
NetDial: func(net, addr string) (net.Conn, error) {
return netConn, nil
},
}
return d.Dial(u.String(), requestHeader)
}
// A Dialer contains options for connecting to WebSocket server.
//
// It is safe to call Dialer's methods concurrently.
type Dialer struct {
// NetDial specifies the dial function for creating TCP connections. If
// NetDial is nil, net.Dial is used.
NetDial func(network, addr string) (net.Conn, error)
// NetDialContext specifies the dial function for creating TCP connections. If
// NetDialContext is nil, NetDial is used.
NetDialContext func(ctx context.Context, network, addr string) (net.Conn, error)
// NetDialTLSContext specifies the dial function for creating TLS/TCP connections. If
// NetDialTLSContext is nil, NetDialContext is used.
// If NetDialTLSContext is set, Dial assumes the TLS handshake is done there and
// TLSClientConfig is ignored.
NetDialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error)
// Proxy specifies a function to return a proxy for a given
// Request. If the function returns a non-nil error, the
// request is aborted with the provided error.
// If Proxy is nil or returns a nil *URL, no proxy is used.
Proxy func(*http.Request) (*url.URL, error)
// TLSClientConfig specifies the TLS configuration to use with tls.Client.
// If nil, the default configuration is used.
// If either NetDialTLS or NetDialTLSContext are set, Dial assumes the TLS handshake
// is done there and TLSClientConfig is ignored.
TLSClientConfig *tls.Config
// HandshakeTimeout specifies the duration for the handshake to complete.
HandshakeTimeout time.Duration
// ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer
// size is zero, then a useful default size is used. The I/O buffer sizes
// do not limit the size of the messages that can be sent or received.
ReadBufferSize, WriteBufferSize int
// WriteBufferPool is a pool of buffers for write operations. If the value
// is not set, then write buffers are allocated to the connection for the
// lifetime of the connection.
//
// A pool is most useful when the application has a modest volume of writes
// across a large number of connections.
//
// Applications should use a single pool for each unique value of
// WriteBufferSize.
WriteBufferPool BufferPool
// Subprotocols specifies the client's requested subprotocols.
Subprotocols []string
// EnableCompression specifies if the client should attempt to negotiate
// per message compression (RFC 7692). Setting this value to true does not
// guarantee that compression will be supported. Currently only "no context
// takeover" modes are supported.
EnableCompression bool
// Jar specifies the cookie jar.
// If Jar is nil, cookies are not sent in requests and ignored
// in responses.
Jar http.CookieJar
}
// Dial creates a new client connection by calling DialContext with a background context.
func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) {
return d.DialContext(context.Background(), urlStr, requestHeader)
}
var errMalformedURL = errors.New("malformed ws or wss URL")
func hostPortNoPort(u *url.URL) (hostPort, hostNoPort string) {
hostPort = u.Host
hostNoPort = u.Host
if i := strings.LastIndex(u.Host, ":"); i > strings.LastIndex(u.Host, "]") {
hostNoPort = hostNoPort[:i]
} else {
switch u.Scheme {
case "wss":
hostPort += ":443"
case "https":
hostPort += ":443"
default:
hostPort += ":80"
}
}
return hostPort, hostNoPort
}
// DefaultDialer is a dialer with all fields set to the default values.
var DefaultDialer = &Dialer{
Proxy: http.ProxyFromEnvironment,
HandshakeTimeout: 45 * time.Second,
}
// nilDialer is dialer to use when receiver is nil.
var nilDialer = *DefaultDialer
// DialContext creates a new client connection. Use requestHeader to specify the
// origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies (Cookie).
// Use the response.Header to get the selected subprotocol
// (Sec-WebSocket-Protocol) and cookies (Set-Cookie).
//
// The context will be used in the request and in the Dialer.
//
// If the WebSocket handshake fails, ErrBadHandshake is returned along with a
// non-nil *http.Response so that callers can handle redirects, authentication,
// etcetera. The response body may not contain the entire response and does not
// need to be closed by the application.
func (d *Dialer) DialContext(ctx context.Context, urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) {
if d == nil {
d = &nilDialer
}
challengeKey, err := generateChallengeKey()
if err != nil {
return nil, nil, err
}
u, err := url.Parse(urlStr)
if err != nil {
return nil, nil, err
}
switch u.Scheme {
case "ws":
u.Scheme = "http"
case "wss":
u.Scheme = "https"
default:
return nil, nil, errMalformedURL
}
if u.User != nil {
// User name and password are not allowed in websocket URIs.
return nil, nil, errMalformedURL
}
req := &http.Request{
Method: http.MethodGet,
URL: u,
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
Header: make(http.Header),
Host: u.Host,
}
req = req.WithContext(ctx)
// Set the cookies present in the cookie jar of the dialer
if d.Jar != nil {
for _, cookie := range d.Jar.Cookies(u) {
req.AddCookie(cookie)
}
}
// Set the request headers using the capitalization for names and values in
// RFC examples. Although the capitalization shouldn't matter, there are
// servers that depend on it. The Header.Set method is not used because the
// method canonicalizes the header names.
req.Header["Upgrade"] = []string{"websocket"}
req.Header["Connection"] = []string{"Upgrade"}
req.Header["Sec-WebSocket-Key"] = []string{challengeKey}
req.Header["Sec-WebSocket-Version"] = []string{"13"}
if len(d.Subprotocols) > 0 {
req.Header["Sec-WebSocket-Protocol"] = []string{strings.Join(d.Subprotocols, ", ")}
}
for k, vs := range requestHeader {
switch {
case k == "Host":
if len(vs) > 0 {
req.Host = vs[0]
}
case k == "Upgrade" ||
k == "Connection" ||
k == "Sec-Websocket-Key" ||
k == "Sec-Websocket-Version" ||
//#nosec G101 (CWE-798): Potential HTTP request smuggling via parameter pollution
k == "Sec-Websocket-Extensions" ||
(k == "Sec-Websocket-Protocol" && len(d.Subprotocols) > 0):
return nil, nil, errors.New("websocket: duplicate header not allowed: " + k)
case k == "Sec-Websocket-Protocol":
req.Header["Sec-WebSocket-Protocol"] = vs
default:
req.Header[k] = vs
}
}
if d.EnableCompression {
req.Header["Sec-WebSocket-Extensions"] = []string{"permessage-deflate; server_no_context_takeover; client_no_context_takeover"}
}
if d.HandshakeTimeout != 0 {
var cancel func()
ctx, cancel = context.WithTimeout(ctx, d.HandshakeTimeout)
defer cancel()
}
// Get network dial function.
var netDial func(network, add string) (net.Conn, error)
switch u.Scheme {
case "http":
if d.NetDialContext != nil {
netDial = func(network, addr string) (net.Conn, error) {
return d.NetDialContext(ctx, network, addr)
}
} else if d.NetDial != nil {
netDial = d.NetDial
}
case "https":
if d.NetDialTLSContext != nil {
netDial = func(network, addr string) (net.Conn, error) {
return d.NetDialTLSContext(ctx, network, addr)
}
} else if d.NetDialContext != nil {
netDial = func(network, addr string) (net.Conn, error) {
return d.NetDialContext(ctx, network, addr)
}
} else if d.NetDial != nil {
netDial = d.NetDial
}
default:
return nil, nil, errMalformedURL
}
if netDial == nil {
netDialer := &net.Dialer{}
netDial = func(network, addr string) (net.Conn, error) {
return netDialer.DialContext(ctx, network, addr)
}
}
// If needed, wrap the dial function to set the connection deadline.
if deadline, ok := ctx.Deadline(); ok {
forwardDial := netDial
netDial = func(network, addr string) (net.Conn, error) {
c, err := forwardDial(network, addr)
if err != nil {
return nil, err
}
err = c.SetDeadline(deadline)
if err != nil {
if err := c.Close(); err != nil {
log.Printf("websocket: failed to close network connection: %v", err)
}
return nil, err
}
return c, nil
}
}
// If needed, wrap the dial function to connect through a proxy.
if d.Proxy != nil {
proxyURL, err := d.Proxy(req)
if err != nil {
return nil, nil, err
}
if proxyURL != nil {
dialer, err := proxy.FromURL(proxyURL, netDialerFunc(netDial))
if err != nil {
return nil, nil, err
}
netDial = dialer.Dial
}
}
hostPort, hostNoPort := hostPortNoPort(u)
trace := httptrace.ContextClientTrace(ctx)
if trace != nil && trace.GetConn != nil {
trace.GetConn(hostPort)
}
netConn, err := netDial("tcp", hostPort)
if err != nil {
return nil, nil, err
}
if trace != nil && trace.GotConn != nil {
trace.GotConn(httptrace.GotConnInfo{
Conn: netConn,
})
}
defer func() {
if netConn != nil {
if err := netConn.Close(); err != nil {
log.Printf("websocket: failed to close network connection: %v", err)
}
}
}()
if u.Scheme == "https" && d.NetDialTLSContext == nil {
// If NetDialTLSContext is set, assume that the TLS handshake has already been done
cfg := cloneTLSConfig(d.TLSClientConfig)
if cfg.ServerName == "" {
cfg.ServerName = hostNoPort
}
tlsConn := tls.Client(netConn, cfg)
netConn = tlsConn
if trace != nil && trace.TLSHandshakeStart != nil {
trace.TLSHandshakeStart()
}
err := doHandshake(ctx, tlsConn, cfg)
if trace != nil && trace.TLSHandshakeDone != nil {
trace.TLSHandshakeDone(tlsConn.ConnectionState(), err)
}
if err != nil {
return nil, nil, err
}
}
conn := newConn(netConn, false, d.ReadBufferSize, d.WriteBufferSize, d.WriteBufferPool, nil, nil)
if err := req.Write(netConn); err != nil {
return nil, nil, err
}
if trace != nil && trace.GotFirstResponseByte != nil {
if peek, err := conn.br.Peek(1); err == nil && len(peek) == 1 {
trace.GotFirstResponseByte()
}
}
resp, err := http.ReadResponse(conn.br, req)
if err != nil {
if d.TLSClientConfig != nil {
for _, proto := range d.TLSClientConfig.NextProtos {
if proto != "http/1.1" {
return nil, nil, fmt.Errorf(
"websocket: protocol %q was given but is not supported;"+
"sharing tls.Config with net/http Transport can cause this error: %w",
proto, err,
)
}
}
}
return nil, nil, err
}
if d.Jar != nil {
if rc := resp.Cookies(); len(rc) > 0 {
d.Jar.SetCookies(u, rc)
}
}
if resp.StatusCode != 101 ||
!tokenListContainsValue(resp.Header, "Upgrade", "websocket") ||
!tokenListContainsValue(resp.Header, "Connection", "upgrade") ||
resp.Header.Get("Sec-Websocket-Accept") != computeAcceptKey(challengeKey) {
// Before closing the network connection on return from this
// function, slurp up some of the response to aid application
// debugging.
buf := make([]byte, 1024)
n, _ := io.ReadFull(resp.Body, buf)
resp.Body = io.NopCloser(bytes.NewReader(buf[:n]))
return nil, resp, ErrBadHandshake
}
for _, ext := range parseExtensions(resp.Header) {
if ext[""] != "permessage-deflate" {
continue
}
_, snct := ext["server_no_context_takeover"]
_, cnct := ext["client_no_context_takeover"]
if !snct || !cnct {
return nil, resp, errInvalidCompression
}
conn.newCompressionWriter = compressNoContextTakeover
conn.newDecompressionReader = decompressNoContextTakeover
break
}
resp.Body = io.NopCloser(bytes.NewReader([]byte{}))
conn.subprotocol = resp.Header.Get("Sec-Websocket-Protocol")
if err := netConn.SetDeadline(time.Time{}); err != nil {
return nil, nil, err
}
netConn = nil // to avoid close in defer.
return conn, resp, nil
}
func cloneTLSConfig(cfg *tls.Config) *tls.Config {
if cfg == nil {
return &tls.Config{MinVersion: tls.VersionTLS12}
}
return cfg.Clone()
}

View File

@ -1,153 +0,0 @@
// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"compress/flate"
"errors"
"io"
"log"
"strings"
"sync"
)
const (
minCompressionLevel = -2 // flate.HuffmanOnly not defined in Go < 1.6
maxCompressionLevel = flate.BestCompression
defaultCompressionLevel = 1
)
var (
flateWriterPools [maxCompressionLevel - minCompressionLevel + 1]sync.Pool
flateReaderPool = sync.Pool{New: func() interface{} {
return flate.NewReader(nil)
}}
)
func decompressNoContextTakeover(r io.Reader) io.ReadCloser {
const tail =
// Add four bytes as specified in RFC
"\x00\x00\xff\xff" +
// Add final block to squelch unexpected EOF error from flate reader.
"\x01\x00\x00\xff\xff"
fr, _ := flateReaderPool.Get().(io.ReadCloser)
if err := fr.(flate.Resetter).Reset(io.MultiReader(r, strings.NewReader(tail)), nil); err != nil {
panic(err)
}
return &flateReadWrapper{fr}
}
func isValidCompressionLevel(level int) bool {
return minCompressionLevel <= level && level <= maxCompressionLevel
}
func compressNoContextTakeover(w io.WriteCloser, level int) io.WriteCloser {
p := &flateWriterPools[level-minCompressionLevel]
tw := &truncWriter{w: w}
fw, _ := p.Get().(*flate.Writer)
if fw == nil {
fw, _ = flate.NewWriter(tw, level)
} else {
fw.Reset(tw)
}
return &flateWriteWrapper{fw: fw, tw: tw, p: p}
}
// truncWriter is an io.Writer that writes all but the last four bytes of the
// stream to another io.Writer.
type truncWriter struct {
w io.WriteCloser
n int
p [4]byte
}
func (w *truncWriter) Write(p []byte) (int, error) {
n := 0
// fill buffer first for simplicity.
if w.n < len(w.p) {
n = copy(w.p[w.n:], p)
p = p[n:]
w.n += n
if len(p) == 0 {
return n, nil
}
}
m := len(p)
if m > len(w.p) {
m = len(w.p)
}
if nn, err := w.w.Write(w.p[:m]); err != nil {
return n + nn, err
}
copy(w.p[:], w.p[m:])
copy(w.p[len(w.p)-m:], p[len(p)-m:])
nn, err := w.w.Write(p[:len(p)-m])
return n + nn, err
}
type flateWriteWrapper struct {
fw *flate.Writer
tw *truncWriter
p *sync.Pool
}
func (w *flateWriteWrapper) Write(p []byte) (int, error) {
if w.fw == nil {
return 0, errWriteClosed
}
return w.fw.Write(p)
}
func (w *flateWriteWrapper) Close() error {
if w.fw == nil {
return errWriteClosed
}
err1 := w.fw.Flush()
w.p.Put(w.fw)
w.fw = nil
if w.tw.p != [4]byte{0, 0, 0xff, 0xff} {
return errors.New("websocket: internal error, unexpected bytes at end of flate stream")
}
err2 := w.tw.w.Close()
if err1 != nil {
return err1
}
return err2
}
type flateReadWrapper struct {
fr io.ReadCloser
}
func (r *flateReadWrapper) Read(p []byte) (int, error) {
if r.fr == nil {
return 0, io.ErrClosedPipe
}
n, err := r.fr.Read(p)
if err == io.EOF {
// Preemptively place the reader back in the pool. This helps with
// scenarios where the application does not call NextReader() soon after
// this final read.
if err := r.Close(); err != nil {
log.Printf("websocket: flateReadWrapper.Close() returned error: %v", err)
}
}
return n, err
}
func (r *flateReadWrapper) Close() error {
if r.fr == nil {
return io.ErrClosedPipe
}
err := r.fr.Close()
flateReaderPool.Put(r.fr)
r.fr = nil
return err
}

File diff suppressed because it is too large Load Diff

View File

@ -1,227 +0,0 @@
// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package websocket implements the WebSocket protocol defined in RFC 6455.
//
// Overview
//
// The Conn type represents a WebSocket connection. A server application calls
// the Upgrader.Upgrade method from an HTTP request handler to get a *Conn:
//
// var upgrader = websocket.Upgrader{
// ReadBufferSize: 1024,
// WriteBufferSize: 1024,
// }
//
// func handler(w http.ResponseWriter, r *http.Request) {
// conn, err := upgrader.Upgrade(w, r, nil)
// if err != nil {
// log.Println(err)
// return
// }
// ... Use conn to send and receive messages.
// }
//
// Call the connection's WriteMessage and ReadMessage methods to send and
// receive messages as a slice of bytes. This snippet of code shows how to echo
// messages using these methods:
//
// for {
// messageType, p, err := conn.ReadMessage()
// if err != nil {
// log.Println(err)
// return
// }
// if err := conn.WriteMessage(messageType, p); err != nil {
// log.Println(err)
// return
// }
// }
//
// In above snippet of code, p is a []byte and messageType is an int with value
// websocket.BinaryMessage or websocket.TextMessage.
//
// An application can also send and receive messages using the io.WriteCloser
// and io.Reader interfaces. To send a message, call the connection NextWriter
// method to get an io.WriteCloser, write the message to the writer and close
// the writer when done. To receive a message, call the connection NextReader
// method to get an io.Reader and read until io.EOF is returned. This snippet
// shows how to echo messages using the NextWriter and NextReader methods:
//
// for {
// messageType, r, err := conn.NextReader()
// if err != nil {
// return
// }
// w, err := conn.NextWriter(messageType)
// if err != nil {
// return err
// }
// if _, err := io.Copy(w, r); err != nil {
// return err
// }
// if err := w.Close(); err != nil {
// return err
// }
// }
//
// Data Messages
//
// The WebSocket protocol distinguishes between text and binary data messages.
// Text messages are interpreted as UTF-8 encoded text. The interpretation of
// binary messages is left to the application.
//
// This package uses the TextMessage and BinaryMessage integer constants to
// identify the two data message types. The ReadMessage and NextReader methods
// return the type of the received message. The messageType argument to the
// WriteMessage and NextWriter methods specifies the type of a sent message.
//
// It is the application's responsibility to ensure that text messages are
// valid UTF-8 encoded text.
//
// Control Messages
//
// The WebSocket protocol defines three types of control messages: close, ping
// and pong. Call the connection WriteControl, WriteMessage or NextWriter
// methods to send a control message to the peer.
//
// Connections handle received close messages by calling the handler function
// set with the SetCloseHandler method and by returning a *CloseError from the
// NextReader, ReadMessage or the message Read method. The default close
// handler sends a close message to the peer.
//
// Connections handle received ping messages by calling the handler function
// set with the SetPingHandler method. The default ping handler sends a pong
// message to the peer.
//
// Connections handle received pong messages by calling the handler function
// set with the SetPongHandler method. The default pong handler does nothing.
// If an application sends ping messages, then the application should set a
// pong handler to receive the corresponding pong.
//
// The control message handler functions are called from the NextReader,
// ReadMessage and message reader Read methods. The default close and ping
// handlers can block these methods for a short time when the handler writes to
// the connection.
//
// The application must read the connection to process close, ping and pong
// messages sent from the peer. If the application is not otherwise interested
// in messages from the peer, then the application should start a goroutine to
// read and discard messages from the peer. A simple example is:
//
// func readLoop(c *websocket.Conn) {
// for {
// if _, _, err := c.NextReader(); err != nil {
// c.Close()
// break
// }
// }
// }
//
// Concurrency
//
// Connections support one concurrent reader and one concurrent writer.
//
// Applications are responsible for ensuring that no more than one goroutine
// calls the write methods (NextWriter, SetWriteDeadline, WriteMessage,
// WriteJSON, EnableWriteCompression, SetCompressionLevel) concurrently and
// that no more than one goroutine calls the read methods (NextReader,
// SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, SetPingHandler)
// concurrently.
//
// The Close and WriteControl methods can be called concurrently with all other
// methods.
//
// Origin Considerations
//
// Web browsers allow Javascript applications to open a WebSocket connection to
// any host. It's up to the server to enforce an origin policy using the Origin
// request header sent by the browser.
//
// The Upgrader calls the function specified in the CheckOrigin field to check
// the origin. If the CheckOrigin function returns false, then the Upgrade
// method fails the WebSocket handshake with HTTP status 403.
//
// If the CheckOrigin field is nil, then the Upgrader uses a safe default: fail
// the handshake if the Origin request header is present and the Origin host is
// not equal to the Host request header.
//
// The deprecated package-level Upgrade function does not perform origin
// checking. The application is responsible for checking the Origin header
// before calling the Upgrade function.
//
// Buffers
//
// Connections buffer network input and output to reduce the number
// of system calls when reading or writing messages.
//
// Write buffers are also used for constructing WebSocket frames. See RFC 6455,
// Section 5 for a discussion of message framing. A WebSocket frame header is
// written to the network each time a write buffer is flushed to the network.
// Decreasing the size of the write buffer can increase the amount of framing
// overhead on the connection.
//
// The buffer sizes in bytes are specified by the ReadBufferSize and
// WriteBufferSize fields in the Dialer and Upgrader. The Dialer uses a default
// size of 4096 when a buffer size field is set to zero. The Upgrader reuses
// buffers created by the HTTP server when a buffer size field is set to zero.
// The HTTP server buffers have a size of 4096 at the time of this writing.
//
// The buffer sizes do not limit the size of a message that can be read or
// written by a connection.
//
// Buffers are held for the lifetime of the connection by default. If the
// Dialer or Upgrader WriteBufferPool field is set, then a connection holds the
// write buffer only when writing a message.
//
// Applications should tune the buffer sizes to balance memory use and
// performance. Increasing the buffer size uses more memory, but can reduce the
// number of system calls to read or write the network. In the case of writing,
// increasing the buffer size can reduce the number of frame headers written to
// the network.
//
// Some guidelines for setting buffer parameters are:
//
// Limit the buffer sizes to the maximum expected message size. Buffers larger
// than the largest message do not provide any benefit.
//
// Depending on the distribution of message sizes, setting the buffer size to
// a value less than the maximum expected message size can greatly reduce memory
// use with a small impact on performance. Here's an example: If 99% of the
// messages are smaller than 256 bytes and the maximum message size is 512
// bytes, then a buffer size of 256 bytes will result in 1.01 more system calls
// than a buffer size of 512 bytes. The memory savings is 50%.
//
// A write buffer pool is useful when the application has a modest number
// writes over a large number of connections. when buffers are pooled, a larger
// buffer size has a reduced impact on total memory use and has the benefit of
// reducing system calls and frame overhead.
//
// Compression EXPERIMENTAL
//
// Per message compression extensions (RFC 7692) are experimentally supported
// by this package in a limited capacity. Setting the EnableCompression option
// to true in Dialer or Upgrader will attempt to negotiate per message deflate
// support.
//
// var upgrader = websocket.Upgrader{
// EnableCompression: true,
// }
//
// If compression was successfully negotiated with the connection's peer, any
// message received in compressed form will be automatically decompressed.
// All Read methods will return uncompressed bytes.
//
// Per message compression of messages written to a connection can be enabled
// or disabled by calling the corresponding Conn method:
//
// conn.EnableWriteCompression(false)
//
// Currently this package does not support compression with "context takeover".
// This means that messages must be compressed and decompressed in isolation,
// without retaining sliding window or dictionary state across messages. For
// more details refer to RFC 7692.
//
// Use of compression is experimental and may result in decreased performance.
package websocket

View File

@ -1,42 +0,0 @@
// Copyright 2019 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"io"
"strings"
)
// JoinMessages concatenates received messages to create a single io.Reader.
// The string term is appended to each message. The returned reader does not
// support concurrent calls to the Read method.
func JoinMessages(c *Conn, term string) io.Reader {
return &joinReader{c: c, term: term}
}
type joinReader struct {
c *Conn
term string
r io.Reader
}
func (r *joinReader) Read(p []byte) (int, error) {
if r.r == nil {
var err error
_, r.r, err = r.c.NextReader()
if err != nil {
return 0, err
}
if r.term != "" {
r.r = io.MultiReader(r.r, strings.NewReader(r.term))
}
}
n, err := r.r.Read(p)
if err == io.EOF {
err = nil
r.r = nil
}
return n, err
}

View File

@ -1,60 +0,0 @@
// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"encoding/json"
"io"
)
// WriteJSON writes the JSON encoding of v as a message.
//
// Deprecated: Use c.WriteJSON instead.
func WriteJSON(c *Conn, v interface{}) error {
return c.WriteJSON(v)
}
// WriteJSON writes the JSON encoding of v as a message.
//
// See the documentation for encoding/json Marshal for details about the
// conversion of Go values to JSON.
func (c *Conn) WriteJSON(v interface{}) error {
w, err := c.NextWriter(TextMessage)
if err != nil {
return err
}
err1 := json.NewEncoder(w).Encode(v)
err2 := w.Close()
if err1 != nil {
return err1
}
return err2
}
// ReadJSON reads the next JSON-encoded message from the connection and stores
// it in the value pointed to by v.
//
// Deprecated: Use c.ReadJSON instead.
func ReadJSON(c *Conn, v interface{}) error {
return c.ReadJSON(v)
}
// ReadJSON reads the next JSON-encoded message from the connection and stores
// it in the value pointed to by v.
//
// See the documentation for the encoding/json Unmarshal function for details
// about the conversion of JSON to a Go value.
func (c *Conn) ReadJSON(v interface{}) error {
_, r, err := c.NextReader()
if err != nil {
return err
}
err = json.NewDecoder(r).Decode(v)
if err == io.EOF {
// One value is expected in the message.
err = io.ErrUnexpectedEOF
}
return err
}

View File

@ -1,59 +0,0 @@
// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of
// this source code is governed by a BSD-style license that can be found in the
// LICENSE file.
//go:build !appengine
// +build !appengine
package websocket
import "unsafe"
// #nosec G103 -- (CWE-242) Has been audited
const wordSize = int(unsafe.Sizeof(uintptr(0)))
func maskBytes(key [4]byte, pos int, b []byte) int {
// Mask one byte at a time for small buffers.
if len(b) < 2*wordSize {
for i := range b {
b[i] ^= key[pos&3]
pos++
}
return pos & 3
}
// Mask one byte at a time to word boundary.
//#nosec G103 -- (CWE-242) Has been audited
if n := int(uintptr(unsafe.Pointer(&b[0]))) % wordSize; n != 0 {
n = wordSize - n
for i := range b[:n] {
b[i] ^= key[pos&3]
pos++
}
b = b[n:]
}
// Create aligned word size key.
var k [wordSize]byte
for i := range k {
k[i] = key[(pos+i)&3]
}
//#nosec G103 -- (CWE-242) Has been audited
kw := *(*uintptr)(unsafe.Pointer(&k))
// Mask one word at a time.
n := (len(b) / wordSize) * wordSize
for i := 0; i < n; i += wordSize {
//#nosec G103 -- (CWE-242) Has been audited
*(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&b[0])) + uintptr(i))) ^= kw
}
// Mask one byte at a time for remaining bytes.
b = b[n:]
for i := range b {
b[i] ^= key[pos&3]
pos++
}
return pos & 3
}

View File

@ -1,16 +0,0 @@
// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of
// this source code is governed by a BSD-style license that can be found in the
// LICENSE file.
//go:build appengine
// +build appengine
package websocket
func maskBytes(key [4]byte, pos int, b []byte) int {
for i := range b {
b[i] ^= key[pos&3]
pos++
}
return pos & 3
}

View File

@ -1,102 +0,0 @@
// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"bytes"
"net"
"sync"
"time"
)
// PreparedMessage caches on the wire representations of a message payload.
// Use PreparedMessage to efficiently send a message payload to multiple
// connections. PreparedMessage is especially useful when compression is used
// because the CPU and memory expensive compression operation can be executed
// once for a given set of compression options.
type PreparedMessage struct {
messageType int
data []byte
mu sync.Mutex
frames map[prepareKey]*preparedFrame
}
// prepareKey defines a unique set of options to cache prepared frames in PreparedMessage.
type prepareKey struct {
isServer bool
compress bool
compressionLevel int
}
// preparedFrame contains data in wire representation.
type preparedFrame struct {
once sync.Once
data []byte
}
// NewPreparedMessage returns an initialized PreparedMessage. You can then send
// it to connection using WritePreparedMessage method. Valid wire
// representation will be calculated lazily only once for a set of current
// connection options.
func NewPreparedMessage(messageType int, data []byte) (*PreparedMessage, error) {
pm := &PreparedMessage{
messageType: messageType,
frames: make(map[prepareKey]*preparedFrame),
data: data,
}
// Prepare a plain server frame.
_, frameData, err := pm.frame(prepareKey{isServer: true, compress: false})
if err != nil {
return nil, err
}
// To protect against caller modifying the data argument, remember the data
// copied to the plain server frame.
pm.data = frameData[len(frameData)-len(data):]
return pm, nil
}
func (pm *PreparedMessage) frame(key prepareKey) (int, []byte, error) {
pm.mu.Lock()
frame, ok := pm.frames[key]
if !ok {
frame = &preparedFrame{}
pm.frames[key] = frame
}
pm.mu.Unlock()
var err error
frame.once.Do(func() {
// Prepare a frame using a 'fake' connection.
// TODO: Refactor code in conn.go to allow more direct construction of
// the frame.
mu := make(chan struct{}, 1)
mu <- struct{}{}
var nc prepareConn
c := &Conn{
conn: &nc,
mu: mu,
isServer: key.isServer,
compressionLevel: key.compressionLevel,
enableWriteCompression: true,
writeBuf: make([]byte, defaultWriteBufferSize+maxFrameHeaderSize),
}
if key.compress {
c.newCompressionWriter = compressNoContextTakeover
}
err = c.WriteMessage(pm.messageType, pm.data)
frame.data = nc.buf.Bytes()
})
return pm.messageType, frame.data, err
}
type prepareConn struct {
buf bytes.Buffer
net.Conn
}
func (pc *prepareConn) Write(p []byte) (int, error) { return pc.buf.Write(p) }
func (pc *prepareConn) SetWriteDeadline(t time.Time) error { return nil }

View File

@ -1,86 +0,0 @@
// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"bufio"
"encoding/base64"
"errors"
"log"
"net"
"net/http"
"net/url"
"strings"
"golang.org/x/net/proxy"
)
type netDialerFunc func(network, addr string) (net.Conn, error)
func (fn netDialerFunc) Dial(network, addr string) (net.Conn, error) {
return fn(network, addr)
}
func init() {
proxy.RegisterDialerType("http", func(proxyURL *url.URL, forwardDialer proxy.Dialer) (proxy.Dialer, error) {
return &httpProxyDialer{proxyURL: proxyURL, forwardDial: forwardDialer.Dial}, nil
})
}
type httpProxyDialer struct {
proxyURL *url.URL
forwardDial func(network, addr string) (net.Conn, error)
}
func (hpd *httpProxyDialer) Dial(network string, addr string) (net.Conn, error) {
hostPort, _ := hostPortNoPort(hpd.proxyURL)
conn, err := hpd.forwardDial(network, hostPort)
if err != nil {
return nil, err
}
connectHeader := make(http.Header)
if user := hpd.proxyURL.User; user != nil {
proxyUser := user.Username()
if proxyPassword, passwordSet := user.Password(); passwordSet {
credential := base64.StdEncoding.EncodeToString([]byte(proxyUser + ":" + proxyPassword))
connectHeader.Set("Proxy-Authorization", "Basic "+credential)
}
}
connectReq := &http.Request{
Method: http.MethodConnect,
URL: &url.URL{Opaque: addr},
Host: addr,
Header: connectHeader,
}
if err := connectReq.Write(conn); err != nil {
if err := conn.Close(); err != nil {
log.Printf("httpProxyDialer: failed to close connection: %v", err)
}
return nil, err
}
// Read response. It's OK to use and discard buffered reader here becaue
// the remote server does not speak until spoken to.
br := bufio.NewReader(conn)
resp, err := http.ReadResponse(br, connectReq)
if err != nil {
if err := conn.Close(); err != nil {
log.Printf("httpProxyDialer: failed to close connection: %v", err)
}
return nil, err
}
if resp.StatusCode != 200 {
if err := conn.Close(); err != nil {
log.Printf("httpProxyDialer: failed to close connection: %v", err)
}
f := strings.SplitN(resp.Status, " ", 2)
return nil, errors.New(f[1])
}
return conn, nil
}

View File

@ -1,389 +0,0 @@
// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"bufio"
"errors"
"io"
"log"
"net/http"
"net/url"
"strings"
"time"
)
// HandshakeError describes an error with the handshake from the peer.
type HandshakeError struct {
message string
}
func (e HandshakeError) Error() string { return e.message }
// Upgrader specifies parameters for upgrading an HTTP connection to a
// WebSocket connection.
//
// It is safe to call Upgrader's methods concurrently.
type Upgrader struct {
// HandshakeTimeout specifies the duration for the handshake to complete.
HandshakeTimeout time.Duration
// ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer
// size is zero, then buffers allocated by the HTTP server are used. The
// I/O buffer sizes do not limit the size of the messages that can be sent
// or received.
ReadBufferSize, WriteBufferSize int
// WriteBufferPool is a pool of buffers for write operations. If the value
// is not set, then write buffers are allocated to the connection for the
// lifetime of the connection.
//
// A pool is most useful when the application has a modest volume of writes
// across a large number of connections.
//
// Applications should use a single pool for each unique value of
// WriteBufferSize.
WriteBufferPool BufferPool
// Subprotocols specifies the server's supported protocols in order of
// preference. If this field is not nil, then the Upgrade method negotiates a
// subprotocol by selecting the first match in this list with a protocol
// requested by the client. If there's no match, then no protocol is
// negotiated (the Sec-Websocket-Protocol header is not included in the
// handshake response).
Subprotocols []string
// Error specifies the function for generating HTTP error responses. If Error
// is nil, then http.Error is used to generate the HTTP response.
Error func(w http.ResponseWriter, r *http.Request, status int, reason error)
// CheckOrigin returns true if the request Origin header is acceptable. If
// CheckOrigin is nil, then a safe default is used: return false if the
// Origin request header is present and the origin host is not equal to
// request Host header.
//
// A CheckOrigin function should carefully validate the request origin to
// prevent cross-site request forgery.
CheckOrigin func(r *http.Request) bool
// EnableCompression specify if the server should attempt to negotiate per
// message compression (RFC 7692). Setting this value to true does not
// guarantee that compression will be supported. Currently only "no context
// takeover" modes are supported.
EnableCompression bool
}
func (u *Upgrader) returnError(w http.ResponseWriter, r *http.Request, status int, reason string) (*Conn, error) {
err := HandshakeError{reason}
if u.Error != nil {
u.Error(w, r, status, err)
} else {
w.Header().Set("Sec-Websocket-Version", "13")
http.Error(w, http.StatusText(status), status)
}
return nil, err
}
// checkSameOrigin returns true if the origin is not set or is equal to the request host.
func checkSameOrigin(r *http.Request) bool {
origin := r.Header["Origin"]
if len(origin) == 0 {
return true
}
u, err := url.Parse(origin[0])
if err != nil {
return false
}
return equalASCIIFold(u.Host, r.Host)
}
func (u *Upgrader) selectSubprotocol(r *http.Request, responseHeader http.Header) string {
if u.Subprotocols != nil {
clientProtocols := Subprotocols(r)
for _, serverProtocol := range u.Subprotocols {
for _, clientProtocol := range clientProtocols {
if clientProtocol == serverProtocol {
return clientProtocol
}
}
}
} else if responseHeader != nil {
return responseHeader.Get("Sec-Websocket-Protocol")
}
return ""
}
// Upgrade upgrades the HTTP server connection to the WebSocket protocol.
//
// The responseHeader is included in the response to the client's upgrade
// request. Use the responseHeader to specify cookies (Set-Cookie). To specify
// subprotocols supported by the server, set Upgrader.Subprotocols directly.
//
// If the upgrade fails, then Upgrade replies to the client with an HTTP error
// response.
func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) {
const badHandshake = "websocket: the client is not using the websocket protocol: "
if !tokenListContainsValue(r.Header, "Connection", "upgrade") {
return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'upgrade' token not found in 'Connection' header")
}
if !tokenListContainsValue(r.Header, "Upgrade", "websocket") {
return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'websocket' token not found in 'Upgrade' header")
}
if r.Method != http.MethodGet {
return u.returnError(w, r, http.StatusMethodNotAllowed, badHandshake+"request method is not GET")
}
if !tokenListContainsValue(r.Header, "Sec-Websocket-Version", "13") {
return u.returnError(w, r, http.StatusBadRequest, "websocket: unsupported version: 13 not found in 'Sec-Websocket-Version' header")
}
if _, ok := responseHeader["Sec-Websocket-Extensions"]; ok {
return u.returnError(w, r, http.StatusInternalServerError, "websocket: application specific 'Sec-WebSocket-Extensions' headers are unsupported")
}
checkOrigin := u.CheckOrigin
if checkOrigin == nil {
checkOrigin = checkSameOrigin
}
if !checkOrigin(r) {
return u.returnError(w, r, http.StatusForbidden, "websocket: request origin not allowed by Upgrader.CheckOrigin")
}
challengeKey := r.Header.Get("Sec-Websocket-Key")
if !isValidChallengeKey(challengeKey) {
return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: 'Sec-WebSocket-Key' header must be Base64 encoded value of 16-byte in length")
}
subprotocol := u.selectSubprotocol(r, responseHeader)
// Negotiate PMCE
var compress bool
if u.EnableCompression {
for _, ext := range parseExtensions(r.Header) {
if ext[""] != "permessage-deflate" {
continue
}
compress = true
break
}
}
h, ok := w.(http.Hijacker)
if !ok {
return u.returnError(w, r, http.StatusInternalServerError, "websocket: response does not implement http.Hijacker")
}
var brw *bufio.ReadWriter
netConn, brw, err := h.Hijack()
if err != nil {
return u.returnError(w, r, http.StatusInternalServerError, err.Error())
}
if brw.Reader.Buffered() > 0 {
if err := netConn.Close(); err != nil {
log.Printf("websocket: failed to close network connection: %v", err)
}
return nil, errors.New("websocket: client sent data before handshake is complete")
}
var br *bufio.Reader
if u.ReadBufferSize == 0 && bufioReaderSize(netConn, brw.Reader) > 256 {
// Reuse hijacked buffered reader as connection reader.
br = brw.Reader
}
buf := bufioWriterBuffer(netConn, brw.Writer)
var writeBuf []byte
if u.WriteBufferPool == nil && u.WriteBufferSize == 0 && len(buf) >= maxFrameHeaderSize+256 {
// Reuse hijacked write buffer as connection buffer.
writeBuf = buf
}
c := newConn(netConn, true, u.ReadBufferSize, u.WriteBufferSize, u.WriteBufferPool, br, writeBuf)
c.subprotocol = subprotocol
if compress {
c.newCompressionWriter = compressNoContextTakeover
c.newDecompressionReader = decompressNoContextTakeover
}
// Use larger of hijacked buffer and connection write buffer for header.
p := buf
if len(c.writeBuf) > len(p) {
p = c.writeBuf
}
p = p[:0]
p = append(p, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "...)
p = append(p, computeAcceptKey(challengeKey)...)
p = append(p, "\r\n"...)
if c.subprotocol != "" {
p = append(p, "Sec-WebSocket-Protocol: "...)
p = append(p, c.subprotocol...)
p = append(p, "\r\n"...)
}
if compress {
p = append(p, "Sec-WebSocket-Extensions: permessage-deflate; server_no_context_takeover; client_no_context_takeover\r\n"...)
}
for k, vs := range responseHeader {
if k == "Sec-Websocket-Protocol" {
continue
}
for _, v := range vs {
p = append(p, k...)
p = append(p, ": "...)
for i := 0; i < len(v); i++ {
b := v[i]
if b <= 31 {
// prevent response splitting.
b = ' '
}
p = append(p, b)
}
p = append(p, "\r\n"...)
}
}
p = append(p, "\r\n"...)
// Clear deadlines set by HTTP server.
if err := netConn.SetDeadline(time.Time{}); err != nil {
if err := netConn.Close(); err != nil {
log.Printf("websocket: failed to close network connection: %v", err)
}
return nil, err
}
if u.HandshakeTimeout > 0 {
if err := netConn.SetWriteDeadline(time.Now().Add(u.HandshakeTimeout)); err != nil {
if err := netConn.Close(); err != nil {
log.Printf("websocket: failed to close network connection: %v", err)
}
return nil, err
}
}
if _, err = netConn.Write(p); err != nil {
if err := netConn.Close(); err != nil {
log.Printf("websocket: failed to close network connection: %v", err)
}
return nil, err
}
if u.HandshakeTimeout > 0 {
if err := netConn.SetWriteDeadline(time.Time{}); err != nil {
if err := netConn.Close(); err != nil {
log.Printf("websocket: failed to close network connection: %v", err)
}
return nil, err
}
}
return c, nil
}
// Upgrade upgrades the HTTP server connection to the WebSocket protocol.
//
// Deprecated: Use websocket.Upgrader instead.
//
// Upgrade does not perform origin checking. The application is responsible for
// checking the Origin header before calling Upgrade. An example implementation
// of the same origin policy check is:
//
// if req.Header.Get("Origin") != "http://"+req.Host {
// http.Error(w, "Origin not allowed", http.StatusForbidden)
// return
// }
//
// If the endpoint supports subprotocols, then the application is responsible
// for negotiating the protocol used on the connection. Use the Subprotocols()
// function to get the subprotocols requested by the client. Use the
// Sec-Websocket-Protocol response header to specify the subprotocol selected
// by the application.
//
// The responseHeader is included in the response to the client's upgrade
// request. Use the responseHeader to specify cookies (Set-Cookie) and the
// negotiated subprotocol (Sec-Websocket-Protocol).
//
// The connection buffers IO to the underlying network connection. The
// readBufSize and writeBufSize parameters specify the size of the buffers to
// use. Messages can be larger than the buffers.
//
// If the request is not a valid WebSocket handshake, then Upgrade returns an
// error of type HandshakeError. Applications should handle this error by
// replying to the client with an HTTP error response.
func Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header, readBufSize, writeBufSize int) (*Conn, error) {
u := Upgrader{ReadBufferSize: readBufSize, WriteBufferSize: writeBufSize}
u.Error = func(w http.ResponseWriter, r *http.Request, status int, reason error) {
// don't return errors to maintain backwards compatibility
}
u.CheckOrigin = func(r *http.Request) bool {
// allow all connections by default
return true
}
return u.Upgrade(w, r, responseHeader)
}
// Subprotocols returns the subprotocols requested by the client in the
// Sec-Websocket-Protocol header.
func Subprotocols(r *http.Request) []string {
h := strings.TrimSpace(r.Header.Get("Sec-Websocket-Protocol"))
if h == "" {
return nil
}
protocols := strings.Split(h, ",")
for i := range protocols {
protocols[i] = strings.TrimSpace(protocols[i])
}
return protocols
}
// IsWebSocketUpgrade returns true if the client requested upgrade to the
// WebSocket protocol.
func IsWebSocketUpgrade(r *http.Request) bool {
return tokenListContainsValue(r.Header, "Connection", "upgrade") &&
tokenListContainsValue(r.Header, "Upgrade", "websocket")
}
// bufioReaderSize size returns the size of a bufio.Reader.
func bufioReaderSize(originalReader io.Reader, br *bufio.Reader) int {
// This code assumes that peek on a reset reader returns
// bufio.Reader.buf[:0].
// TODO: Use bufio.Reader.Size() after Go 1.10
br.Reset(originalReader)
if p, err := br.Peek(0); err == nil {
return cap(p)
}
return 0
}
// writeHook is an io.Writer that records the last slice passed to it vio
// io.Writer.Write.
type writeHook struct {
p []byte
}
func (wh *writeHook) Write(p []byte) (int, error) {
wh.p = p
return len(p), nil
}
// bufioWriterBuffer grabs the buffer from a bufio.Writer.
func bufioWriterBuffer(originalWriter io.Writer, bw *bufio.Writer) []byte {
// This code assumes that bufio.Writer.buf[:1] is passed to the
// bufio.Writer's underlying writer.
var wh writeHook
bw.Reset(&wh)
if err := bw.WriteByte(0); err != nil {
panic(err)
}
if err := bw.Flush(); err != nil {
log.Printf("websocket: bufioWriterBuffer: Flush: %v", err)
}
bw.Reset(originalWriter)
return wh.p[:cap(wh.p)]
}

View File

@ -1,18 +0,0 @@
package websocket
import (
"context"
"crypto/tls"
)
func doHandshake(ctx context.Context, tlsConn *tls.Conn, cfg *tls.Config) error {
if err := tlsConn.HandshakeContext(ctx); err != nil {
return err
}
if !cfg.InsecureSkipVerify {
if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
return err
}
}
return nil
}

View File

@ -1,298 +0,0 @@
// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package websocket
import (
"crypto/rand"
"crypto/sha1" //#nosec G505 -- (CWE-327) https://datatracker.ietf.org/doc/html/rfc6455#page-54
"encoding/base64"
"io"
"net/http"
"strings"
"unicode/utf8"
)
var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11")
func computeAcceptKey(challengeKey string) string {
h := sha1.New() //#nosec G401 -- (CWE-326) https://datatracker.ietf.org/doc/html/rfc6455#page-54
h.Write([]byte(challengeKey))
h.Write(keyGUID)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func generateChallengeKey() (string, error) {
p := make([]byte, 16)
if _, err := io.ReadFull(rand.Reader, p); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(p), nil
}
// Token octets per RFC 2616.
var isTokenOctet = [256]bool{
'!': true,
'#': true,
'$': true,
'%': true,
'&': true,
'\'': true,
'*': true,
'+': true,
'-': true,
'.': true,
'0': true,
'1': true,
'2': true,
'3': true,
'4': true,
'5': true,
'6': true,
'7': true,
'8': true,
'9': true,
'A': true,
'B': true,
'C': true,
'D': true,
'E': true,
'F': true,
'G': true,
'H': true,
'I': true,
'J': true,
'K': true,
'L': true,
'M': true,
'N': true,
'O': true,
'P': true,
'Q': true,
'R': true,
'S': true,
'T': true,
'U': true,
'W': true,
'V': true,
'X': true,
'Y': true,
'Z': true,
'^': true,
'_': true,
'`': true,
'a': true,
'b': true,
'c': true,
'd': true,
'e': true,
'f': true,
'g': true,
'h': true,
'i': true,
'j': true,
'k': true,
'l': true,
'm': true,
'n': true,
'o': true,
'p': true,
'q': true,
'r': true,
's': true,
't': true,
'u': true,
'v': true,
'w': true,
'x': true,
'y': true,
'z': true,
'|': true,
'~': true,
}
// skipSpace returns a slice of the string s with all leading RFC 2616 linear
// whitespace removed.
func skipSpace(s string) (rest string) {
i := 0
for ; i < len(s); i++ {
if b := s[i]; b != ' ' && b != '\t' {
break
}
}
return s[i:]
}
// nextToken returns the leading RFC 2616 token of s and the string following
// the token.
func nextToken(s string) (token, rest string) {
i := 0
for ; i < len(s); i++ {
if !isTokenOctet[s[i]] {
break
}
}
return s[:i], s[i:]
}
// nextTokenOrQuoted returns the leading token or quoted string per RFC 2616
// and the string following the token or quoted string.
func nextTokenOrQuoted(s string) (value string, rest string) {
if !strings.HasPrefix(s, "\"") {
return nextToken(s)
}
s = s[1:]
for i := 0; i < len(s); i++ {
switch s[i] {
case '"':
return s[:i], s[i+1:]
case '\\':
p := make([]byte, len(s)-1)
j := copy(p, s[:i])
escape := true
for i = i + 1; i < len(s); i++ {
b := s[i]
switch {
case escape:
escape = false
p[j] = b
j++
case b == '\\':
escape = true
case b == '"':
return string(p[:j]), s[i+1:]
default:
p[j] = b
j++
}
}
return "", ""
}
}
return "", ""
}
// equalASCIIFold returns true if s is equal to t with ASCII case folding as
// defined in RFC 4790.
func equalASCIIFold(s, t string) bool {
for s != "" && t != "" {
sr, size := utf8.DecodeRuneInString(s)
s = s[size:]
tr, size := utf8.DecodeRuneInString(t)
t = t[size:]
if sr == tr {
continue
}
if 'A' <= sr && sr <= 'Z' {
sr = sr + 'a' - 'A'
}
if 'A' <= tr && tr <= 'Z' {
tr = tr + 'a' - 'A'
}
if sr != tr {
return false
}
}
return s == t
}
// tokenListContainsValue returns true if the 1#token header with the given
// name contains a token equal to value with ASCII case folding.
func tokenListContainsValue(header http.Header, name string, value string) bool {
headers:
for _, s := range header[name] {
for {
var t string
t, s = nextToken(skipSpace(s))
if t == "" {
continue headers
}
s = skipSpace(s)
if s != "" && s[0] != ',' {
continue headers
}
if equalASCIIFold(t, value) {
return true
}
if s == "" {
continue headers
}
s = s[1:]
}
}
return false
}
// parseExtensions parses WebSocket extensions from a header.
func parseExtensions(header http.Header) []map[string]string {
// From RFC 6455:
//
// Sec-WebSocket-Extensions = extension-list
// extension-list = 1#extension
// extension = extension-token *( ";" extension-param )
// extension-token = registered-token
// registered-token = token
// extension-param = token [ "=" (token | quoted-string) ]
// ;When using the quoted-string syntax variant, the value
// ;after quoted-string unescaping MUST conform to the
// ;'token' ABNF.
var result []map[string]string
headers:
for _, s := range header["Sec-Websocket-Extensions"] {
for {
var t string
t, s = nextToken(skipSpace(s))
if t == "" {
continue headers
}
ext := map[string]string{"": t}
for {
s = skipSpace(s)
if !strings.HasPrefix(s, ";") {
break
}
var k string
k, s = nextToken(skipSpace(s[1:]))
if k == "" {
continue headers
}
s = skipSpace(s)
var v string
if strings.HasPrefix(s, "=") {
v, s = nextTokenOrQuoted(skipSpace(s[1:]))
s = skipSpace(s)
}
if s != "" && s[0] != ',' && s[0] != ';' {
continue headers
}
ext[k] = v
}
if s != "" && s[0] != ',' {
continue headers
}
result = append(result, ext)
if s == "" {
continue headers
}
s = s[1:]
}
}
return result
}
// isValidChallengeKey checks if the argument meets RFC6455 specification.
func isValidChallengeKey(s string) bool {
// From RFC6455:
//
// A |Sec-WebSocket-Key| header field with a base64-encoded (see
// Section 4 of [RFC4648]) value that, when decoded, is 16 bytes in
// length.
if s == "" {
return false
}
decoded, err := base64.StdEncoding.DecodeString(s)
return err == nil && len(decoded) == 16
}

View File

@ -1 +0,0 @@
submodules/

View File

@ -1,3 +0,0 @@
[submodule "submodule/okex"]
path = submodule/okex
url = baidu:/root/repos/go/okexV5Api

View File

@ -1,13 +0,0 @@
## Init
mkdir submodules && cd submodules
git submodule add baidu:/root/repos/go/okexV5Api okex
cd ../
git pull
git submodule init
git submodule update --force --recursive --init --remote
go mod tidy
go mod vendor

View File

@ -1,481 +0,0 @@
package core
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"math/rand"
"os"
"strconv"
"strings"
"time"
// "v5sdk_go/rest"
simple "github.com/bitly/go-simplejson"
"github.com/go-redis/redis"
"github.com/phyer/texus/utils"
)
type Candle struct {
Id string `json:"_id"`
core *Core
InstId string
Period string
Data []interface{}
From string
Timestamp time.Time
Open float64
High float64
Low float64
Close float64
VolCcy float64
Confirm bool
}
type MaX struct {
Core *Core
InstId string
Period string
KeyName string
Count int
Ts int64
Value float64
Data []interface{}
From string
}
type MatchCheck struct {
Minutes int64
Matched bool
}
func (cd *Candle) Filter(cr *Core) bool {
myFocusList := cr.Cfg.Config.Get("focusList").MustArray()
founded := false
for _, v := range myFocusList {
if v.(string) == cd.InstId {
founded = true
break
}
}
return founded
}
func (mc *MatchCheck) SetMatched(value bool) {
mc.Matched = value
}
func (core *Core) GetCandlesWithRest(instId string, kidx int, dura time.Duration, maxCandles int) error {
ary := []string{}
wsary := core.Cfg.CandleDimentions
for k, v := range wsary {
matched := false
// 这个算法的目的是越靠后的candles维度被命中的概率越低第一个百分之百命中后面开始越来越低, 每分钟都会发生这样的计算,
// 因为维度多了的话,照顾不过来
rand.New(rand.NewSource(time.Now().UnixNano()))
rand.Seed(time.Now().UnixNano())
n := (k*2 + 2) * 3
if n < 1 {
n = 1
}
b := rand.Intn(n)
if b < 8 {
matched = true
}
if matched {
ary = append(ary, v)
}
}
mdura := dura/(time.Duration(len(ary)+1)) - 50*time.Millisecond
// fmt.Println("loop4 Ticker Start instId, dura: ", instId, dura, dura/10, mdura, len(ary), " idx: ", kidx)
// time.Duration(len(ary)+1)
ticker := time.NewTicker(mdura)
done := make(chan bool)
idx := 0
go func(i int) {
for {
select {
case <-ticker.C:
if i >= (len(ary)) {
done <- true
break
}
rand.Seed(time.Now().UnixNano())
b := rand.Intn(2)
maxCandles = maxCandles * (i + b) * 2
if maxCandles < 3 {
maxCandles = 3
}
if maxCandles > 30 {
maxCandles = 30
}
mx := strconv.Itoa(maxCandles)
// fmt.Println("loop4 getCandlesWithRest, instId, period,limit,dura, t: ", instId, ary[i], mx, mdura)
go func(ii int) {
restQ := RestQueue{
InstId: instId,
Bar: ary[ii],
Limit: mx,
Duration: mdura,
WithWs: true,
}
js, _ := json.Marshal(restQ)
core.RedisCli.LPush("restQueue", js)
}(i)
i++
}
}
}(idx)
time.Sleep(dura - 10*time.Millisecond)
ticker.Stop()
// fmt.Println("loop4 Ticker stopped instId, dura: ", instId, dura, mdura)
done <- true
return nil
}
// 当前的时间毫秒数 对于某个时间段比如3分钟10分钟是否可以被整除
func IsModOf(curInt int64, duration time.Duration) bool {
vol := int64(0)
if duration < 24*time.Hour {
// 小于1天
vol = (curInt + 28800000)
} else if duration >= 24*time.Hour && duration < 48*time.Hour {
// 1天
vol = curInt - 1633881600000
} else if duration >= 48*time.Hour && duration < 72*time.Hour {
// 2天
vol = curInt - 1633795200000
} else if duration >= 72*time.Hour && duration < 120*time.Hour {
// 3天
vol = curInt - 1633708800000
} else if duration >= 120*time.Hour {
// 5天
vol = curInt - 1633795200000
} else {
// fmt.Println("noMatched:", curInt)
}
mody := vol % duration.Milliseconds()
if mody == 0 {
return true
}
return false
}
func (core *Core) SaveCandle(instId string, period string, rsp *CandleData, dura time.Duration, withWs bool) {
leng := len(rsp.Data)
for _, v := range rsp.Data {
candle := Candle{
InstId: instId,
Period: period,
Data: v,
From: "rest",
}
//存到elasticSearch
candle.PushToWriteLogChan(core)
//保存rest得到的candle
// 发布到allCandles|publish, 给外部订阅者用于setToKey
arys := []string{ALLCANDLES_PUBLISH}
if withWs {
arys = append(arys, ALLCANDLES_INNER_PUBLISH)
}
// 如果candle都不需要存到redis那么AddToGeneralCandleChnl也没有意义
saveCandle := os.Getenv("TEXUS_SAVECANDLE")
if saveCandle == "true" {
candle.SetToKey(core)
core.AddToGeneralCandleChnl(&candle, arys)
time.Sleep(dura / time.Duration(leng))
}
}
}
func (candle *Candle) PushToWriteLogChan(cr *Core) error {
did := candle.InstId + candle.Period + candle.Data[0].(string)
candle.Id = HashString(did)
ncd, _ := candle.ToStruct(cr)
fmt.Println("ncd: ", ncd)
cd, _ := json.Marshal(ncd)
wg := WriteLog{
Content: cd,
Tag: "sardine.log.candle." + candle.Period,
Id: candle.Id,
}
cr.WriteLogChan <- &wg
return nil
}
func Daoxu(arr []interface{}) {
var temp interface{}
length := len(arr)
for i := 0; i < length/2; i++ {
temp = arr[i]
arr[i] = arr[length-1-i]
arr[length-1-i] = temp
}
}
func HashString(input string) string {
// 计算SHA-256哈希值
hash := sha256.Sum256([]byte(input))
// 转换为十六进制字符串
hashHex := hex.EncodeToString(hash[:])
// 返回前20位
return hashHex[:23]
}
func (cl *Candle) ToStruct(core *Core) (*Candle, error) {
// cl.Timestamp
ncd := Candle{}
ncd.Id = cl.Id
ncd.Period = cl.Period
ncd.InstId = cl.InstId
ncd.From = cl.From
// 将字符串转换为 int64 类型的时间戳
ts, err := strconv.ParseInt(cl.Data[0].(string), 10, 64)
if err != nil {
fmt.Println("Error parsing timestamp:", err)
return nil, err
}
ncd.Timestamp = time.Unix(ts/1000, (ts%1000)*1000000) // 纳秒级别
op, err := strconv.ParseFloat(cl.Data[1].(string), 64)
if err != nil {
fmt.Println("Error parsing string to float64:", err)
return nil, err
}
ncd.Open = op
hi, err := strconv.ParseFloat(cl.Data[2].(string), 64)
if err != nil {
fmt.Println("Error parsing string to float64:", err)
return nil, err
}
ncd.High = hi
lo, err := strconv.ParseFloat(cl.Data[3].(string), 64)
if err != nil {
fmt.Println("Error parsing string to float64:", err)
return nil, err
}
ncd.Low = lo
clse, err := strconv.ParseFloat(cl.Data[4].(string), 64)
if err != nil {
fmt.Println("Error parsing string to float64:", err)
return nil, err
}
ncd.Close = clse
ncd.VolCcy, err = strconv.ParseFloat(cl.Data[6].(string), 64)
if err != nil {
fmt.Println("Error parsing string to float64:", err)
return nil, err
}
if cl.Data[6].(string) == "1" {
ncd.Confirm = true
} else {
ncd.Confirm = false
}
return &ncd, nil
}
func (mx *MaX) SetToKey() ([]interface{}, error) {
cstr := strconv.Itoa(mx.Count)
tss := strconv.FormatInt(mx.Ts, 10)
keyName := "ma" + cstr + "|candle" + mx.Period + "|" + mx.InstId + "|ts:" + tss
//过期时间:根号(当前candle的周期/1分钟)*10000
dt := []interface{}{}
dt = append(dt, mx.Ts)
dt = append(dt, mx.Value)
dj, _ := json.Marshal(dt)
exp := mx.Core.PeriodToMinutes(mx.Period)
expf := utils.Sqrt(float64(exp)) * 100
extt := time.Duration(expf) * time.Minute
// loc, _ := time.LoadLocation("Asia/Shanghai")
// tm := time.UnixMilli(mx.Ts).In(loc).Format("2006-01-02 15:04")
// fmt.Println("setToKey:", keyName, "ts:", tm, string(dj), "from: ", mx.From)
_, err := mx.Core.RedisCli.GetSet(keyName, dj).Result()
mx.Core.RedisCli.Expire(keyName, extt)
return dt, err
}
// 保证同一个 period, keyName 在一个周期里SaveToSortSet只会被执行一次
func (core *Core) SaveUniKey(period string, keyName string, extt time.Duration, tsi int64, cl *Candle) {
refName := keyName + "|refer"
refRes, _ := core.RedisCli.GetSet(refName, 1).Result()
core.RedisCli.Expire(refName, extt)
// 为保证唯一性机制防止SaveToSortSet 被重复执行
if len(refRes) != 0 {
fmt.Println("refName exist: ", refName)
return
}
core.SaveToSortSet(period, keyName, extt, tsi)
}
// tsi: 上报时间timeStamp millinSecond
func (core *Core) SaveToSortSet(period string, keyName string, extt time.Duration, tsi int64) {
ary := strings.Split(keyName, "ts:")
setName := ary[0] + "sortedSet"
z := redis.Z{
Score: float64(tsi),
Member: keyName,
}
rs, err := core.RedisCli.ZAdd(setName, z).Result()
if err != nil {
fmt.Println("err of ma7|ma30 add to redis:", err)
} else {
fmt.Println("sortedSet added to redis:", rs, keyName)
}
}
func (cr *Core) PeriodToMinutes(period string) int64 {
ary := strings.Split(period, "")
beiStr := "1"
danwei := ""
if len(ary) == 3 {
beiStr = ary[0] + ary[1]
danwei = ary[2]
} else {
beiStr = ary[0]
danwei = ary[1]
}
cheng := 1
bei, _ := strconv.Atoi(beiStr)
switch danwei {
case "m":
{
cheng = bei
break
}
case "H":
{
cheng = bei * 60
break
}
case "D":
{
cheng = bei * 60 * 24
break
}
case "W":
{
cheng = bei * 60 * 24 * 7
break
}
case "M":
{
cheng = bei * 60 * 24 * 30
break
}
case "Y":
{
cheng = bei * 60 * 24 * 365
break
}
default:
{
fmt.Println("notmatch:", danwei)
}
}
return int64(cheng)
}
// type ScanCmd struct {
// baseCmd
//
// page []string
// cursor uint64
//
// process func(cmd Cmder) error
// }
func (core *Core) GetRangeKeyList(pattern string, from time.Time) ([]*simple.Json, error) {
// 比如用来计算ma30或ma7倒推多少时间范围
redisCli := core.RedisCli
cursor := uint64(0)
n := 0
allTs := []int64{}
var keys []string
for {
var err error
keys, cursor, _ = redisCli.Scan(cursor, pattern+"*", 2000).Result()
if err != nil {
panic(err)
}
n += len(keys)
if n == 0 {
break
}
}
// keys, _ := redisCli.Keys(pattern + "*").Result()
for _, key := range keys {
keyAry := strings.Split(key, ":")
key = keyAry[1]
keyi64, _ := strconv.ParseInt(key, 10, 64)
allTs = append(allTs, keyi64)
}
nary := utils.RecursiveBubble(allTs, len(allTs))
tt := from.UnixMilli()
ff := tt - tt%60000
fi := int64(ff)
mary := []int64{}
for _, v := range nary {
if v < fi {
break
}
mary = append(mary, v)
}
res := []*simple.Json{}
for _, v := range mary {
// if k > 1 {
// break
// }
nv := pattern + strconv.FormatInt(v, 10)
str, err := redisCli.Get(nv).Result()
if err != nil {
fmt.Println("err of redis get key:", nv, err)
}
cur, err := simple.NewJson([]byte(str))
if err != nil {
fmt.Println("err of create newJson:", str, err)
}
res = append(res, cur)
}
return res, nil
}
func (cl *Candle) SetToKey(core *Core) ([]interface{}, error) {
data := cl.Data
tsi, err := strconv.ParseInt(data[0].(string), 10, 64)
tss := strconv.FormatInt(tsi, 10)
keyName := "candle" + cl.Period + "|" + cl.InstId + "|ts:" + tss
//过期时间:根号(当前candle的周期/1分钟)*10000
dt, err := json.Marshal(cl.Data)
exp := core.PeriodToMinutes(cl.Period)
// expf := float64(exp) * 60
expf := utils.Sqrt(float64(exp)) * 100
extt := time.Duration(expf) * time.Minute
curVolstr, _ := data[5].(string)
curVol, err := strconv.ParseFloat(curVolstr, 64)
if err != nil {
fmt.Println("err of convert ts:", err)
}
curVolCcystr, _ := data[6].(string)
curVolCcy, err := strconv.ParseFloat(curVolCcystr, 64)
curPrice := curVolCcy / curVol
if curPrice <= 0 {
fmt.Println("price有问题", curPrice, "dt: ", string(dt), "from:", cl.From)
err = errors.New("price有问题")
return cl.Data, err
}
redisCli := core.RedisCli
// tm := time.UnixMilli(tsi).Format("2006-01-02 15:04")
fmt.Println("setToKey:", keyName, "ts: ", "price: ", curPrice, "from:", cl.From)
redisCli.Set(keyName, dt, extt).Result()
core.SaveUniKey(cl.Period, keyName, extt, tsi, cl)
return cl.Data, err
}

View File

@ -1,122 +0,0 @@
package core
import (
"fmt"
"io/ioutil"
"log"
"os"
"strings"
simple "github.com/bitly/go-simplejson"
)
type MyConfig struct {
Env string `json:"env", string`
Config *simple.Json
CandleDimentions []string `json:"candleDimentions"`
RedisConf *RedisConfig `json:"redis"`
CredentialReadOnlyConf *CredentialConfig `json:"credential"`
CredentialMutableConf *CredentialConfig `json:"credential"`
ConnectConf *ConnectConfig `json:"connect"`
// ThreadsConf *ThreadsConfig `json:"threads"`
}
type RedisConfig struct {
Url string `json:"url", string`
Password string `json:"password", string`
Index int `json:"index", string`
}
type CredentialConfig struct {
SecretKey string `json:"secretKey", string`
BaseUrl string `json:"baseUrl", string`
OkAccessKey string `json:"okAccessKey", string`
OkAccessPassphrase string `json:"okAccessPassphrase", string`
}
type ConnectConfig struct {
LoginSubUrl string `json:"loginSubUrl", string`
WsPrivateBaseUrl string `json:"wsPrivateBaseUrl", string`
WsPublicBaseUrl string `json:"wsPublicBaseUrl", string`
RestBaseUrl string `json:"restBaseUrl", string`
}
type ThreadsConfig struct {
MaxLenTickerStream int `json:"maxLenTickerStream", int`
MaxCandles int `json:"maxCandles", string`
AsyncChannels int `json:"asyncChannels", int`
MaxTickers int `json:"maxTickers", int`
RestPeriod int `json:"restPeriod", int`
WaitWs int `json:"waitWs", int`
}
func (cfg MyConfig) Init() (MyConfig, error) {
env := os.Getenv("GO_ENV")
arystr := os.Getenv("TUNAS_CANDLESDIMENTIONS")
ary := strings.Split(arystr, "|")
cfg.CandleDimentions = ary
jsonStr, err := ioutil.ReadFile("/go/json/basicConfig.json")
if err != nil {
jsonStr, err = ioutil.ReadFile("configs/basicConfig.json")
if err != nil {
fmt.Println("err2:", err.Error())
return cfg, err
}
cfg.Config, err = simple.NewJson([]byte(jsonStr))
if err != nil {
fmt.Println("err2:", err.Error())
return cfg, err
}
cfg.Env = env
}
cfg.Config = cfg.Config.Get(env)
ru, err := cfg.Config.Get("redis").Get("url").String()
rp, _ := cfg.Config.Get("redis").Get("password").String()
ri, _ := cfg.Config.Get("redis").Get("index").Int()
redisConf := RedisConfig{
Url: ru,
Password: rp,
Index: ri,
}
// fmt.Println("cfg: ", cfg)
cfg.RedisConf = &redisConf
ls, _ := cfg.Config.Get("connect").Get("loginSubUrl").String()
wsPub, _ := cfg.Config.Get("connect").Get("wsPrivateBaseUrl").String()
wsPri, _ := cfg.Config.Get("connect").Get("wsPublicBaseUrl").String()
restBu, _ := cfg.Config.Get("connect").Get("restBaseUrl").String()
connectConfig := ConnectConfig{
LoginSubUrl: ls,
WsPublicBaseUrl: wsPub,
WsPrivateBaseUrl: wsPri,
RestBaseUrl: restBu,
}
cfg.ConnectConf = &connectConfig
return cfg, nil
}
func (cfg *MyConfig) GetConfigJson(arr []string) *simple.Json {
env := os.Getenv("GO_ENV")
fmt.Println("env: ", env)
cfg.Env = env
json, err := ioutil.ReadFile("/go/json/basicConfig.json")
if err != nil {
json, err = ioutil.ReadFile("configs/basicConfig.json")
if err != nil {
log.Panic("read config error: ", err.Error())
}
}
if err != nil {
fmt.Println("read file err: ", err)
}
rjson, err := simple.NewJson(json)
if err != nil {
fmt.Println("newJson err: ", err)
}
for _, s := range arr {
rjson = rjson.Get(s)
// fmt.Println(s, ": ", rjson)
}
return rjson
}

View File

@ -1,12 +0,0 @@
package core
const MAIN_ALLCOINS_PERIOD_MINUTES = 1
const MAIN_ALLCOINS_ONCE_COUNTS = 3
const MAIN_ALLCOINS_BAR_PERIOD = "3m"
const ALLCANDLES_PUBLISH = "allCandles|publish"
const ALLCANDLES_INNER_PUBLISH = "allCandlesInner|publish"
const ORDER_PUBLISH = "private|order|publish"
const TICKERINFO_PUBLISH = "tickerInfo|publish"
const CCYPOSISTIONS_PUBLISH = "ccyPositions|publish"
const SUBACTIONS_PUBLISH = "subActions|publish"
const ORDER_RESP_PUBLISH = "private|actionResp|publish"

554
vendor/github.com/phyer/core/core.go generated vendored
View File

@ -1,554 +0,0 @@
package core
import (
"context"
"encoding/json"
// "errors"
"fmt"
// "math/rand"
"io/ioutil"
"net/http"
"os"
"strconv"
"strings"
"sync"
"time"
"v5sdk_go/rest"
"v5sdk_go/ws"
// "v5sdk_go/ws/wImpl"
simple "github.com/bitly/go-simplejson"
"github.com/go-redis/redis"
"github.com/phyer/texus/private"
"github.com/phyer/texus/utils"
)
type Core struct {
Env string
Cfg *MyConfig
RedisRemoteCli *redis.Client
RedisCli *redis.Client
FluentBitUrl string
Wg sync.WaitGroup
RestQueueChan chan *RestQueue
OrderChan chan *private.Order
WriteLogChan chan *WriteLog
}
type RestQueue struct {
InstId string
Bar string
After int64
Before int64
Limit string
Duration time.Duration
WithWs bool
}
type CandleData struct {
Code string `json:"code"`
Msg string `json:"msg"`
Data [][]interface{} `json:"data"` // 用二维数组来接收 candles 数据
}
type SubAction struct {
ActionName string
ForAll bool
MetaInfo map[string]interface{}
}
func (rst *RestQueue) Show(cr *Core) {
fmt.Println("restQueue:", rst.InstId, rst.Bar, rst.Limit)
}
func (rst *RestQueue) Save(cr *Core) {
afterSec := ""
if rst.After > 0 {
afterSec = fmt.Sprint("&after=", rst.After)
}
beforeSec := ""
if rst.Before > 0 {
beforeSec = fmt.Sprint("&before=", rst.Before)
}
limitSec := ""
if len(rst.Limit) > 0 {
limitSec = fmt.Sprint("&limit=", rst.Limit)
}
link := "/api/v5/market/candles?instId=" + rst.InstId + "&bar=" + rst.Bar + limitSec + afterSec + beforeSec
fmt.Println("restLink: ", link)
rsp, err := cr.v5PublicInvoke(link)
if err != nil {
fmt.Println("cr.v5PublicInvoke err:", err)
} else {
fmt.Println("cr.v5PublicInvoke result count:", len(rsp.Data))
}
cr.SaveCandle(rst.InstId, rst.Bar, rsp, rst.Duration, rst.WithWs)
}
func WriteLogProcess(cr *Core) {
for {
wg := <-cr.WriteLogChan
go func(wg *WriteLog) {
fmt.Println("start writelog: " + wg.Tag + " " + wg.Id)
wg.Process(cr)
}(wg)
time.Sleep(50 * time.Millisecond)
}
}
func (cr *Core) ShowSysTime() {
rsp, _ := cr.RestInvoke("/api/v5/public/time", rest.GET)
fmt.Println("serverSystem time:", rsp)
}
func (core *Core) Init() {
core.Env = os.Getenv("GO_ENV")
gitBranch := os.Getenv("gitBranchName")
commitID := os.Getenv("gitCommitID")
fmt.Println("当前环境: ", core.Env)
fmt.Println("gitBranch: ", gitBranch)
fmt.Println("gitCommitID: ", commitID)
cfg := MyConfig{}
cfg, _ = cfg.Init()
core.Cfg = &cfg
cli, err := core.GetRedisCli()
core.RedisCli = cli
core.RestQueueChan = make(chan *RestQueue)
core.WriteLogChan = make(chan *WriteLog)
core.OrderChan = make(chan *private.Order)
if err != nil {
fmt.Println("init redis client err: ", err)
}
}
func (core *Core) GetRemoteRedisCli() (*redis.Client, error) {
ru := core.Cfg.RedisConf.Url
rp := core.Cfg.RedisConf.Password
ri := core.Cfg.RedisConf.Index
re := os.Getenv("REDIS_URL")
if len(re) > 0 {
ru = re
}
client := redis.NewClient(&redis.Options{
Addr: ru,
Password: rp, //默认空密码
DB: ri, //使用默认数据库
})
pong, err := client.Ping().Result()
if pong == "PONG" && err == nil {
return client, err
} else {
fmt.Println("redis状态不可用:", ru, rp, ri, err)
}
return client, nil
}
func (core *Core) GetRedisCli() (*redis.Client, error) {
ru := core.Cfg.RedisConf.Url
rp := core.Cfg.RedisConf.Password
ri := core.Cfg.RedisConf.Index
re := os.Getenv("REDIS_URL")
if len(re) > 0 {
ru = re
}
client := redis.NewClient(&redis.Options{
Addr: ru,
Password: rp, //默认空密码
DB: ri, //使用默认数据库
})
pong, err := client.Ping().Result()
if pong == "PONG" && err == nil {
return client, err
} else {
fmt.Println("redis状态不可用:", ru, rp, ri, err)
}
return client, nil
}
func (core *Core) GetAllTickerInfo() (*rest.RESTAPIResult, error) {
// GET / 获取所有产品行情信息
rsp, err := core.RestInvoke("/api/v5/market/tickers?instType=SPOT", rest.GET)
return rsp, err
}
func (core *Core) GetBalances() (*rest.RESTAPIResult, error) {
// TODO 临时用了两个实现restInvoke复用原来的会有bug不知道是谁的bug
rsp, err := core.RestInvoke2("/api/v5/account/balance", rest.GET, nil)
return rsp, err
}
func (core *Core) GetLivingOrderList() ([]*private.Order, error) {
// TODO 临时用了两个实现restInvoke复用原来的会有bug不知道是谁的bug
params := make(map[string]interface{})
data, err := core.RestInvoke2("/api/v5/trade/orders-pending", rest.GET, &params)
odrsp := private.OrderResp{}
err = json.Unmarshal([]byte(data.Body), &odrsp)
str, _ := json.Marshal(odrsp)
fmt.Println("convert: err:", err, " body: ", data.Body, odrsp, " string:", string(str))
list, err := odrsp.Convert()
fmt.Println("loopLivingOrder response data:", str)
fmt.Println(utils.GetFuncName(), " 当前数据是 ", data.V5Response.Code, " list len:", len(list))
return list, err
}
func (core *Core) LoopInstrumentList() error {
for {
time.Sleep(3 * time.Second)
ctype := ws.SPOT
redisCli := core.RedisCli
counts, err := redisCli.HLen("instruments|" + ctype + "|hash").Result()
if err != nil {
fmt.Println("err of hset to redis:", err)
}
if counts == 0 {
continue
}
break
}
return nil
}
func (core *Core) SubscribeTicker(op string) error {
mp := make(map[string]string)
redisCli := core.RedisCli
ctype := ws.SPOT
mp, err := redisCli.HGetAll("instruments|" + ctype + "|hash").Result()
b, err := json.Marshal(mp)
js, err := simple.NewJson(b)
if err != nil {
fmt.Println("err of unMarshalJson3:", js)
}
// fmt.Println("ticker js: ", js)
instAry := js.MustMap()
for k, v := range instAry {
b = []byte(v.(string))
_, err := simple.NewJson(b)
if err != nil {
fmt.Println("err of unMarshalJson4:", js)
}
time.Sleep(5 * time.Second)
go func(instId string, op string) {
redisCli := core.RedisCli
_, err = redisCli.SAdd("tickers|"+op+"|set", instId).Result()
if err != nil {
fmt.Println("err of unMarshalJson5:", js)
}
}(k, op)
}
return nil
}
// 通过接口获取一个币种名下的某个时间范围内的Candle对象集合
func (core *Core) v5PublicInvoke(subUrl string) (*CandleData, error) {
restUrl, _ := core.Cfg.Config.Get("connect").Get("restBaseUrl").String()
url := restUrl + subUrl
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
var result CandleData
if err := json.Unmarshal(body, &result); err != nil {
return nil, err
}
return &result, nil
}
func (core *Core) RestInvoke(subUrl string, method string) (*rest.RESTAPIResult, error) {
restUrl, _ := core.Cfg.Config.Get("connect").Get("restBaseUrl").String()
//ep, method, uri string, param *map[string]interface{}
rest := rest.NewRESTAPI(restUrl, method, subUrl, nil)
key, _ := core.Cfg.Config.Get("credentialReadOnly").Get("okAccessKey").String()
secure, _ := core.Cfg.Config.Get("credentialReadOnly").Get("secretKey").String()
pass, _ := core.Cfg.Config.Get("credentialReadOnly").Get("okAccessPassphrase").String()
isDemo := false
if core.Env == "demoEnv" {
isDemo = true
}
rest.SetSimulate(isDemo).SetAPIKey(key, secure, pass)
response, err := rest.Run(context.Background())
if err != nil {
fmt.Println("restInvoke1 err:", subUrl, err)
}
return response, err
}
func (core *Core) RestInvoke2(subUrl string, method string, param *map[string]interface{}) (*rest.RESTAPIResult, error) {
key, err1 := core.Cfg.Config.Get("credentialReadOnly").Get("okAccessKey").String()
secret, err2 := core.Cfg.Config.Get("credentialReadOnly").Get("secretKey").String()
pass, err3 := core.Cfg.Config.Get("credentialReadOnly").Get("okAccessPassphrase").String()
userId, err4 := core.Cfg.Config.Get("connect").Get("userId").String()
restUrl, err5 := core.Cfg.Config.Get("connect").Get("restBaseUrl").String()
if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil {
fmt.Println(err1, err2, err3, err4, err5)
} else {
fmt.Println("key:", key, secret, pass, "userId:", userId, "restUrl: ", restUrl)
}
// reqParam := make(map[string]interface{})
// if param != nil {
// reqParam = *param
// }
// rs := rest.NewRESTAPI(restUrl, method, subUrl, &reqParam)
isDemo := false
if core.Env == "demoEnv" {
isDemo = true
}
// rs.SetSimulate(isDemo).SetAPIKey(key, secret, pass).SetUserId(userId)
// response, err := rs.Run(context.Background())
// if err != nil {
// fmt.Println("restInvoke2 err:", subUrl, err)
// }
apikey := rest.APIKeyInfo{
ApiKey: key,
SecKey: secret,
PassPhrase: pass,
}
cli := rest.NewRESTClient(restUrl, &apikey, isDemo)
rsp, err := cli.Get(context.Background(), subUrl, param)
if err != nil {
return rsp, err
}
fmt.Println("response:", rsp, err)
return rsp, err
}
func (core *Core) RestPost(subUrl string, param *map[string]interface{}) (*rest.RESTAPIResult, error) {
key, err1 := core.Cfg.Config.Get("credentialMutable").Get("okAccessKey").String()
secret, err2 := core.Cfg.Config.Get("credentialMutable").Get("secretKey").String()
pass, err3 := core.Cfg.Config.Get("credentialMutable").Get("okAccessPassphrase").String()
userId, err4 := core.Cfg.Config.Get("connect").Get("userId").String()
restUrl, err5 := core.Cfg.Config.Get("connect").Get("restBaseUrl").String()
if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil {
fmt.Println(err1, err2, err3, err4, err5)
} else {
fmt.Println("key:", key, secret, pass, "userId:", userId, "restUrl: ", restUrl)
}
// 请求的另一种方式
apikey := rest.APIKeyInfo{
ApiKey: key,
SecKey: secret,
PassPhrase: pass,
}
isDemo := false
if core.Env == "demoEnv" {
isDemo = true
}
cli := rest.NewRESTClient(restUrl, &apikey, isDemo)
rsp, err := cli.Post(context.Background(), subUrl, param)
if err != nil {
return rsp, err
}
return rsp, err
}
// 我当前持有的币,每分钟刷新
func (core *Core) GetMyFavorList() []string {
redisCli := core.RedisCli
opt := redis.ZRangeBy{
Min: "10",
Max: "100000000000",
}
cary, _ := redisCli.ZRevRangeByScore("private|positions|sortedSet", opt).Result()
cl := []string{}
for _, v := range cary {
cry := strings.Split(v, "|")[0] + "-USDT"
cl = append(cl, cry)
}
return cary
}
// 得到交易量排行榜,订阅其中前N名的各维度k线,并merge进来我已经购买的币列表这个列表是动态更新的
// 改了,不需要交易排行榜,我手动指定一个排行即可, tickersVol|sortedSet 改成 tickersList|sortedSet
func (core *Core) GetScoreList(count int) []string {
redisCli := core.RedisCli
curList, err := redisCli.ZRange("tickersList|sortedSet", 0, int64(count-1)).Result()
if err != nil {
fmt.Println("zrevrange err:", err)
}
fmt.Println("curList: ", curList)
return curList
}
func LoopBalances(cr *Core, mdura time.Duration) {
//协程动态维护topScore
ticker := time.NewTicker(mdura)
for {
select {
case <-ticker.C:
//协程循环执行rest请求candle
fmt.Println("loopBalance: receive ccyChannel start")
RestBalances(cr)
}
}
}
func LoopLivingOrders(cr *Core, mdura time.Duration) {
//协程动态维护topScore
ticker := time.NewTicker(mdura)
for {
select {
case <-ticker.C:
//协程循环执行rest请求candle
fmt.Println("loopLivingOrder: receive ccyChannel start")
RestLivingOrder(cr)
}
}
}
func RestBalances(cr *Core) ([]*private.Ccy, error) {
// fmt.Println("restBalance loopBalance loop start")
ccyList := []*private.Ccy{}
rsp, err := cr.GetBalances()
if err != nil {
fmt.Println("loopBalance err00: ", err)
}
fmt.Println("loopBalance balance rsp: ", rsp)
if err != nil {
fmt.Println("loopBalance err01: ", err)
return ccyList, err
}
if len(rsp.Body) == 0 {
fmt.Println("loopBalance err03: rsp body is null")
return ccyList, err
}
js1, err := simple.NewJson([]byte(rsp.Body))
if err != nil {
fmt.Println("loopBalance err1: ", err)
}
itemList := js1.Get("data").GetIndex(0).Get("details").MustArray()
// maxTickers是重点关注的topScore的coins的数量
cli := cr.RedisCli
for _, v := range itemList {
js, _ := json.Marshal(v)
ccyResp := private.CcyResp{}
err := json.Unmarshal(js, &ccyResp)
if err != nil {
fmt.Println("loopBalance err2: ", err)
}
ccy, err := ccyResp.Convert()
ccyList = append(ccyList, ccy)
if err != nil {
fmt.Println("loopBalance err2: ", err)
}
z := redis.Z{
Score: ccy.EqUsd,
Member: ccy.Ccy + "|position|key",
}
res, err := cli.ZAdd("private|positions|sortedSet", z).Result()
if err != nil {
fmt.Println("loopBalance err3: ", res, err)
}
res1, err := cli.Set(ccy.Ccy+"|position|key", js, 0).Result()
if err != nil {
fmt.Println("loopBalance err4: ", res1, err)
}
bjs, _ := json.Marshal(ccy)
tsi := time.Now().Unix()
tsii := tsi - tsi%600
tss := strconv.FormatInt(tsii, 10)
cli.Set(CCYPOSISTIONS_PUBLISH+"|ts:"+tss, 1, 10*time.Minute).Result()
fmt.Println("ccy published: ", string(bjs))
//TODO FIXME 50毫秒每分钟上限是1200个订单超过就无法遍历完成
time.Sleep(50 * time.Millisecond)
suffix := ""
if cr.Env == "demoEnv" {
suffix = "-demoEnv"
}
// TODO FIXME cli2
cli.Publish(CCYPOSISTIONS_PUBLISH+suffix, string(bjs)).Result()
}
return ccyList, nil
}
func RestLivingOrder(cr *Core) ([]*private.Order, error) {
// fmt.Println("restOrder loopOrder loop start")
orderList := []*private.Order{}
list, err := cr.GetLivingOrderList()
if err != nil {
fmt.Println("loopLivingOrder err00: ", err)
}
fmt.Println("loopLivingOrder response:", list)
go func() {
for _, v := range list {
fmt.Println("order orderV:", v)
time.Sleep(30 * time.Millisecond)
cr.OrderChan <- v
}
}()
return orderList, nil
}
func (cr *Core) ProcessOrder(od *private.Order) error {
// publish
go func() {
suffix := ""
if cr.Env == "demoEnv" {
suffix = "-demoEnv"
}
cn := ORDER_PUBLISH + suffix
bj, _ := json.Marshal(od)
// TODO FIXME cli2
res, _ := cr.RedisCli.Publish(cn, string(bj)).Result()
fmt.Println("order publish res: ", res, " content: ", string(bj))
rsch := ORDER_RESP_PUBLISH + suffix
bj1, _ := json.Marshal(res)
// TODO FIXME cli2
res, _ = cr.RedisCli.Publish(rsch, string(bj1)).Result()
}()
return nil
}
func (cr *Core) DispatchSubAction(action *SubAction) error {
go func() {
suffix := ""
if cr.Env == "demoEnv" {
suffix = "-demoEnv"
}
fmt.Println("action: ", action.ActionName, action.MetaInfo)
res, err := cr.RestPostWrapper("/api/v5/trade/"+action.ActionName, action.MetaInfo)
if err != nil {
fmt.Println(utils.GetFuncName(), " actionRes 1:", err)
}
rsch := ORDER_RESP_PUBLISH + suffix
bj1, _ := json.Marshal(res)
// TODO FIXME cli2
rs, _ := cr.RedisCli.Publish(rsch, string(bj1)).Result()
fmt.Println("action rs: ", rs)
}()
return nil
}
func (cr *Core) RestPostWrapper(url string, param map[string]interface{}) (rest.Okexv5APIResponse, error) {
suffix := ""
if cr.Env == "demoEnv" {
suffix = "-demoEnv"
}
res, err := cr.RestPost(url, &param)
fmt.Println("actionRes 2:", res.V5Response.Msg, res.V5Response.Data, err)
bj, _ := json.Marshal(res)
// TODO FIXME cli2
cr.RedisCli.Publish(ORDER_RESP_PUBLISH+suffix, string(bj))
return res.V5Response, nil
}
func (cr *Core) AddToGeneralCandleChnl(candle *Candle, channels []string) {
redisCli := cr.RedisCli
ab, _ := json.Marshal(candle)
for _, v := range channels {
suffix := ""
env := os.Getenv("GO_ENV")
if env == "demoEnv" {
suffix = "-demoEnv"
}
vd := v + suffix
_, err := redisCli.Publish(vd, string(ab)).Result()
if err != nil {
fmt.Println("err of ma7|ma30 add to redis2:", err, candle.From)
}
}
}

View File

@ -1,70 +0,0 @@
package core
import (
"fmt"
"reflect"
"strconv"
)
type TickerInfo struct {
Id string `json:"_id"`
InstId string `json:"instId"`
Last float64 `json:"last"`
InstType string `json:"instType"`
VolCcy24h float64 `json:"volCcy24h"`
Ts int64 `json:"ts"`
}
type TickerInfoResp struct {
InstId string `json:"instId"`
Last string `json:"last"`
InstType string `json:"instType"`
VolCcy24h string `json:"volCcy24h"`
Ts string `json:"ts"`
}
func (tir *TickerInfoResp) Convert() TickerInfo {
ti := TickerInfo{
Id: HashString(tir.InstId + tir.Ts),
InstId: tir.InstId,
InstType: tir.InstType,
Last: ToFloat64(tir.Last),
VolCcy24h: ToFloat64(tir.VolCcy24h),
Ts: ToInt64(tir.Ts),
}
return ti
}
func ToString(val interface{}) string {
valstr := ""
if reflect.TypeOf(val).Name() == "string" {
valstr = val.(string)
} else if reflect.TypeOf(val).Name() == "float64" {
valstr = fmt.Sprintf("%f", val)
} else if reflect.TypeOf(val).Name() == "int64" {
valstr = strconv.FormatInt(val.(int64), 16)
}
return valstr
}
func ToInt64(val interface{}) int64 {
vali := int64(0)
if reflect.TypeOf(val).Name() == "string" {
vali, _ = strconv.ParseInt(val.(string), 10, 64)
} else if reflect.TypeOf(val).Name() == "float64" {
vali = int64(val.(float64))
}
return vali
}
func ToFloat64(val interface{}) float64 {
valf := float64(0)
if reflect.TypeOf(val).Name() == "string" {
valf, _ = strconv.ParseFloat(val.(string), 64)
} else if reflect.TypeOf(val).Name() == "float64" {
valf = val.(float64)
} else if reflect.TypeOf(val).Name() == "int64" {
valf = float64(val.(int64))
}
return valf
}

View File

@ -1,32 +0,0 @@
package core
import (
"bytes"
"fmt"
"net/http"
"os"
logrus "github.com/sirupsen/logrus"
)
type WriteLog struct {
Content []byte
Tag string
Id string
}
func (wg *WriteLog) Process(cr *Core) error {
go func() {
reqBody := bytes.NewBuffer(wg.Content)
cr.Env = os.Getenv("GO_ENV")
cr.FluentBitUrl = os.Getenv("TEXUS_FluentBitUrl")
fullUrl := "http://" + cr.FluentBitUrl + "/" + wg.Tag
res, err := http.Post(fullUrl, "application/json", reqBody)
fmt.Println("requested, response:", fullUrl, string(wg.Content), res)
if err != nil {
logrus.Error(err)
}
}()
return nil
}

View File

@ -1,73 +0,0 @@
package private
import (
"strconv"
)
// {"availEq":0,"cashBal":2397.6,"ccy":"FODL","disEq":0,"eq":2397.6,"eqUsd":340.3419189984,"frozenBal":2397.6,"ordFrozen":2397.6,"uTime":1649902953671}
// {"availEq":0,"cashBal":12987,"ccy":"ANW","disEq":0,"eq":12987,"eqUsd":358.555901184,"frozenBal":12987,"ordFrozen":12987,"uTime":1649596232202}
// {"availEq":0,"cashBal":6995.8,"ccy":"ABT","disEq":0,"eq":6995.8,"eqUsd":1082.5316450676,"frozenBal":6995.8,"ordFrozen":6995.8,"uTime":1648835098048}
// "availBal string ",
// "crossLiab string`json:""`
// "interest string`json:""`
// "isoEq string`json:""`
// "isoLiab string`json:""`
// "isoUpl":"0",
// "liab string`json:""`
// "maxLoan string 1453.92289531493594",
// "mgnRatio string ",
// "notionalLever string ",
// "ordFrozen string`json:""`
// "twap string`json:""`
// "upl string 0.570822125136023",
// "uplLiab string`json:""`
// "stgyEq":"0"
type CcyResp struct {
AvailEq string `json:"availEq"`
CashBal string `json:"cashBal"`
Ccy string `json:"ccy"`
DisEq string `json:"disEq"`
Eq string `json:"eq"`
EqUsd string `json:"eqUsd"`
FrozenBal string `json:"frozenBal"`
OrdFrozen string `json:"ordFrozen"`
UTime string `json:"uTime"`
AvailBal string `json:"availBal"`
}
type Ccy struct {
AvailEq float64 `json:"availEq"`
CashBal float64 `json:"cashBal"`
Ccy string `json:"ccy"`
DisEq float64 `json:"disEq"`
Eq float64 `json:"eq":wa`
EqUsd float64 `json:"eqUsd"`
FrozenBal float64 `json:"frozenBal"`
OrdFrozen float64 `json:"ordFrozen"`
UTime int64 `json:"uTime"`
AvailBal float64 `json:"availBal"`
}
type BalanceResp struct {
}
type Balance struct {
}
func (ccyResp *CcyResp) Convert() (*Ccy, error) {
ccy := Ccy{}
ccy.Ccy = ccyResp.Ccy
ccy.AvailEq, _ = strconv.ParseFloat(ccyResp.AvailEq, 64)
ccy.CashBal, _ = strconv.ParseFloat(ccyResp.CashBal, 64)
ccy.DisEq, _ = strconv.ParseFloat(ccyResp.DisEq, 64)
ccy.Eq, _ = strconv.ParseFloat(ccyResp.Eq, 64)
ccy.EqUsd, _ = strconv.ParseFloat(ccyResp.EqUsd, 64)
ccy.FrozenBal, _ = strconv.ParseFloat(ccyResp.FrozenBal, 64)
ccy.OrdFrozen, _ = strconv.ParseFloat(ccyResp.OrdFrozen, 64)
ccy.UTime, _ = strconv.ParseInt(ccyResp.UTime, 10, 64)
return &ccy, nil
}
func (balanceResp *BalanceResp) Convert() (*Balance, error) {
return nil, nil
}

View File

@ -1,107 +0,0 @@
package private
import (
"github.com/phyer/texus/utils"
)
type OrderResp struct {
Arg ArgResp `json:"arg"`
Data []*OrderDataResp `json:"data"`
}
type ArgResp struct {
Channel string `json:"channel"`
InstType string `json:"instType"`
InstId string `json:"instId"`
}
type OrderDataResp struct {
InstType string `json:"instType"`
InstId string `json:"instId"`
OrdId string `json:"ordId"`
ClOrdId string `json:"clOrdId"`
Tag string `json:"tag"`
Px string `json:"Px"`
Sz string `json:"sz"`
NotionalUsd string `json:"notionalUsd"`
OrdType string `json:"ordType"`
Side string `json:"side"`
PosSide string `json:"posSide"`
TdMode string `json:"tdMode"`
TgtCcy string `json:"tgtCcy"`
FillSz string `json:"fillSz"`
FillPx string `json:"fillPx"`
TradeId string `json:"tradeId"`
AccFillSz string `json:"accFillSz"`
FillNotionalUsd string `json:"FillNotionalUsd"`
FillTime string `json:"fillTime"`
FillFee string `json:"fillFee"`
FillFeeCcy string `json:"fillFeeCcy"`
ExecType string `json:"execType"`
Source string `json:"source"`
State string `json:"state"`
AvgPx string `json:"avgPx"`
Lever string `json:"lever"`
TpTriggerPxstring string `json:"tpTriggerPxstring"`
TpTriggerPxType string `json:"tpTriggerPxType"`
TpOrdPx string `json:"tpOrdPx"`
SlTriggerPx string `json:"slTriggerPx"`
SlTriggerPxType string `json:"slTriggerPxType"`
SlOrdPx string `json:"slOrdPx"`
FeeCcy string `json:"feeCcy"`
Fee string `json:"fee"`
RebateCcy string `json:"rebateCcy"`
Rebate string `json:"rebate"`
TgtCcystring string `json:"tgtCcystring"`
Pnl string `json:"pnl"`
Category string `json:"category"`
UTime string `json:"uTime"`
CTime string `json:"cTime"`
ReqId string `json:"reqId"`
AmendResult string `json:"amendResult"`
Code string `json:"code"`
Msg string `json:"msg"`
}
type Order struct {
CTime int64 `json:"cTime,number"` // 订单创建时间
UTime int64 `json:"uTime,number"` // 订单状态更新时间
InstId string `json:"instId"`
OrdId string `json:"ordId"`
ClOrdId string `json:"clOrdId"` // 自定义订单ID
Px float64 `json:"px,number"` // 委托价格
Side string `json:"side"` //交易方向 sell, buy
Sz float64 `json:"sz,number"` // 委托总量
AccFillSz float64 `json:"accFillSz,number"` //累计成交数量
AvgPx float64 `json:"avgPx,number"` //成交均价
State string `json:"state"` //订单状态
TgtCcy string `json:"tgtCcy"` // 限定了size的单位两个选项base_ccy: 交易货币 quote_ccy计价货币(美元)
// canceled撤单成功
// live等待成交
// partially_filled部分成交
// filled完全成交
}
func (orderResp *OrderResp) Convert() ([]*Order, error) {
orderList := []*Order{}
for _, v := range orderResp.Data {
// fmt.Println("v.Sz:", v.Sz, reflect.TypeOf(v.Sz).Name())
curOrder := Order{}
curOrder.CTime = utils.ToInt64(v.CTime)
curOrder.UTime = utils.ToInt64(v.UTime)
curOrder.InstId = v.InstId
curOrder.OrdId = v.OrdId
curOrder.ClOrdId = v.ClOrdId
curOrder.Side = v.Side
curOrder.Px = utils.ToFloat64(v.Px)
curOrder.Sz = utils.ToFloat64(v.Sz)
curOrder.AccFillSz = utils.ToFloat64(v.AccFillSz)
curOrder.AvgPx = utils.ToFloat64(v.AvgPx)
curOrder.State = v.State
orderList = append(orderList, &curOrder)
}
return orderList, nil
}
// func (order *Order) Process(cr *core.Core) error {
// return nil
// }

View File

@ -1,14 +0,0 @@
package utils
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
)
func ComputeHmac256(message string, secret string) string {
key := []byte(secret)
h := hmac.New(sha256.New, key)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}

View File

@ -1,21 +0,0 @@
package utils
func RecursiveBubble(ary []int64, length int) []int64 {
if length == 0 {
return ary
}
for idx, _ := range ary {
if idx >= length-1 {
break
}
temp := int64(0)
if ary[idx] < ary[idx+1] { //改变成>,换成从小到大排序
temp = ary[idx]
ary[idx] = ary[idx+1]
ary[idx+1] = temp
}
}
length--
RecursiveBubble(ary, length)
return ary
}

View File

@ -1,51 +0,0 @@
package utils
import (
"fmt"
"sync"
)
type MyStack struct {
Stack []interface{}
CType string
lock sync.RWMutex
Len int
}
func (c *MyStack) Push(obj interface{}) {
c.lock.Lock()
if c.Len == len(c.Stack) {
c.Stack = c.Stack[1:]
}
defer c.lock.Unlock()
c.Stack = append(c.Stack, obj)
}
func (c *MyStack) Pop() error {
len := len(c.Stack)
if len > 0 {
c.lock.Lock()
defer c.lock.Unlock()
c.Stack = c.Stack[:len-1]
return nil
}
return fmt.Errorf("Pop Error: Stack is empty")
}
func (c *MyStack) Front() (interface{}, error) {
len := len(c.Stack)
if len > 0 {
c.lock.Lock()
defer c.lock.Unlock()
return c.Stack[len-1], nil
}
return "", fmt.Errorf("Peep Error: Stack is empty")
}
func (c *MyStack) Size() int {
return len(c.Stack)
}
func (c *MyStack) Empty() bool {
return len(c.Stack) == 0
}

View File

@ -1,51 +0,0 @@
package utils
import (
"math/rand"
"reflect"
"strconv"
"time"
)
func GetRandListChan(rg int, count int) []string {
strAry := []string{}
for i := 0; i < count; i++ {
rand.Seed(time.Now().UnixNano())
b := rand.Intn(rg)
bs := strconv.Itoa(b)
strAry = append(strAry, bs)
}
return strAry
}
func GetRandomString(l int) string {
str := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
bytes := []byte(str)
result := []byte{}
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for i := 0; i < l; i++ {
result = append(result, bytes[r.Intn(len(bytes))])
}
return string(result)
}
//判断某一个值是否含在切片之中
func In_Array(val interface{}, array interface{}) (exists bool, index int) {
exists = false
index = -1
switch reflect.TypeOf(array).Kind() {
case reflect.Slice:
s := reflect.ValueOf(array)
for i := 0; i < s.Len(); i++ {
if reflect.DeepEqual(val, s.Index(i).Interface()) == true {
index = i
exists = true
return
}
}
}
return
}

View File

@ -1,11 +0,0 @@
package utils
import "math"
func Sqrt(x float64) float64 {
z := 1.0
for math.Abs(z*z-x) > 0.000001 {
z -= (z*z - x) / (2 * z)
}
return z
}

View File

@ -1,116 +0,0 @@
package utils
import (
"crypto/md5"
"fmt"
"reflect"
"runtime"
"strconv"
"time"
)
type PushRestQ func(int, []string) error
// 获取当前函数名字
func GetFuncName() string {
pc := make([]uintptr, 1)
runtime.Callers(2, pc)
f := runtime.FuncForPC(pc[0])
return f.Name()
}
func TickerWrapper(mdura time.Duration, ary []string, pr PushRestQ) {
done := make(chan bool)
idx := 0
per2 := time.Duration(mdura) / time.Duration(len(ary)+2)
// fmt.Println("mdura, len of ary, per2: ", mdura, len(ary), per2)
if per2 < 100*time.Millisecond {
per2 = 100 * time.Millisecond
}
ticker := time.NewTicker(per2)
go func(i int) {
for {
select {
case <-ticker.C:
if i >= (len(ary) - 1) {
done <- true
break
}
go func(i int) {
err := pr(i, ary)
if err != nil {
fmt.Println("inner err: ", err)
}
}(i)
i++
}
}
}(idx)
time.Sleep(mdura)
}
func HashDispatch(originName string, count uint8) int {
data := []byte(originName)
bytes := md5.Sum(data)
res := uint8(0)
for _, v := range bytes {
res += v
}
res = res % count
return int(res)
}
func IsoTime() string {
utcTime := time.Now().UTC()
iso := utcTime.String()
isoBytes := []byte(iso)
iso = string(isoBytes[:10]) + "T" + string(isoBytes[11:23]) + "Z"
return iso
}
// 闹钟,
func Fenci(count int) bool {
tsi := time.Now().Unix()
tsi = tsi - tsi%60
cha := tsi % (int64(count))
if cha == 0 {
return true
}
return false
}
func ToString(val interface{}) string {
valstr := ""
if reflect.TypeOf(val).Name() == "string" {
valstr = val.(string)
} else if reflect.TypeOf(val).Name() == "float64" {
valstr = fmt.Sprintf("%f", val)
} else if reflect.TypeOf(val).Name() == "int64" {
valstr = strconv.FormatInt(val.(int64), 16)
}
return valstr
}
func ToInt64(val interface{}) int64 {
vali := int64(0)
if reflect.TypeOf(val).Name() == "string" {
vali, _ = strconv.ParseInt(val.(string), 10, 64)
} else if reflect.TypeOf(val).Name() == "float64" {
vali = int64(val.(float64))
}
return vali
}
func ToFloat64(val interface{}) float64 {
valf := float64(0)
ctype := reflect.TypeOf(val).Name()
if ctype == "string" {
valf, _ = strconv.ParseFloat(val.(string), 64)
} else if reflect.TypeOf(val).Name() == "float64" {
valf = val.(float64)
} else if reflect.TypeOf(val).Name() == "int64" {
valf = float64(val.(int64))
} else {
fmt.Println("convert err:", val, ":", ctype)
}
return valf
}

View File

@ -1,4 +0,0 @@
logrus
vendor
.idea/

View File

@ -1,40 +0,0 @@
run:
# do not run on test files yet
tests: false
# all available settings of specific linters
linters-settings:
errcheck:
# report about not checking of errors in type assetions: `a := b.(MyStruct)`;
# default is false: such cases aren't reported by default.
check-type-assertions: false
# report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
# default is false: such cases aren't reported by default.
check-blank: false
lll:
line-length: 100
tab-width: 4
prealloc:
simple: false
range-loops: false
for-loops: false
whitespace:
multi-if: false # Enforces newlines (or comments) after every multi-line if statement
multi-func: false # Enforces newlines (or comments) after every multi-line function signature
linters:
enable:
- megacheck
- govet
disable:
- maligned
- prealloc
disable-all: false
presets:
- bugs
- unused
fast: false

View File

@ -1,15 +0,0 @@
language: go
go_import_path: github.com/sirupsen/logrus
git:
depth: 1
env:
- GO111MODULE=on
go: 1.15.x
os: linux
install:
- ./travis/install.sh
script:
- cd ci
- go run mage.go -v -w ../ crossBuild
- go run mage.go -v -w ../ lint
- go run mage.go -v -w ../ test

View File

@ -1,259 +0,0 @@
# 1.8.1
Code quality:
* move magefile in its own subdir/submodule to remove magefile dependency on logrus consumer
* improve timestamp format documentation
Fixes:
* fix race condition on logger hooks
# 1.8.0
Correct versioning number replacing v1.7.1.
# 1.7.1
Beware this release has introduced a new public API and its semver is therefore incorrect.
Code quality:
* use go 1.15 in travis
* use magefile as task runner
Fixes:
* small fixes about new go 1.13 error formatting system
* Fix for long time race condiction with mutating data hooks
Features:
* build support for zos
# 1.7.0
Fixes:
* the dependency toward a windows terminal library has been removed
Features:
* a new buffer pool management API has been added
* a set of `<LogLevel>Fn()` functions have been added
# 1.6.0
Fixes:
* end of line cleanup
* revert the entry concurrency bug fix whic leads to deadlock under some circumstances
* update dependency on go-windows-terminal-sequences to fix a crash with go 1.14
Features:
* add an option to the `TextFormatter` to completely disable fields quoting
# 1.5.0
Code quality:
* add golangci linter run on travis
Fixes:
* add mutex for hooks concurrent access on `Entry` data
* caller function field for go1.14
* fix build issue for gopherjs target
Feature:
* add an hooks/writer sub-package whose goal is to split output on different stream depending on the trace level
* add a `DisableHTMLEscape` option in the `JSONFormatter`
* add `ForceQuote` and `PadLevelText` options in the `TextFormatter`
# 1.4.2
* Fixes build break for plan9, nacl, solaris
# 1.4.1
This new release introduces:
* Enhance TextFormatter to not print caller information when they are empty (#944)
* Remove dependency on golang.org/x/crypto (#932, #943)
Fixes:
* Fix Entry.WithContext method to return a copy of the initial entry (#941)
# 1.4.0
This new release introduces:
* Add `DeferExitHandler`, similar to `RegisterExitHandler` but prepending the handler to the list of handlers (semantically like `defer`) (#848).
* Add `CallerPrettyfier` to `JSONFormatter` and `TextFormatter` (#909, #911)
* Add `Entry.WithContext()` and `Entry.Context`, to set a context on entries to be used e.g. in hooks (#919).
Fixes:
* Fix wrong method calls `Logger.Print` and `Logger.Warningln` (#893).
* Update `Entry.Logf` to not do string formatting unless the log level is enabled (#903)
* Fix infinite recursion on unknown `Level.String()` (#907)
* Fix race condition in `getCaller` (#916).
# 1.3.0
This new release introduces:
* Log, Logf, Logln functions for Logger and Entry that take a Level
Fixes:
* Building prometheus node_exporter on AIX (#840)
* Race condition in TextFormatter (#468)
* Travis CI import path (#868)
* Remove coloured output on Windows (#862)
* Pointer to func as field in JSONFormatter (#870)
* Properly marshal Levels (#873)
# 1.2.0
This new release introduces:
* A new method `SetReportCaller` in the `Logger` to enable the file, line and calling function from which the trace has been issued
* A new trace level named `Trace` whose level is below `Debug`
* A configurable exit function to be called upon a Fatal trace
* The `Level` object now implements `encoding.TextUnmarshaler` interface
# 1.1.1
This is a bug fix release.
* fix the build break on Solaris
* don't drop a whole trace in JSONFormatter when a field param is a function pointer which can not be serialized
# 1.1.0
This new release introduces:
* several fixes:
* a fix for a race condition on entry formatting
* proper cleanup of previously used entries before putting them back in the pool
* the extra new line at the end of message in text formatter has been removed
* a new global public API to check if a level is activated: IsLevelEnabled
* the following methods have been added to the Logger object
* IsLevelEnabled
* SetFormatter
* SetOutput
* ReplaceHooks
* introduction of go module
* an indent configuration for the json formatter
* output colour support for windows
* the field sort function is now configurable for text formatter
* the CLICOLOR and CLICOLOR\_FORCE environment variable support in text formater
# 1.0.6
This new release introduces:
* a new api WithTime which allows to easily force the time of the log entry
which is mostly useful for logger wrapper
* a fix reverting the immutability of the entry given as parameter to the hooks
a new configuration field of the json formatter in order to put all the fields
in a nested dictionnary
* a new SetOutput method in the Logger
* a new configuration of the textformatter to configure the name of the default keys
* a new configuration of the text formatter to disable the level truncation
# 1.0.5
* Fix hooks race (#707)
* Fix panic deadlock (#695)
# 1.0.4
* Fix race when adding hooks (#612)
* Fix terminal check in AppEngine (#635)
# 1.0.3
* Replace example files with testable examples
# 1.0.2
* bug: quote non-string values in text formatter (#583)
* Make (*Logger) SetLevel a public method
# 1.0.1
* bug: fix escaping in text formatter (#575)
# 1.0.0
* Officially changed name to lower-case
* bug: colors on Windows 10 (#541)
* bug: fix race in accessing level (#512)
# 0.11.5
* feature: add writer and writerlevel to entry (#372)
# 0.11.4
* bug: fix undefined variable on solaris (#493)
# 0.11.3
* formatter: configure quoting of empty values (#484)
* formatter: configure quoting character (default is `"`) (#484)
* bug: fix not importing io correctly in non-linux environments (#481)
# 0.11.2
* bug: fix windows terminal detection (#476)
# 0.11.1
* bug: fix tty detection with custom out (#471)
# 0.11.0
* performance: Use bufferpool to allocate (#370)
* terminal: terminal detection for app-engine (#343)
* feature: exit handler (#375)
# 0.10.0
* feature: Add a test hook (#180)
* feature: `ParseLevel` is now case-insensitive (#326)
* feature: `FieldLogger` interface that generalizes `Logger` and `Entry` (#308)
* performance: avoid re-allocations on `WithFields` (#335)
# 0.9.0
* logrus/text_formatter: don't emit empty msg
* logrus/hooks/airbrake: move out of main repository
* logrus/hooks/sentry: move out of main repository
* logrus/hooks/papertrail: move out of main repository
* logrus/hooks/bugsnag: move out of main repository
* logrus/core: run tests with `-race`
* logrus/core: detect TTY based on `stderr`
* logrus/core: support `WithError` on logger
* logrus/core: Solaris support
# 0.8.7
* logrus/core: fix possible race (#216)
* logrus/doc: small typo fixes and doc improvements
# 0.8.6
* hooks/raven: allow passing an initialized client
# 0.8.5
* logrus/core: revert #208
# 0.8.4
* formatter/text: fix data race (#218)
# 0.8.3
* logrus/core: fix entry log level (#208)
* logrus/core: improve performance of text formatter by 40%
* logrus/core: expose `LevelHooks` type
* logrus/core: add support for DragonflyBSD and NetBSD
* formatter/text: print structs more verbosely
# 0.8.2
* logrus: fix more Fatal family functions
# 0.8.1
* logrus: fix not exiting on `Fatalf` and `Fatalln`
# 0.8.0
* logrus: defaults to stderr instead of stdout
* hooks/sentry: add special field for `*http.Request`
* formatter/text: ignore Windows for colors
# 0.7.3
* formatter/\*: allow configuration of timestamp layout
# 0.7.2
* formatter/text: Add configuration option for time format (#158)

View File

@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) 2014 Simon Eskildsen
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@ -1,515 +0,0 @@
# Logrus <img src="http://i.imgur.com/hTeVwmJ.png" width="40" height="40" alt=":walrus:" class="emoji" title=":walrus:"/> [![Build Status](https://github.com/sirupsen/logrus/workflows/CI/badge.svg)](https://github.com/sirupsen/logrus/actions?query=workflow%3ACI) [![Build Status](https://travis-ci.org/sirupsen/logrus.svg?branch=master)](https://travis-ci.org/sirupsen/logrus) [![Go Reference](https://pkg.go.dev/badge/github.com/sirupsen/logrus.svg)](https://pkg.go.dev/github.com/sirupsen/logrus)
Logrus is a structured logger for Go (golang), completely API compatible with
the standard library logger.
**Logrus is in maintenance-mode.** We will not be introducing new features. It's
simply too hard to do in a way that won't break many people's projects, which is
the last thing you want from your Logging library (again...).
This does not mean Logrus is dead. Logrus will continue to be maintained for
security, (backwards compatible) bug fixes, and performance (where we are
limited by the interface).
I believe Logrus' biggest contribution is to have played a part in today's
widespread use of structured logging in Golang. There doesn't seem to be a
reason to do a major, breaking iteration into Logrus V2, since the fantastic Go
community has built those independently. Many fantastic alternatives have sprung
up. Logrus would look like those, had it been re-designed with what we know
about structured logging in Go today. Check out, for example,
[Zerolog][zerolog], [Zap][zap], and [Apex][apex].
[zerolog]: https://github.com/rs/zerolog
[zap]: https://github.com/uber-go/zap
[apex]: https://github.com/apex/log
**Seeing weird case-sensitive problems?** It's in the past been possible to
import Logrus as both upper- and lower-case. Due to the Go package environment,
this caused issues in the community and we needed a standard. Some environments
experienced problems with the upper-case variant, so the lower-case was decided.
Everything using `logrus` will need to use the lower-case:
`github.com/sirupsen/logrus`. Any package that isn't, should be changed.
To fix Glide, see [these
comments](https://github.com/sirupsen/logrus/issues/553#issuecomment-306591437).
For an in-depth explanation of the casing issue, see [this
comment](https://github.com/sirupsen/logrus/issues/570#issuecomment-313933276).
Nicely color-coded in development (when a TTY is attached, otherwise just
plain text):
![Colored](http://i.imgur.com/PY7qMwd.png)
With `log.SetFormatter(&log.JSONFormatter{})`, for easy parsing by logstash
or Splunk:
```text
{"animal":"walrus","level":"info","msg":"A group of walrus emerges from the
ocean","size":10,"time":"2014-03-10 19:57:38.562264131 -0400 EDT"}
{"level":"warning","msg":"The group's number increased tremendously!",
"number":122,"omg":true,"time":"2014-03-10 19:57:38.562471297 -0400 EDT"}
{"animal":"walrus","level":"info","msg":"A giant walrus appears!",
"size":10,"time":"2014-03-10 19:57:38.562500591 -0400 EDT"}
{"animal":"walrus","level":"info","msg":"Tremendously sized cow enters the ocean.",
"size":9,"time":"2014-03-10 19:57:38.562527896 -0400 EDT"}
{"level":"fatal","msg":"The ice breaks!","number":100,"omg":true,
"time":"2014-03-10 19:57:38.562543128 -0400 EDT"}
```
With the default `log.SetFormatter(&log.TextFormatter{})` when a TTY is not
attached, the output is compatible with the
[logfmt](http://godoc.org/github.com/kr/logfmt) format:
```text
time="2015-03-26T01:27:38-04:00" level=debug msg="Started observing beach" animal=walrus number=8
time="2015-03-26T01:27:38-04:00" level=info msg="A group of walrus emerges from the ocean" animal=walrus size=10
time="2015-03-26T01:27:38-04:00" level=warning msg="The group's number increased tremendously!" number=122 omg=true
time="2015-03-26T01:27:38-04:00" level=debug msg="Temperature changes" temperature=-4
time="2015-03-26T01:27:38-04:00" level=panic msg="It's over 9000!" animal=orca size=9009
time="2015-03-26T01:27:38-04:00" level=fatal msg="The ice breaks!" err=&{0x2082280c0 map[animal:orca size:9009] 2015-03-26 01:27:38.441574009 -0400 EDT panic It's over 9000!} number=100 omg=true
```
To ensure this behaviour even if a TTY is attached, set your formatter as follows:
```go
log.SetFormatter(&log.TextFormatter{
DisableColors: true,
FullTimestamp: true,
})
```
#### Logging Method Name
If you wish to add the calling method as a field, instruct the logger via:
```go
log.SetReportCaller(true)
```
This adds the caller as 'method' like so:
```json
{"animal":"penguin","level":"fatal","method":"github.com/sirupsen/arcticcreatures.migrate","msg":"a penguin swims by",
"time":"2014-03-10 19:57:38.562543129 -0400 EDT"}
```
```text
time="2015-03-26T01:27:38-04:00" level=fatal method=github.com/sirupsen/arcticcreatures.migrate msg="a penguin swims by" animal=penguin
```
Note that this does add measurable overhead - the cost will depend on the version of Go, but is
between 20 and 40% in recent tests with 1.6 and 1.7. You can validate this in your
environment via benchmarks:
```
go test -bench=.*CallerTracing
```
#### Case-sensitivity
The organization's name was changed to lower-case--and this will not be changed
back. If you are getting import conflicts due to case sensitivity, please use
the lower-case import: `github.com/sirupsen/logrus`.
#### Example
The simplest way to use Logrus is simply the package-level exported logger:
```go
package main
import (
log "github.com/sirupsen/logrus"
)
func main() {
log.WithFields(log.Fields{
"animal": "walrus",
}).Info("A walrus appears")
}
```
Note that it's completely api-compatible with the stdlib logger, so you can
replace your `log` imports everywhere with `log "github.com/sirupsen/logrus"`
and you'll now have the flexibility of Logrus. You can customize it all you
want:
```go
package main
import (
"os"
log "github.com/sirupsen/logrus"
)
func init() {
// Log as JSON instead of the default ASCII formatter.
log.SetFormatter(&log.JSONFormatter{})
// Output to stdout instead of the default stderr
// Can be any io.Writer, see below for File example
log.SetOutput(os.Stdout)
// Only log the warning severity or above.
log.SetLevel(log.WarnLevel)
}
func main() {
log.WithFields(log.Fields{
"animal": "walrus",
"size": 10,
}).Info("A group of walrus emerges from the ocean")
log.WithFields(log.Fields{
"omg": true,
"number": 122,
}).Warn("The group's number increased tremendously!")
log.WithFields(log.Fields{
"omg": true,
"number": 100,
}).Fatal("The ice breaks!")
// A common pattern is to re-use fields between logging statements by re-using
// the logrus.Entry returned from WithFields()
contextLogger := log.WithFields(log.Fields{
"common": "this is a common field",
"other": "I also should be logged always",
})
contextLogger.Info("I'll be logged with common and other field")
contextLogger.Info("Me too")
}
```
For more advanced usage such as logging to multiple locations from the same
application, you can also create an instance of the `logrus` Logger:
```go
package main
import (
"os"
"github.com/sirupsen/logrus"
)
// Create a new instance of the logger. You can have any number of instances.
var log = logrus.New()
func main() {
// The API for setting attributes is a little different than the package level
// exported logger. See Godoc.
log.Out = os.Stdout
// You could set this to any `io.Writer` such as a file
// file, err := os.OpenFile("logrus.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
// if err == nil {
// log.Out = file
// } else {
// log.Info("Failed to log to file, using default stderr")
// }
log.WithFields(logrus.Fields{
"animal": "walrus",
"size": 10,
}).Info("A group of walrus emerges from the ocean")
}
```
#### Fields
Logrus encourages careful, structured logging through logging fields instead of
long, unparseable error messages. For example, instead of: `log.Fatalf("Failed
to send event %s to topic %s with key %d")`, you should log the much more
discoverable:
```go
log.WithFields(log.Fields{
"event": event,
"topic": topic,
"key": key,
}).Fatal("Failed to send event")
```
We've found this API forces you to think about logging in a way that produces
much more useful logging messages. We've been in countless situations where just
a single added field to a log statement that was already there would've saved us
hours. The `WithFields` call is optional.
In general, with Logrus using any of the `printf`-family functions should be
seen as a hint you should add a field, however, you can still use the
`printf`-family functions with Logrus.
#### Default Fields
Often it's helpful to have fields _always_ attached to log statements in an
application or parts of one. For example, you may want to always log the
`request_id` and `user_ip` in the context of a request. Instead of writing
`log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip})` on
every line, you can create a `logrus.Entry` to pass around instead:
```go
requestLogger := log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip})
requestLogger.Info("something happened on that request") # will log request_id and user_ip
requestLogger.Warn("something not great happened")
```
#### Hooks
You can add hooks for logging levels. For example to send errors to an exception
tracking service on `Error`, `Fatal` and `Panic`, info to StatsD or log to
multiple places simultaneously, e.g. syslog.
Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in
`init`:
```go
import (
log "github.com/sirupsen/logrus"
"gopkg.in/gemnasium/logrus-airbrake-hook.v2" // the package is named "airbrake"
logrus_syslog "github.com/sirupsen/logrus/hooks/syslog"
"log/syslog"
)
func init() {
// Use the Airbrake hook to report errors that have Error severity or above to
// an exception tracker. You can create custom hooks, see the Hooks section.
log.AddHook(airbrake.NewHook(123, "xyz", "production"))
hook, err := logrus_syslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_INFO, "")
if err != nil {
log.Error("Unable to connect to local syslog daemon")
} else {
log.AddHook(hook)
}
}
```
Note: Syslog hook also support connecting to local syslog (Ex. "/dev/log" or "/var/run/syslog" or "/var/run/log"). For the detail, please check the [syslog hook README](hooks/syslog/README.md).
A list of currently known service hooks can be found in this wiki [page](https://github.com/sirupsen/logrus/wiki/Hooks)
#### Level logging
Logrus has seven logging levels: Trace, Debug, Info, Warning, Error, Fatal and Panic.
```go
log.Trace("Something very low level.")
log.Debug("Useful debugging information.")
log.Info("Something noteworthy happened!")
log.Warn("You should probably take a look at this.")
log.Error("Something failed but I'm not quitting.")
// Calls os.Exit(1) after logging
log.Fatal("Bye.")
// Calls panic() after logging
log.Panic("I'm bailing.")
```
You can set the logging level on a `Logger`, then it will only log entries with
that severity or anything above it:
```go
// Will log anything that is info or above (warn, error, fatal, panic). Default.
log.SetLevel(log.InfoLevel)
```
It may be useful to set `log.Level = logrus.DebugLevel` in a debug or verbose
environment if your application has that.
Note: If you want different log levels for global (`log.SetLevel(...)`) and syslog logging, please check the [syslog hook README](hooks/syslog/README.md#different-log-levels-for-local-and-remote-logging).
#### Entries
Besides the fields added with `WithField` or `WithFields` some fields are
automatically added to all logging events:
1. `time`. The timestamp when the entry was created.
2. `msg`. The logging message passed to `{Info,Warn,Error,Fatal,Panic}` after
the `AddFields` call. E.g. `Failed to send event.`
3. `level`. The logging level. E.g. `info`.
#### Environments
Logrus has no notion of environment.
If you wish for hooks and formatters to only be used in specific environments,
you should handle that yourself. For example, if your application has a global
variable `Environment`, which is a string representation of the environment you
could do:
```go
import (
log "github.com/sirupsen/logrus"
)
func init() {
// do something here to set environment depending on an environment variable
// or command-line flag
if Environment == "production" {
log.SetFormatter(&log.JSONFormatter{})
} else {
// The TextFormatter is default, you don't actually have to do this.
log.SetFormatter(&log.TextFormatter{})
}
}
```
This configuration is how `logrus` was intended to be used, but JSON in
production is mostly only useful if you do log aggregation with tools like
Splunk or Logstash.
#### Formatters
The built-in logging formatters are:
* `logrus.TextFormatter`. Logs the event in colors if stdout is a tty, otherwise
without colors.
* *Note:* to force colored output when there is no TTY, set the `ForceColors`
field to `true`. To force no colored output even if there is a TTY set the
`DisableColors` field to `true`. For Windows, see
[github.com/mattn/go-colorable](https://github.com/mattn/go-colorable).
* When colors are enabled, levels are truncated to 4 characters by default. To disable
truncation set the `DisableLevelTruncation` field to `true`.
* When outputting to a TTY, it's often helpful to visually scan down a column where all the levels are the same width. Setting the `PadLevelText` field to `true` enables this behavior, by adding padding to the level text.
* All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#TextFormatter).
* `logrus.JSONFormatter`. Logs fields as JSON.
* All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#JSONFormatter).
Third party logging formatters:
* [`FluentdFormatter`](https://github.com/joonix/log). Formats entries that can be parsed by Kubernetes and Google Container Engine.
* [`GELF`](https://github.com/fabienm/go-logrus-formatters). Formats entries so they comply to Graylog's [GELF 1.1 specification](http://docs.graylog.org/en/2.4/pages/gelf.html).
* [`logstash`](https://github.com/bshuster-repo/logrus-logstash-hook). Logs fields as [Logstash](http://logstash.net) Events.
* [`prefixed`](https://github.com/x-cray/logrus-prefixed-formatter). Displays log entry source along with alternative layout.
* [`zalgo`](https://github.com/aybabtme/logzalgo). Invoking the Power of Zalgo.
* [`nested-logrus-formatter`](https://github.com/antonfisher/nested-logrus-formatter). Converts logrus fields to a nested structure.
* [`powerful-logrus-formatter`](https://github.com/zput/zxcTool). get fileName, log's line number and the latest function's name when print log; Sava log to files.
* [`caption-json-formatter`](https://github.com/nolleh/caption_json_formatter). logrus's message json formatter with human-readable caption added.
You can define your formatter by implementing the `Formatter` interface,
requiring a `Format` method. `Format` takes an `*Entry`. `entry.Data` is a
`Fields` type (`map[string]interface{}`) with all your fields as well as the
default ones (see Entries section above):
```go
type MyJSONFormatter struct {
}
log.SetFormatter(new(MyJSONFormatter))
func (f *MyJSONFormatter) Format(entry *Entry) ([]byte, error) {
// Note this doesn't include Time, Level and Message which are available on
// the Entry. Consult `godoc` on information about those fields or read the
// source of the official loggers.
serialized, err := json.Marshal(entry.Data)
if err != nil {
return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err)
}
return append(serialized, '\n'), nil
}
```
#### Logger as an `io.Writer`
Logrus can be transformed into an `io.Writer`. That writer is the end of an `io.Pipe` and it is your responsibility to close it.
```go
w := logger.Writer()
defer w.Close()
srv := http.Server{
// create a stdlib log.Logger that writes to
// logrus.Logger.
ErrorLog: log.New(w, "", 0),
}
```
Each line written to that writer will be printed the usual way, using formatters
and hooks. The level for those entries is `info`.
This means that we can override the standard library logger easily:
```go
logger := logrus.New()
logger.Formatter = &logrus.JSONFormatter{}
// Use logrus for standard log output
// Note that `log` here references stdlib's log
// Not logrus imported under the name `log`.
log.SetOutput(logger.Writer())
```
#### Rotation
Log rotation is not provided with Logrus. Log rotation should be done by an
external program (like `logrotate(8)`) that can compress and delete old log
entries. It should not be a feature of the application-level logger.
#### Tools
| Tool | Description |
| ---- | ----------- |
|[Logrus Mate](https://github.com/gogap/logrus_mate)|Logrus mate is a tool for Logrus to manage loggers, you can initial logger's level, hook and formatter by config file, the logger will be generated with different configs in different environments.|
|[Logrus Viper Helper](https://github.com/heirko/go-contrib/tree/master/logrusHelper)|An Helper around Logrus to wrap with spf13/Viper to load configuration with fangs! And to simplify Logrus configuration use some behavior of [Logrus Mate](https://github.com/gogap/logrus_mate). [sample](https://github.com/heirko/iris-contrib/blob/master/middleware/logrus-logger/example) |
#### Testing
Logrus has a built in facility for asserting the presence of log messages. This is implemented through the `test` hook and provides:
* decorators for existing logger (`test.NewLocal` and `test.NewGlobal`) which basically just adds the `test` hook
* a test logger (`test.NewNullLogger`) that just records log messages (and does not output any):
```go
import(
"github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"testing"
)
func TestSomething(t*testing.T){
logger, hook := test.NewNullLogger()
logger.Error("Helloerror")
assert.Equal(t, 1, len(hook.Entries))
assert.Equal(t, logrus.ErrorLevel, hook.LastEntry().Level)
assert.Equal(t, "Helloerror", hook.LastEntry().Message)
hook.Reset()
assert.Nil(t, hook.LastEntry())
}
```
#### Fatal handlers
Logrus can register one or more functions that will be called when any `fatal`
level message is logged. The registered handlers will be executed before
logrus performs an `os.Exit(1)`. This behavior may be helpful if callers need
to gracefully shutdown. Unlike a `panic("Something went wrong...")` call which can be intercepted with a deferred `recover` a call to `os.Exit(1)` can not be intercepted.
```
...
handler := func() {
// gracefully shutdown something...
}
logrus.RegisterExitHandler(handler)
...
```
#### Thread safety
By default, Logger is protected by a mutex for concurrent writes. The mutex is held when calling hooks and writing logs.
If you are sure such locking is not needed, you can call logger.SetNoLock() to disable the locking.
Situation when locking is not needed includes:
* You have no hooks registered, or hooks calling is already thread-safe.
* Writing to logger.Out is already thread-safe, for example:
1) logger.Out is protected by locks.
2) logger.Out is an os.File handler opened with `O_APPEND` flag, and every write is smaller than 4k. (This allows multi-thread/multi-process writing)
(Refer to http://www.notthewizard.com/2014/06/17/are-files-appends-really-atomic/)

View File

@ -1,76 +0,0 @@
package logrus
// The following code was sourced and modified from the
// https://github.com/tebeka/atexit package governed by the following license:
//
// Copyright (c) 2012 Miki Tebeka <miki.tebeka@gmail.com>.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import (
"fmt"
"os"
)
var handlers = []func(){}
func runHandler(handler func()) {
defer func() {
if err := recover(); err != nil {
fmt.Fprintln(os.Stderr, "Error: Logrus exit handler error:", err)
}
}()
handler()
}
func runHandlers() {
for _, handler := range handlers {
runHandler(handler)
}
}
// Exit runs all the Logrus atexit handlers and then terminates the program using os.Exit(code)
func Exit(code int) {
runHandlers()
os.Exit(code)
}
// RegisterExitHandler appends a Logrus Exit handler to the list of handlers,
// call logrus.Exit to invoke all handlers. The handlers will also be invoked when
// any Fatal log entry is made.
//
// This method is useful when a caller wishes to use logrus to log a fatal
// message but also needs to gracefully shutdown. An example usecase could be
// closing database connections, or sending a alert that the application is
// closing.
func RegisterExitHandler(handler func()) {
handlers = append(handlers, handler)
}
// DeferExitHandler prepends a Logrus Exit handler to the list of handlers,
// call logrus.Exit to invoke all handlers. The handlers will also be invoked when
// any Fatal log entry is made.
//
// This method is useful when a caller wishes to use logrus to log a fatal
// message but also needs to gracefully shutdown. An example usecase could be
// closing database connections, or sending a alert that the application is
// closing.
func DeferExitHandler(handler func()) {
handlers = append([]func(){handler}, handlers...)
}

View File

@ -1,14 +0,0 @@
version: "{build}"
platform: x64
clone_folder: c:\gopath\src\github.com\sirupsen\logrus
environment:
GOPATH: c:\gopath
branches:
only:
- master
install:
- set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
- go version
build_script:
- go get -t
- go test

View File

@ -1,43 +0,0 @@
package logrus
import (
"bytes"
"sync"
)
var (
bufferPool BufferPool
)
type BufferPool interface {
Put(*bytes.Buffer)
Get() *bytes.Buffer
}
type defaultPool struct {
pool *sync.Pool
}
func (p *defaultPool) Put(buf *bytes.Buffer) {
p.pool.Put(buf)
}
func (p *defaultPool) Get() *bytes.Buffer {
return p.pool.Get().(*bytes.Buffer)
}
// SetBufferPool allows to replace the default logrus buffer pool
// to better meets the specific needs of an application.
func SetBufferPool(bp BufferPool) {
bufferPool = bp
}
func init() {
SetBufferPool(&defaultPool{
pool: &sync.Pool{
New: func() interface{} {
return new(bytes.Buffer)
},
},
})
}

View File

@ -1,26 +0,0 @@
/*
Package logrus is a structured logger for Go, completely API compatible with the standard library logger.
The simplest way to use Logrus is simply the package-level exported logger:
package main
import (
log "github.com/sirupsen/logrus"
)
func main() {
log.WithFields(log.Fields{
"animal": "walrus",
"number": 1,
"size": 10,
}).Info("A walrus appears")
}
Output:
time="2015-09-07T08:48:33Z" level=info msg="A walrus appears" animal=walrus number=1 size=10
For a full guide visit https://github.com/sirupsen/logrus
*/
package logrus

View File

@ -1,442 +0,0 @@
package logrus
import (
"bytes"
"context"
"fmt"
"os"
"reflect"
"runtime"
"strings"
"sync"
"time"
)
var (
// qualified package name, cached at first use
logrusPackage string
// Positions in the call stack when tracing to report the calling method
minimumCallerDepth int
// Used for caller information initialisation
callerInitOnce sync.Once
)
const (
maximumCallerDepth int = 25
knownLogrusFrames int = 4
)
func init() {
// start at the bottom of the stack before the package-name cache is primed
minimumCallerDepth = 1
}
// Defines the key when adding errors using WithError.
var ErrorKey = "error"
// An entry is the final or intermediate Logrus logging entry. It contains all
// the fields passed with WithField{,s}. It's finally logged when Trace, Debug,
// Info, Warn, Error, Fatal or Panic is called on it. These objects can be
// reused and passed around as much as you wish to avoid field duplication.
type Entry struct {
Logger *Logger
// Contains all the fields set by the user.
Data Fields
// Time at which the log entry was created
Time time.Time
// Level the log entry was logged at: Trace, Debug, Info, Warn, Error, Fatal or Panic
// This field will be set on entry firing and the value will be equal to the one in Logger struct field.
Level Level
// Calling method, with package name
Caller *runtime.Frame
// Message passed to Trace, Debug, Info, Warn, Error, Fatal or Panic
Message string
// When formatter is called in entry.log(), a Buffer may be set to entry
Buffer *bytes.Buffer
// Contains the context set by the user. Useful for hook processing etc.
Context context.Context
// err may contain a field formatting error
err string
}
func NewEntry(logger *Logger) *Entry {
return &Entry{
Logger: logger,
// Default is three fields, plus one optional. Give a little extra room.
Data: make(Fields, 6),
}
}
func (entry *Entry) Dup() *Entry {
data := make(Fields, len(entry.Data))
for k, v := range entry.Data {
data[k] = v
}
return &Entry{Logger: entry.Logger, Data: data, Time: entry.Time, Context: entry.Context, err: entry.err}
}
// Returns the bytes representation of this entry from the formatter.
func (entry *Entry) Bytes() ([]byte, error) {
return entry.Logger.Formatter.Format(entry)
}
// Returns the string representation from the reader and ultimately the
// formatter.
func (entry *Entry) String() (string, error) {
serialized, err := entry.Bytes()
if err != nil {
return "", err
}
str := string(serialized)
return str, nil
}
// Add an error as single field (using the key defined in ErrorKey) to the Entry.
func (entry *Entry) WithError(err error) *Entry {
return entry.WithField(ErrorKey, err)
}
// Add a context to the Entry.
func (entry *Entry) WithContext(ctx context.Context) *Entry {
dataCopy := make(Fields, len(entry.Data))
for k, v := range entry.Data {
dataCopy[k] = v
}
return &Entry{Logger: entry.Logger, Data: dataCopy, Time: entry.Time, err: entry.err, Context: ctx}
}
// Add a single field to the Entry.
func (entry *Entry) WithField(key string, value interface{}) *Entry {
return entry.WithFields(Fields{key: value})
}
// Add a map of fields to the Entry.
func (entry *Entry) WithFields(fields Fields) *Entry {
data := make(Fields, len(entry.Data)+len(fields))
for k, v := range entry.Data {
data[k] = v
}
fieldErr := entry.err
for k, v := range fields {
isErrField := false
if t := reflect.TypeOf(v); t != nil {
switch {
case t.Kind() == reflect.Func, t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Func:
isErrField = true
}
}
if isErrField {
tmp := fmt.Sprintf("can not add field %q", k)
if fieldErr != "" {
fieldErr = entry.err + ", " + tmp
} else {
fieldErr = tmp
}
} else {
data[k] = v
}
}
return &Entry{Logger: entry.Logger, Data: data, Time: entry.Time, err: fieldErr, Context: entry.Context}
}
// Overrides the time of the Entry.
func (entry *Entry) WithTime(t time.Time) *Entry {
dataCopy := make(Fields, len(entry.Data))
for k, v := range entry.Data {
dataCopy[k] = v
}
return &Entry{Logger: entry.Logger, Data: dataCopy, Time: t, err: entry.err, Context: entry.Context}
}
// getPackageName reduces a fully qualified function name to the package name
// There really ought to be to be a better way...
func getPackageName(f string) string {
for {
lastPeriod := strings.LastIndex(f, ".")
lastSlash := strings.LastIndex(f, "/")
if lastPeriod > lastSlash {
f = f[:lastPeriod]
} else {
break
}
}
return f
}
// getCaller retrieves the name of the first non-logrus calling function
func getCaller() *runtime.Frame {
// cache this package's fully-qualified name
callerInitOnce.Do(func() {
pcs := make([]uintptr, maximumCallerDepth)
_ = runtime.Callers(0, pcs)
// dynamic get the package name and the minimum caller depth
for i := 0; i < maximumCallerDepth; i++ {
funcName := runtime.FuncForPC(pcs[i]).Name()
if strings.Contains(funcName, "getCaller") {
logrusPackage = getPackageName(funcName)
break
}
}
minimumCallerDepth = knownLogrusFrames
})
// Restrict the lookback frames to avoid runaway lookups
pcs := make([]uintptr, maximumCallerDepth)
depth := runtime.Callers(minimumCallerDepth, pcs)
frames := runtime.CallersFrames(pcs[:depth])
for f, again := frames.Next(); again; f, again = frames.Next() {
pkg := getPackageName(f.Function)
// If the caller isn't part of this package, we're done
if pkg != logrusPackage {
return &f //nolint:scopelint
}
}
// if we got here, we failed to find the caller's context
return nil
}
func (entry Entry) HasCaller() (has bool) {
return entry.Logger != nil &&
entry.Logger.ReportCaller &&
entry.Caller != nil
}
func (entry *Entry) log(level Level, msg string) {
var buffer *bytes.Buffer
newEntry := entry.Dup()
if newEntry.Time.IsZero() {
newEntry.Time = time.Now()
}
newEntry.Level = level
newEntry.Message = msg
newEntry.Logger.mu.Lock()
reportCaller := newEntry.Logger.ReportCaller
bufPool := newEntry.getBufferPool()
newEntry.Logger.mu.Unlock()
if reportCaller {
newEntry.Caller = getCaller()
}
newEntry.fireHooks()
buffer = bufPool.Get()
defer func() {
newEntry.Buffer = nil
buffer.Reset()
bufPool.Put(buffer)
}()
buffer.Reset()
newEntry.Buffer = buffer
newEntry.write()
newEntry.Buffer = nil
// To avoid Entry#log() returning a value that only would make sense for
// panic() to use in Entry#Panic(), we avoid the allocation by checking
// directly here.
if level <= PanicLevel {
panic(newEntry)
}
}
func (entry *Entry) getBufferPool() (pool BufferPool) {
if entry.Logger.BufferPool != nil {
return entry.Logger.BufferPool
}
return bufferPool
}
func (entry *Entry) fireHooks() {
var tmpHooks LevelHooks
entry.Logger.mu.Lock()
tmpHooks = make(LevelHooks, len(entry.Logger.Hooks))
for k, v := range entry.Logger.Hooks {
tmpHooks[k] = v
}
entry.Logger.mu.Unlock()
err := tmpHooks.Fire(entry.Level, entry)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to fire hook: %v\n", err)
}
}
func (entry *Entry) write() {
entry.Logger.mu.Lock()
defer entry.Logger.mu.Unlock()
serialized, err := entry.Logger.Formatter.Format(entry)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to obtain reader, %v\n", err)
return
}
if _, err := entry.Logger.Out.Write(serialized); err != nil {
fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err)
}
}
// Log will log a message at the level given as parameter.
// Warning: using Log at Panic or Fatal level will not respectively Panic nor Exit.
// For this behaviour Entry.Panic or Entry.Fatal should be used instead.
func (entry *Entry) Log(level Level, args ...interface{}) {
if entry.Logger.IsLevelEnabled(level) {
entry.log(level, fmt.Sprint(args...))
}
}
func (entry *Entry) Trace(args ...interface{}) {
entry.Log(TraceLevel, args...)
}
func (entry *Entry) Debug(args ...interface{}) {
entry.Log(DebugLevel, args...)
}
func (entry *Entry) Print(args ...interface{}) {
entry.Info(args...)
}
func (entry *Entry) Info(args ...interface{}) {
entry.Log(InfoLevel, args...)
}
func (entry *Entry) Warn(args ...interface{}) {
entry.Log(WarnLevel, args...)
}
func (entry *Entry) Warning(args ...interface{}) {
entry.Warn(args...)
}
func (entry *Entry) Error(args ...interface{}) {
entry.Log(ErrorLevel, args...)
}
func (entry *Entry) Fatal(args ...interface{}) {
entry.Log(FatalLevel, args...)
entry.Logger.Exit(1)
}
func (entry *Entry) Panic(args ...interface{}) {
entry.Log(PanicLevel, args...)
}
// Entry Printf family functions
func (entry *Entry) Logf(level Level, format string, args ...interface{}) {
if entry.Logger.IsLevelEnabled(level) {
entry.Log(level, fmt.Sprintf(format, args...))
}
}
func (entry *Entry) Tracef(format string, args ...interface{}) {
entry.Logf(TraceLevel, format, args...)
}
func (entry *Entry) Debugf(format string, args ...interface{}) {
entry.Logf(DebugLevel, format, args...)
}
func (entry *Entry) Infof(format string, args ...interface{}) {
entry.Logf(InfoLevel, format, args...)
}
func (entry *Entry) Printf(format string, args ...interface{}) {
entry.Infof(format, args...)
}
func (entry *Entry) Warnf(format string, args ...interface{}) {
entry.Logf(WarnLevel, format, args...)
}
func (entry *Entry) Warningf(format string, args ...interface{}) {
entry.Warnf(format, args...)
}
func (entry *Entry) Errorf(format string, args ...interface{}) {
entry.Logf(ErrorLevel, format, args...)
}
func (entry *Entry) Fatalf(format string, args ...interface{}) {
entry.Logf(FatalLevel, format, args...)
entry.Logger.Exit(1)
}
func (entry *Entry) Panicf(format string, args ...interface{}) {
entry.Logf(PanicLevel, format, args...)
}
// Entry Println family functions
func (entry *Entry) Logln(level Level, args ...interface{}) {
if entry.Logger.IsLevelEnabled(level) {
entry.Log(level, entry.sprintlnn(args...))
}
}
func (entry *Entry) Traceln(args ...interface{}) {
entry.Logln(TraceLevel, args...)
}
func (entry *Entry) Debugln(args ...interface{}) {
entry.Logln(DebugLevel, args...)
}
func (entry *Entry) Infoln(args ...interface{}) {
entry.Logln(InfoLevel, args...)
}
func (entry *Entry) Println(args ...interface{}) {
entry.Infoln(args...)
}
func (entry *Entry) Warnln(args ...interface{}) {
entry.Logln(WarnLevel, args...)
}
func (entry *Entry) Warningln(args ...interface{}) {
entry.Warnln(args...)
}
func (entry *Entry) Errorln(args ...interface{}) {
entry.Logln(ErrorLevel, args...)
}
func (entry *Entry) Fatalln(args ...interface{}) {
entry.Logln(FatalLevel, args...)
entry.Logger.Exit(1)
}
func (entry *Entry) Panicln(args ...interface{}) {
entry.Logln(PanicLevel, args...)
}
// Sprintlnn => Sprint no newline. This is to get the behavior of how
// fmt.Sprintln where spaces are always added between operands, regardless of
// their type. Instead of vendoring the Sprintln implementation to spare a
// string allocation, we do the simplest thing.
func (entry *Entry) sprintlnn(args ...interface{}) string {
msg := fmt.Sprintln(args...)
return msg[:len(msg)-1]
}

View File

@ -1,270 +0,0 @@
package logrus
import (
"context"
"io"
"time"
)
var (
// std is the name of the standard logger in stdlib `log`
std = New()
)
func StandardLogger() *Logger {
return std
}
// SetOutput sets the standard logger output.
func SetOutput(out io.Writer) {
std.SetOutput(out)
}
// SetFormatter sets the standard logger formatter.
func SetFormatter(formatter Formatter) {
std.SetFormatter(formatter)
}
// SetReportCaller sets whether the standard logger will include the calling
// method as a field.
func SetReportCaller(include bool) {
std.SetReportCaller(include)
}
// SetLevel sets the standard logger level.
func SetLevel(level Level) {
std.SetLevel(level)
}
// GetLevel returns the standard logger level.
func GetLevel() Level {
return std.GetLevel()
}
// IsLevelEnabled checks if the log level of the standard logger is greater than the level param
func IsLevelEnabled(level Level) bool {
return std.IsLevelEnabled(level)
}
// AddHook adds a hook to the standard logger hooks.
func AddHook(hook Hook) {
std.AddHook(hook)
}
// WithError creates an entry from the standard logger and adds an error to it, using the value defined in ErrorKey as key.
func WithError(err error) *Entry {
return std.WithField(ErrorKey, err)
}
// WithContext creates an entry from the standard logger and adds a context to it.
func WithContext(ctx context.Context) *Entry {
return std.WithContext(ctx)
}
// WithField creates an entry from the standard logger and adds a field to
// it. If you want multiple fields, use `WithFields`.
//
// Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal
// or Panic on the Entry it returns.
func WithField(key string, value interface{}) *Entry {
return std.WithField(key, value)
}
// WithFields creates an entry from the standard logger and adds multiple
// fields to it. This is simply a helper for `WithField`, invoking it
// once for each field.
//
// Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal
// or Panic on the Entry it returns.
func WithFields(fields Fields) *Entry {
return std.WithFields(fields)
}
// WithTime creates an entry from the standard logger and overrides the time of
// logs generated with it.
//
// Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal
// or Panic on the Entry it returns.
func WithTime(t time.Time) *Entry {
return std.WithTime(t)
}
// Trace logs a message at level Trace on the standard logger.
func Trace(args ...interface{}) {
std.Trace(args...)
}
// Debug logs a message at level Debug on the standard logger.
func Debug(args ...interface{}) {
std.Debug(args...)
}
// Print logs a message at level Info on the standard logger.
func Print(args ...interface{}) {
std.Print(args...)
}
// Info logs a message at level Info on the standard logger.
func Info(args ...interface{}) {
std.Info(args...)
}
// Warn logs a message at level Warn on the standard logger.
func Warn(args ...interface{}) {
std.Warn(args...)
}
// Warning logs a message at level Warn on the standard logger.
func Warning(args ...interface{}) {
std.Warning(args...)
}
// Error logs a message at level Error on the standard logger.
func Error(args ...interface{}) {
std.Error(args...)
}
// Panic logs a message at level Panic on the standard logger.
func Panic(args ...interface{}) {
std.Panic(args...)
}
// Fatal logs a message at level Fatal on the standard logger then the process will exit with status set to 1.
func Fatal(args ...interface{}) {
std.Fatal(args...)
}
// TraceFn logs a message from a func at level Trace on the standard logger.
func TraceFn(fn LogFunction) {
std.TraceFn(fn)
}
// DebugFn logs a message from a func at level Debug on the standard logger.
func DebugFn(fn LogFunction) {
std.DebugFn(fn)
}
// PrintFn logs a message from a func at level Info on the standard logger.
func PrintFn(fn LogFunction) {
std.PrintFn(fn)
}
// InfoFn logs a message from a func at level Info on the standard logger.
func InfoFn(fn LogFunction) {
std.InfoFn(fn)
}
// WarnFn logs a message from a func at level Warn on the standard logger.
func WarnFn(fn LogFunction) {
std.WarnFn(fn)
}
// WarningFn logs a message from a func at level Warn on the standard logger.
func WarningFn(fn LogFunction) {
std.WarningFn(fn)
}
// ErrorFn logs a message from a func at level Error on the standard logger.
func ErrorFn(fn LogFunction) {
std.ErrorFn(fn)
}
// PanicFn logs a message from a func at level Panic on the standard logger.
func PanicFn(fn LogFunction) {
std.PanicFn(fn)
}
// FatalFn logs a message from a func at level Fatal on the standard logger then the process will exit with status set to 1.
func FatalFn(fn LogFunction) {
std.FatalFn(fn)
}
// Tracef logs a message at level Trace on the standard logger.
func Tracef(format string, args ...interface{}) {
std.Tracef(format, args...)
}
// Debugf logs a message at level Debug on the standard logger.
func Debugf(format string, args ...interface{}) {
std.Debugf(format, args...)
}
// Printf logs a message at level Info on the standard logger.
func Printf(format string, args ...interface{}) {
std.Printf(format, args...)
}
// Infof logs a message at level Info on the standard logger.
func Infof(format string, args ...interface{}) {
std.Infof(format, args...)
}
// Warnf logs a message at level Warn on the standard logger.
func Warnf(format string, args ...interface{}) {
std.Warnf(format, args...)
}
// Warningf logs a message at level Warn on the standard logger.
func Warningf(format string, args ...interface{}) {
std.Warningf(format, args...)
}
// Errorf logs a message at level Error on the standard logger.
func Errorf(format string, args ...interface{}) {
std.Errorf(format, args...)
}
// Panicf logs a message at level Panic on the standard logger.
func Panicf(format string, args ...interface{}) {
std.Panicf(format, args...)
}
// Fatalf logs a message at level Fatal on the standard logger then the process will exit with status set to 1.
func Fatalf(format string, args ...interface{}) {
std.Fatalf(format, args...)
}
// Traceln logs a message at level Trace on the standard logger.
func Traceln(args ...interface{}) {
std.Traceln(args...)
}
// Debugln logs a message at level Debug on the standard logger.
func Debugln(args ...interface{}) {
std.Debugln(args...)
}
// Println logs a message at level Info on the standard logger.
func Println(args ...interface{}) {
std.Println(args...)
}
// Infoln logs a message at level Info on the standard logger.
func Infoln(args ...interface{}) {
std.Infoln(args...)
}
// Warnln logs a message at level Warn on the standard logger.
func Warnln(args ...interface{}) {
std.Warnln(args...)
}
// Warningln logs a message at level Warn on the standard logger.
func Warningln(args ...interface{}) {
std.Warningln(args...)
}
// Errorln logs a message at level Error on the standard logger.
func Errorln(args ...interface{}) {
std.Errorln(args...)
}
// Panicln logs a message at level Panic on the standard logger.
func Panicln(args ...interface{}) {
std.Panicln(args...)
}
// Fatalln logs a message at level Fatal on the standard logger then the process will exit with status set to 1.
func Fatalln(args ...interface{}) {
std.Fatalln(args...)
}

View File

@ -1,78 +0,0 @@
package logrus
import "time"
// Default key names for the default fields
const (
defaultTimestampFormat = time.RFC3339
FieldKeyMsg = "msg"
FieldKeyLevel = "level"
FieldKeyTime = "time"
FieldKeyLogrusError = "logrus_error"
FieldKeyFunc = "func"
FieldKeyFile = "file"
)
// The Formatter interface is used to implement a custom Formatter. It takes an
// `Entry`. It exposes all the fields, including the default ones:
//
// * `entry.Data["msg"]`. The message passed from Info, Warn, Error ..
// * `entry.Data["time"]`. The timestamp.
// * `entry.Data["level"]. The level the entry was logged at.
//
// Any additional fields added with `WithField` or `WithFields` are also in
// `entry.Data`. Format is expected to return an array of bytes which are then
// logged to `logger.Out`.
type Formatter interface {
Format(*Entry) ([]byte, error)
}
// This is to not silently overwrite `time`, `msg`, `func` and `level` fields when
// dumping it. If this code wasn't there doing:
//
// logrus.WithField("level", 1).Info("hello")
//
// Would just silently drop the user provided level. Instead with this code
// it'll logged as:
//
// {"level": "info", "fields.level": 1, "msg": "hello", "time": "..."}
//
// It's not exported because it's still using Data in an opinionated way. It's to
// avoid code duplication between the two default formatters.
func prefixFieldClashes(data Fields, fieldMap FieldMap, reportCaller bool) {
timeKey := fieldMap.resolve(FieldKeyTime)
if t, ok := data[timeKey]; ok {
data["fields."+timeKey] = t
delete(data, timeKey)
}
msgKey := fieldMap.resolve(FieldKeyMsg)
if m, ok := data[msgKey]; ok {
data["fields."+msgKey] = m
delete(data, msgKey)
}
levelKey := fieldMap.resolve(FieldKeyLevel)
if l, ok := data[levelKey]; ok {
data["fields."+levelKey] = l
delete(data, levelKey)
}
logrusErrKey := fieldMap.resolve(FieldKeyLogrusError)
if l, ok := data[logrusErrKey]; ok {
data["fields."+logrusErrKey] = l
delete(data, logrusErrKey)
}
// If reportCaller is not set, 'func' will not conflict.
if reportCaller {
funcKey := fieldMap.resolve(FieldKeyFunc)
if l, ok := data[funcKey]; ok {
data["fields."+funcKey] = l
}
fileKey := fieldMap.resolve(FieldKeyFile)
if l, ok := data[fileKey]; ok {
data["fields."+fileKey] = l
}
}
}

View File

@ -1,34 +0,0 @@
package logrus
// A hook to be fired when logging on the logging levels returned from
// `Levels()` on your implementation of the interface. Note that this is not
// fired in a goroutine or a channel with workers, you should handle such
// functionality yourself if your call is non-blocking and you don't wish for
// the logging calls for levels returned from `Levels()` to block.
type Hook interface {
Levels() []Level
Fire(*Entry) error
}
// Internal type for storing the hooks on a logger instance.
type LevelHooks map[Level][]Hook
// Add a hook to an instance of logger. This is called with
// `log.Hooks.Add(new(MyHook))` where `MyHook` implements the `Hook` interface.
func (hooks LevelHooks) Add(hook Hook) {
for _, level := range hook.Levels() {
hooks[level] = append(hooks[level], hook)
}
}
// Fire all the hooks for the passed level. Used by `entry.log` to fire
// appropriate hooks for a log entry.
func (hooks LevelHooks) Fire(level Level, entry *Entry) error {
for _, hook := range hooks[level] {
if err := hook.Fire(entry); err != nil {
return err
}
}
return nil
}

View File

@ -1,128 +0,0 @@
package logrus
import (
"bytes"
"encoding/json"
"fmt"
"runtime"
)
type fieldKey string
// FieldMap allows customization of the key names for default fields.
type FieldMap map[fieldKey]string
func (f FieldMap) resolve(key fieldKey) string {
if k, ok := f[key]; ok {
return k
}
return string(key)
}
// JSONFormatter formats logs into parsable json
type JSONFormatter struct {
// TimestampFormat sets the format used for marshaling timestamps.
// The format to use is the same than for time.Format or time.Parse from the standard
// library.
// The standard Library already provides a set of predefined format.
TimestampFormat string
// DisableTimestamp allows disabling automatic timestamps in output
DisableTimestamp bool
// DisableHTMLEscape allows disabling html escaping in output
DisableHTMLEscape bool
// DataKey allows users to put all the log entry parameters into a nested dictionary at a given key.
DataKey string
// FieldMap allows users to customize the names of keys for default fields.
// As an example:
// formatter := &JSONFormatter{
// FieldMap: FieldMap{
// FieldKeyTime: "@timestamp",
// FieldKeyLevel: "@level",
// FieldKeyMsg: "@message",
// FieldKeyFunc: "@caller",
// },
// }
FieldMap FieldMap
// CallerPrettyfier can be set by the user to modify the content
// of the function and file keys in the json data when ReportCaller is
// activated. If any of the returned value is the empty string the
// corresponding key will be removed from json fields.
CallerPrettyfier func(*runtime.Frame) (function string, file string)
// PrettyPrint will indent all json logs
PrettyPrint bool
}
// Format renders a single log entry
func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) {
data := make(Fields, len(entry.Data)+4)
for k, v := range entry.Data {
switch v := v.(type) {
case error:
// Otherwise errors are ignored by `encoding/json`
// https://github.com/sirupsen/logrus/issues/137
data[k] = v.Error()
default:
data[k] = v
}
}
if f.DataKey != "" {
newData := make(Fields, 4)
newData[f.DataKey] = data
data = newData
}
prefixFieldClashes(data, f.FieldMap, entry.HasCaller())
timestampFormat := f.TimestampFormat
if timestampFormat == "" {
timestampFormat = defaultTimestampFormat
}
if entry.err != "" {
data[f.FieldMap.resolve(FieldKeyLogrusError)] = entry.err
}
if !f.DisableTimestamp {
data[f.FieldMap.resolve(FieldKeyTime)] = entry.Time.Format(timestampFormat)
}
data[f.FieldMap.resolve(FieldKeyMsg)] = entry.Message
data[f.FieldMap.resolve(FieldKeyLevel)] = entry.Level.String()
if entry.HasCaller() {
funcVal := entry.Caller.Function
fileVal := fmt.Sprintf("%s:%d", entry.Caller.File, entry.Caller.Line)
if f.CallerPrettyfier != nil {
funcVal, fileVal = f.CallerPrettyfier(entry.Caller)
}
if funcVal != "" {
data[f.FieldMap.resolve(FieldKeyFunc)] = funcVal
}
if fileVal != "" {
data[f.FieldMap.resolve(FieldKeyFile)] = fileVal
}
}
var b *bytes.Buffer
if entry.Buffer != nil {
b = entry.Buffer
} else {
b = &bytes.Buffer{}
}
encoder := json.NewEncoder(b)
encoder.SetEscapeHTML(!f.DisableHTMLEscape)
if f.PrettyPrint {
encoder.SetIndent("", " ")
}
if err := encoder.Encode(data); err != nil {
return nil, fmt.Errorf("failed to marshal fields to JSON, %w", err)
}
return b.Bytes(), nil
}

View File

@ -1,417 +0,0 @@
package logrus
import (
"context"
"io"
"os"
"sync"
"sync/atomic"
"time"
)
// LogFunction For big messages, it can be more efficient to pass a function
// and only call it if the log level is actually enables rather than
// generating the log message and then checking if the level is enabled
type LogFunction func() []interface{}
type Logger struct {
// The logs are `io.Copy`'d to this in a mutex. It's common to set this to a
// file, or leave it default which is `os.Stderr`. You can also set this to
// something more adventurous, such as logging to Kafka.
Out io.Writer
// Hooks for the logger instance. These allow firing events based on logging
// levels and log entries. For example, to send errors to an error tracking
// service, log to StatsD or dump the core on fatal errors.
Hooks LevelHooks
// All log entries pass through the formatter before logged to Out. The
// included formatters are `TextFormatter` and `JSONFormatter` for which
// TextFormatter is the default. In development (when a TTY is attached) it
// logs with colors, but to a file it wouldn't. You can easily implement your
// own that implements the `Formatter` interface, see the `README` or included
// formatters for examples.
Formatter Formatter
// Flag for whether to log caller info (off by default)
ReportCaller bool
// The logging level the logger should log at. This is typically (and defaults
// to) `logrus.Info`, which allows Info(), Warn(), Error() and Fatal() to be
// logged.
Level Level
// Used to sync writing to the log. Locking is enabled by Default
mu MutexWrap
// Reusable empty entry
entryPool sync.Pool
// Function to exit the application, defaults to `os.Exit()`
ExitFunc exitFunc
// The buffer pool used to format the log. If it is nil, the default global
// buffer pool will be used.
BufferPool BufferPool
}
type exitFunc func(int)
type MutexWrap struct {
lock sync.Mutex
disabled bool
}
func (mw *MutexWrap) Lock() {
if !mw.disabled {
mw.lock.Lock()
}
}
func (mw *MutexWrap) Unlock() {
if !mw.disabled {
mw.lock.Unlock()
}
}
func (mw *MutexWrap) Disable() {
mw.disabled = true
}
// Creates a new logger. Configuration should be set by changing `Formatter`,
// `Out` and `Hooks` directly on the default logger instance. You can also just
// instantiate your own:
//
// var log = &logrus.Logger{
// Out: os.Stderr,
// Formatter: new(logrus.TextFormatter),
// Hooks: make(logrus.LevelHooks),
// Level: logrus.DebugLevel,
// }
//
// It's recommended to make this a global instance called `log`.
func New() *Logger {
return &Logger{
Out: os.Stderr,
Formatter: new(TextFormatter),
Hooks: make(LevelHooks),
Level: InfoLevel,
ExitFunc: os.Exit,
ReportCaller: false,
}
}
func (logger *Logger) newEntry() *Entry {
entry, ok := logger.entryPool.Get().(*Entry)
if ok {
return entry
}
return NewEntry(logger)
}
func (logger *Logger) releaseEntry(entry *Entry) {
entry.Data = map[string]interface{}{}
logger.entryPool.Put(entry)
}
// WithField allocates a new entry and adds a field to it.
// Debug, Print, Info, Warn, Error, Fatal or Panic must be then applied to
// this new returned entry.
// If you want multiple fields, use `WithFields`.
func (logger *Logger) WithField(key string, value interface{}) *Entry {
entry := logger.newEntry()
defer logger.releaseEntry(entry)
return entry.WithField(key, value)
}
// Adds a struct of fields to the log entry. All it does is call `WithField` for
// each `Field`.
func (logger *Logger) WithFields(fields Fields) *Entry {
entry := logger.newEntry()
defer logger.releaseEntry(entry)
return entry.WithFields(fields)
}
// Add an error as single field to the log entry. All it does is call
// `WithError` for the given `error`.
func (logger *Logger) WithError(err error) *Entry {
entry := logger.newEntry()
defer logger.releaseEntry(entry)
return entry.WithError(err)
}
// Add a context to the log entry.
func (logger *Logger) WithContext(ctx context.Context) *Entry {
entry := logger.newEntry()
defer logger.releaseEntry(entry)
return entry.WithContext(ctx)
}
// Overrides the time of the log entry.
func (logger *Logger) WithTime(t time.Time) *Entry {
entry := logger.newEntry()
defer logger.releaseEntry(entry)
return entry.WithTime(t)
}
func (logger *Logger) Logf(level Level, format string, args ...interface{}) {
if logger.IsLevelEnabled(level) {
entry := logger.newEntry()
entry.Logf(level, format, args...)
logger.releaseEntry(entry)
}
}
func (logger *Logger) Tracef(format string, args ...interface{}) {
logger.Logf(TraceLevel, format, args...)
}
func (logger *Logger) Debugf(format string, args ...interface{}) {
logger.Logf(DebugLevel, format, args...)
}
func (logger *Logger) Infof(format string, args ...interface{}) {
logger.Logf(InfoLevel, format, args...)
}
func (logger *Logger) Printf(format string, args ...interface{}) {
entry := logger.newEntry()
entry.Printf(format, args...)
logger.releaseEntry(entry)
}
func (logger *Logger) Warnf(format string, args ...interface{}) {
logger.Logf(WarnLevel, format, args...)
}
func (logger *Logger) Warningf(format string, args ...interface{}) {
logger.Warnf(format, args...)
}
func (logger *Logger) Errorf(format string, args ...interface{}) {
logger.Logf(ErrorLevel, format, args...)
}
func (logger *Logger) Fatalf(format string, args ...interface{}) {
logger.Logf(FatalLevel, format, args...)
logger.Exit(1)
}
func (logger *Logger) Panicf(format string, args ...interface{}) {
logger.Logf(PanicLevel, format, args...)
}
// Log will log a message at the level given as parameter.
// Warning: using Log at Panic or Fatal level will not respectively Panic nor Exit.
// For this behaviour Logger.Panic or Logger.Fatal should be used instead.
func (logger *Logger) Log(level Level, args ...interface{}) {
if logger.IsLevelEnabled(level) {
entry := logger.newEntry()
entry.Log(level, args...)
logger.releaseEntry(entry)
}
}
func (logger *Logger) LogFn(level Level, fn LogFunction) {
if logger.IsLevelEnabled(level) {
entry := logger.newEntry()
entry.Log(level, fn()...)
logger.releaseEntry(entry)
}
}
func (logger *Logger) Trace(args ...interface{}) {
logger.Log(TraceLevel, args...)
}
func (logger *Logger) Debug(args ...interface{}) {
logger.Log(DebugLevel, args...)
}
func (logger *Logger) Info(args ...interface{}) {
logger.Log(InfoLevel, args...)
}
func (logger *Logger) Print(args ...interface{}) {
entry := logger.newEntry()
entry.Print(args...)
logger.releaseEntry(entry)
}
func (logger *Logger) Warn(args ...interface{}) {
logger.Log(WarnLevel, args...)
}
func (logger *Logger) Warning(args ...interface{}) {
logger.Warn(args...)
}
func (logger *Logger) Error(args ...interface{}) {
logger.Log(ErrorLevel, args...)
}
func (logger *Logger) Fatal(args ...interface{}) {
logger.Log(FatalLevel, args...)
logger.Exit(1)
}
func (logger *Logger) Panic(args ...interface{}) {
logger.Log(PanicLevel, args...)
}
func (logger *Logger) TraceFn(fn LogFunction) {
logger.LogFn(TraceLevel, fn)
}
func (logger *Logger) DebugFn(fn LogFunction) {
logger.LogFn(DebugLevel, fn)
}
func (logger *Logger) InfoFn(fn LogFunction) {
logger.LogFn(InfoLevel, fn)
}
func (logger *Logger) PrintFn(fn LogFunction) {
entry := logger.newEntry()
entry.Print(fn()...)
logger.releaseEntry(entry)
}
func (logger *Logger) WarnFn(fn LogFunction) {
logger.LogFn(WarnLevel, fn)
}
func (logger *Logger) WarningFn(fn LogFunction) {
logger.WarnFn(fn)
}
func (logger *Logger) ErrorFn(fn LogFunction) {
logger.LogFn(ErrorLevel, fn)
}
func (logger *Logger) FatalFn(fn LogFunction) {
logger.LogFn(FatalLevel, fn)
logger.Exit(1)
}
func (logger *Logger) PanicFn(fn LogFunction) {
logger.LogFn(PanicLevel, fn)
}
func (logger *Logger) Logln(level Level, args ...interface{}) {
if logger.IsLevelEnabled(level) {
entry := logger.newEntry()
entry.Logln(level, args...)
logger.releaseEntry(entry)
}
}
func (logger *Logger) Traceln(args ...interface{}) {
logger.Logln(TraceLevel, args...)
}
func (logger *Logger) Debugln(args ...interface{}) {
logger.Logln(DebugLevel, args...)
}
func (logger *Logger) Infoln(args ...interface{}) {
logger.Logln(InfoLevel, args...)
}
func (logger *Logger) Println(args ...interface{}) {
entry := logger.newEntry()
entry.Println(args...)
logger.releaseEntry(entry)
}
func (logger *Logger) Warnln(args ...interface{}) {
logger.Logln(WarnLevel, args...)
}
func (logger *Logger) Warningln(args ...interface{}) {
logger.Warnln(args...)
}
func (logger *Logger) Errorln(args ...interface{}) {
logger.Logln(ErrorLevel, args...)
}
func (logger *Logger) Fatalln(args ...interface{}) {
logger.Logln(FatalLevel, args...)
logger.Exit(1)
}
func (logger *Logger) Panicln(args ...interface{}) {
logger.Logln(PanicLevel, args...)
}
func (logger *Logger) Exit(code int) {
runHandlers()
if logger.ExitFunc == nil {
logger.ExitFunc = os.Exit
}
logger.ExitFunc(code)
}
//When file is opened with appending mode, it's safe to
//write concurrently to a file (within 4k message on Linux).
//In these cases user can choose to disable the lock.
func (logger *Logger) SetNoLock() {
logger.mu.Disable()
}
func (logger *Logger) level() Level {
return Level(atomic.LoadUint32((*uint32)(&logger.Level)))
}
// SetLevel sets the logger level.
func (logger *Logger) SetLevel(level Level) {
atomic.StoreUint32((*uint32)(&logger.Level), uint32(level))
}
// GetLevel returns the logger level.
func (logger *Logger) GetLevel() Level {
return logger.level()
}
// AddHook adds a hook to the logger hooks.
func (logger *Logger) AddHook(hook Hook) {
logger.mu.Lock()
defer logger.mu.Unlock()
logger.Hooks.Add(hook)
}
// IsLevelEnabled checks if the log level of the logger is greater than the level param
func (logger *Logger) IsLevelEnabled(level Level) bool {
return logger.level() >= level
}
// SetFormatter sets the logger formatter.
func (logger *Logger) SetFormatter(formatter Formatter) {
logger.mu.Lock()
defer logger.mu.Unlock()
logger.Formatter = formatter
}
// SetOutput sets the logger output.
func (logger *Logger) SetOutput(output io.Writer) {
logger.mu.Lock()
defer logger.mu.Unlock()
logger.Out = output
}
func (logger *Logger) SetReportCaller(reportCaller bool) {
logger.mu.Lock()
defer logger.mu.Unlock()
logger.ReportCaller = reportCaller
}
// ReplaceHooks replaces the logger hooks and returns the old ones
func (logger *Logger) ReplaceHooks(hooks LevelHooks) LevelHooks {
logger.mu.Lock()
oldHooks := logger.Hooks
logger.Hooks = hooks
logger.mu.Unlock()
return oldHooks
}
// SetBufferPool sets the logger buffer pool.
func (logger *Logger) SetBufferPool(pool BufferPool) {
logger.mu.Lock()
defer logger.mu.Unlock()
logger.BufferPool = pool
}

View File

@ -1,186 +0,0 @@
package logrus
import (
"fmt"
"log"
"strings"
)
// Fields type, used to pass to `WithFields`.
type Fields map[string]interface{}
// Level type
type Level uint32
// Convert the Level to a string. E.g. PanicLevel becomes "panic".
func (level Level) String() string {
if b, err := level.MarshalText(); err == nil {
return string(b)
} else {
return "unknown"
}
}
// ParseLevel takes a string level and returns the Logrus log level constant.
func ParseLevel(lvl string) (Level, error) {
switch strings.ToLower(lvl) {
case "panic":
return PanicLevel, nil
case "fatal":
return FatalLevel, nil
case "error":
return ErrorLevel, nil
case "warn", "warning":
return WarnLevel, nil
case "info":
return InfoLevel, nil
case "debug":
return DebugLevel, nil
case "trace":
return TraceLevel, nil
}
var l Level
return l, fmt.Errorf("not a valid logrus Level: %q", lvl)
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (level *Level) UnmarshalText(text []byte) error {
l, err := ParseLevel(string(text))
if err != nil {
return err
}
*level = l
return nil
}
func (level Level) MarshalText() ([]byte, error) {
switch level {
case TraceLevel:
return []byte("trace"), nil
case DebugLevel:
return []byte("debug"), nil
case InfoLevel:
return []byte("info"), nil
case WarnLevel:
return []byte("warning"), nil
case ErrorLevel:
return []byte("error"), nil
case FatalLevel:
return []byte("fatal"), nil
case PanicLevel:
return []byte("panic"), nil
}
return nil, fmt.Errorf("not a valid logrus level %d", level)
}
// A constant exposing all logging levels
var AllLevels = []Level{
PanicLevel,
FatalLevel,
ErrorLevel,
WarnLevel,
InfoLevel,
DebugLevel,
TraceLevel,
}
// These are the different logging levels. You can set the logging level to log
// on your instance of logger, obtained with `logrus.New()`.
const (
// PanicLevel level, highest level of severity. Logs and then calls panic with the
// message passed to Debug, Info, ...
PanicLevel Level = iota
// FatalLevel level. Logs and then calls `logger.Exit(1)`. It will exit even if the
// logging level is set to Panic.
FatalLevel
// ErrorLevel level. Logs. Used for errors that should definitely be noted.
// Commonly used for hooks to send errors to an error tracking service.
ErrorLevel
// WarnLevel level. Non-critical entries that deserve eyes.
WarnLevel
// InfoLevel level. General operational entries about what's going on inside the
// application.
InfoLevel
// DebugLevel level. Usually only enabled when debugging. Very verbose logging.
DebugLevel
// TraceLevel level. Designates finer-grained informational events than the Debug.
TraceLevel
)
// Won't compile if StdLogger can't be realized by a log.Logger
var (
_ StdLogger = &log.Logger{}
_ StdLogger = &Entry{}
_ StdLogger = &Logger{}
)
// StdLogger is what your logrus-enabled library should take, that way
// it'll accept a stdlib logger and a logrus logger. There's no standard
// interface, this is the closest we get, unfortunately.
type StdLogger interface {
Print(...interface{})
Printf(string, ...interface{})
Println(...interface{})
Fatal(...interface{})
Fatalf(string, ...interface{})
Fatalln(...interface{})
Panic(...interface{})
Panicf(string, ...interface{})
Panicln(...interface{})
}
// The FieldLogger interface generalizes the Entry and Logger types
type FieldLogger interface {
WithField(key string, value interface{}) *Entry
WithFields(fields Fields) *Entry
WithError(err error) *Entry
Debugf(format string, args ...interface{})
Infof(format string, args ...interface{})
Printf(format string, args ...interface{})
Warnf(format string, args ...interface{})
Warningf(format string, args ...interface{})
Errorf(format string, args ...interface{})
Fatalf(format string, args ...interface{})
Panicf(format string, args ...interface{})
Debug(args ...interface{})
Info(args ...interface{})
Print(args ...interface{})
Warn(args ...interface{})
Warning(args ...interface{})
Error(args ...interface{})
Fatal(args ...interface{})
Panic(args ...interface{})
Debugln(args ...interface{})
Infoln(args ...interface{})
Println(args ...interface{})
Warnln(args ...interface{})
Warningln(args ...interface{})
Errorln(args ...interface{})
Fatalln(args ...interface{})
Panicln(args ...interface{})
// IsDebugEnabled() bool
// IsInfoEnabled() bool
// IsWarnEnabled() bool
// IsErrorEnabled() bool
// IsFatalEnabled() bool
// IsPanicEnabled() bool
}
// Ext1FieldLogger (the first extension to FieldLogger) is superfluous, it is
// here for consistancy. Do not use. Use Logger or Entry instead.
type Ext1FieldLogger interface {
FieldLogger
Tracef(format string, args ...interface{})
Trace(args ...interface{})
Traceln(args ...interface{})
}

View File

@ -1,11 +0,0 @@
// +build appengine
package logrus
import (
"io"
)
func checkIfTerminal(w io.Writer) bool {
return true
}

Some files were not shown because too many files have changed in this diff Show More