Merge branch '4.5.0'

This commit is contained in:
ErickSkrauch 2020-04-26 22:06:59 +03:00
commit be30c23823
8 changed files with 617 additions and 255 deletions

View File

@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased] - xxxx-xx-xx ## [Unreleased] - xxxx-xx-xx
### Added
- [#24](https://github.com/elyby/chrly/issues/24): Implemented a new strategy for the queue in the batch provider of
Mojang UUIDs: `full-bus`.
- New configuration param `QUEUE_STRATEGY` with the default value `periodic`.
- New configuration params: `MOJANG_API_BASE_URL` and `MOJANG_SESSION_SERVER_BASE_URL`, that allow you to spoof
Mojang API base addresses.
### Changed
- `ely.skinsystem.{hostname}.app.mojang_textures.usernames.round_time` timer will not be recorded if the iteration was
empty.
## [4.4.1] - 2020-04-24 ## [4.4.1] - 2020-04-24
### Added ### Added

View File

@ -97,6 +97,14 @@ docker-compose up -d app
<td>Sentry can be used to collect app errors</td> <td>Sentry can be used to collect app errors</td>
<td><code>https://public:private@your.sentry.io/1</code></td> <td><code>https://public:private@your.sentry.io/1</code></td>
</tr> </tr>
<tr>
<td>QUEUE_STRATEGY</td>
<td>
Sets the strategy for the queue in the batch provider of Mojang UUIDs. Allowed values are <code>periodic</code>
and <code>full-bus</code> (see <a href="https://github.com/elyby/chrly/issues/24">#24</a>).
</td>
<td><code>periodic</code></td>
</tr>
<tr> <tr>
<td>QUEUE_LOOP_DELAY</td> <td>QUEUE_LOOP_DELAY</td>
<td> <td>
@ -137,6 +145,20 @@ docker-compose up -d app
</td> </td>
<td><code>http://remote-provider.com/api/worker/mojang-uuid</code></td> <td><code>http://remote-provider.com/api/worker/mojang-uuid</code></td>
</tr> </tr>
<tr>
<td>MOJANG_API_BASE_URL</td>
<td>
Allows you to spoof the Mojang's API server address.
</td>
<td><code>https://api.mojang.com</code></td>
</tr>
<tr>
<td>MOJANG_SESSION_SERVER_BASE_URL</td>
<td>
Allows you to spoof the Mojang's Session server address.
</td>
<td><code>https://sessionserver.mojang.com</code></td>
</tr>
<tr> <tr>
<td>TEXTURES_EXTRA_PARAM_NAME</td> <td>TEXTURES_EXTRA_PARAM_NAME</td>
<td> <td>

View File

@ -58,11 +58,17 @@ type ProfileInfo struct {
IsDemo bool `json:"demo,omitempty"` IsDemo bool `json:"demo,omitempty"`
} }
var ApiMojangDotComAddr = "https://api.mojang.com"
var SessionServerMojangComAddr = "https://sessionserver.mojang.com"
// Exchanges usernames array to array of uuids // Exchanges usernames array to array of uuids
// See https://wiki.vg/Mojang_API#Playernames_-.3E_UUIDs // See https://wiki.vg/Mojang_API#Playernames_-.3E_UUIDs
func UsernamesToUuids(usernames []string) ([]*ProfileInfo, error) { func UsernamesToUuids(usernames []string) ([]*ProfileInfo, error) {
requestBody, _ := json.Marshal(usernames) requestBody, _ := json.Marshal(usernames)
request, _ := http.NewRequest("POST", "https://api.mojang.com/profiles/minecraft", bytes.NewBuffer(requestBody)) request, err := http.NewRequest("POST", ApiMojangDotComAddr+"/profiles/minecraft", bytes.NewBuffer(requestBody))
if err != nil {
return nil, err
}
request.Header.Set("Content-Type", "application/json") request.Header.Set("Content-Type", "application/json")
@ -88,12 +94,15 @@ func UsernamesToUuids(usernames []string) ([]*ProfileInfo, error) {
// See https://wiki.vg/Mojang_API#UUID_-.3E_Profile_.2B_Skin.2FCape // See https://wiki.vg/Mojang_API#UUID_-.3E_Profile_.2B_Skin.2FCape
func UuidToTextures(uuid string, signed bool) (*SignedTexturesResponse, error) { func UuidToTextures(uuid string, signed bool) (*SignedTexturesResponse, error) {
normalizedUuid := strings.ReplaceAll(uuid, "-", "") normalizedUuid := strings.ReplaceAll(uuid, "-", "")
url := "https://sessionserver.mojang.com/session/minecraft/profile/" + normalizedUuid url := SessionServerMojangComAddr + "/session/minecraft/profile/" + normalizedUuid
if signed { if signed {
url += "?unsigned=false" url += "?unsigned=false"
} }
request, _ := http.NewRequest("GET", url, nil) request, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
response, err := HttpClient.Do(request) response, err := HttpClient.Do(request)
if err != nil { if err != nil {

View File

@ -1,6 +1,7 @@
package di package di
import ( import (
"context"
"fmt" "fmt"
"net/url" "net/url"
"time" "time"
@ -8,21 +9,50 @@ import (
"github.com/goava/di" "github.com/goava/di"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/elyby/chrly/api/mojang"
es "github.com/elyby/chrly/eventsubscribers" es "github.com/elyby/chrly/eventsubscribers"
"github.com/elyby/chrly/http" "github.com/elyby/chrly/http"
"github.com/elyby/chrly/mojangtextures" "github.com/elyby/chrly/mojangtextures"
) )
var mojangTextures = di.Options( var mojangTextures = di.Options(
di.Invoke(interceptMojangApiUrls),
di.Provide(newMojangTexturesProviderFactory), di.Provide(newMojangTexturesProviderFactory),
di.Provide(newMojangTexturesProvider), di.Provide(newMojangTexturesProvider),
di.Provide(newMojangTexturesUuidsProviderFactory), di.Provide(newMojangTexturesUuidsProviderFactory),
di.Provide(newMojangTexturesBatchUUIDsProvider), di.Provide(newMojangTexturesBatchUUIDsProvider),
di.Provide(newMojangTexturesBatchUUIDsProviderStrategyFactory),
di.Provide(newMojangTexturesBatchUUIDsProviderDelayedStrategy),
di.Provide(newMojangTexturesBatchUUIDsProviderFullBusStrategy),
di.Provide(newMojangTexturesRemoteUUIDsProvider), di.Provide(newMojangTexturesRemoteUUIDsProvider),
di.Provide(newMojangSignedTexturesProvider), di.Provide(newMojangSignedTexturesProvider),
di.Provide(newMojangTexturesStorageFactory), di.Provide(newMojangTexturesStorageFactory),
) )
func interceptMojangApiUrls(config *viper.Viper) error {
apiUrl := config.GetString("mojang.api_base_url")
if apiUrl != "" {
u, err := url.ParseRequestURI(apiUrl)
if err != nil {
return err
}
mojang.ApiMojangDotComAddr = u.String()
}
sessionServerUrl := config.GetString("mojang.session_server_base_url")
if sessionServerUrl != "" {
u, err := url.ParseRequestURI(apiUrl)
if err != nil {
return err
}
mojang.SessionServerMojangComAddr = u.String()
}
return nil
}
func newMojangTexturesProviderFactory( func newMojangTexturesProviderFactory(
container *di.Container, container *di.Container,
config *viper.Viper, config *viper.Viper,
@ -75,7 +105,7 @@ func newMojangTexturesUuidsProviderFactory(
func newMojangTexturesBatchUUIDsProvider( func newMojangTexturesBatchUUIDsProvider(
container *di.Container, container *di.Container,
config *viper.Viper, strategy mojangtextures.BatchUuidsProviderStrategy,
emitter mojangtextures.Emitter, emitter mojangtextures.Emitter,
) (*mojangtextures.BatchUuidsProvider, error) { ) (*mojangtextures.BatchUuidsProvider, error) {
if err := container.Provide(func(emitter es.Subscriber, config *viper.Viper) *namedHealthChecker { if err := container.Provide(func(emitter es.Subscriber, config *viper.Viper) *namedHealthChecker {
@ -106,14 +136,56 @@ func newMojangTexturesBatchUUIDsProvider(
return nil, err return nil, err
} }
return mojangtextures.NewBatchUuidsProvider(context.Background(), strategy, emitter), nil
}
func newMojangTexturesBatchUUIDsProviderStrategyFactory(
container *di.Container,
config *viper.Viper,
) (mojangtextures.BatchUuidsProviderStrategy, error) {
config.SetDefault("queue.strategy", "periodic")
strategyName := config.GetString("queue.strategy")
switch strategyName {
case "periodic":
var strategy *mojangtextures.PeriodicStrategy
err := container.Resolve(&strategy)
if err != nil {
return nil, err
}
return strategy, nil
case "full-bus":
var strategy *mojangtextures.FullBusStrategy
err := container.Resolve(&strategy)
if err != nil {
return nil, err
}
return strategy, nil
default:
return nil, fmt.Errorf("unknown queue strategy \"%s\"", strategyName)
}
}
func newMojangTexturesBatchUUIDsProviderDelayedStrategy(config *viper.Viper) *mojangtextures.PeriodicStrategy {
config.SetDefault("queue.loop_delay", 2*time.Second+500*time.Millisecond) config.SetDefault("queue.loop_delay", 2*time.Second+500*time.Millisecond)
config.SetDefault("queue.batch_size", 10) config.SetDefault("queue.batch_size", 10)
return &mojangtextures.BatchUuidsProvider{ return mojangtextures.NewPeriodicStrategy(
Emitter: emitter, config.GetDuration("queue.loop_delay"),
IterationDelay: config.GetDuration("queue.loop_delay"), config.GetInt("queue.batch_size"),
IterationSize: config.GetInt("queue.batch_size"), )
}, nil }
func newMojangTexturesBatchUUIDsProviderFullBusStrategy(config *viper.Viper) *mojangtextures.FullBusStrategy {
config.SetDefault("queue.loop_delay", 2*time.Second+500*time.Millisecond)
config.SetDefault("queue.batch_size", 10)
return mojangtextures.NewFullBusStrategy(
config.GetDuration("queue.loop_delay"),
config.GetInt("queue.batch_size"),
)
} }
func newMojangTexturesRemoteUUIDsProvider( func newMojangTexturesRemoteUUIDsProvider(

View File

@ -96,12 +96,12 @@ func (s *StatsReporter) ConfigureWithDispatcher(d Subscriber) {
d.Subscribe("mojang_textures:batch_uuids_provider:round", func(usernames []string, queueSize int) { d.Subscribe("mojang_textures:batch_uuids_provider:round", func(usernames []string, queueSize int) {
s.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(usernames))) s.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(usernames)))
s.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize)) s.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize))
if len(usernames) != 0 {
s.startTimeRecording("batch_uuids_provider_round_time_" + strings.Join(usernames, "|"))
}
}) })
d.Subscribe("mojang_textures:batch_uuids_provider:before_round", func() { d.Subscribe("mojang_textures:batch_uuids_provider:result", func(usernames []string, profiles []*mojang.ProfileInfo, err error) {
s.startTimeRecording("batch_uuids_provider_round_time") s.finalizeTimeRecording("batch_uuids_provider_round_time_"+strings.Join(usernames, "|"), "mojang_textures.usernames.round_time")
})
d.Subscribe("mojang_textures:batch_uuids_provider:after_round", func() {
s.finalizeTimeRecording("batch_uuids_provider_round_time", "mojang_textures.usernames.round_time")
}) })
} }

View File

@ -337,19 +337,24 @@ var statsReporterTestCases = []*StatsReporterTestCase{
{ {
Events: [][]interface{}{ Events: [][]interface{}{
{"mojang_textures:batch_uuids_provider:round", []string{"username1", "username2"}, 5}, {"mojang_textures:batch_uuids_provider:round", []string{"username1", "username2"}, 5},
{"mojang_textures:batch_uuids_provider:result", []string{"username1", "username2"}, []*mojang.ProfileInfo{}, nil},
}, },
ExpectedCalls: [][]interface{}{ ExpectedCalls: [][]interface{}{
{"UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)}, {"UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)},
{"UpdateGauge", "mojang_textures.usernames.queue_size", int64(5)}, {"UpdateGauge", "mojang_textures.usernames.queue_size", int64(5)},
{"RecordTimer", "mojang_textures.usernames.round_time", mock.AnythingOfType("time.Duration")},
}, },
}, },
{ {
Events: [][]interface{}{ Events: [][]interface{}{
{"mojang_textures:batch_uuids_provider:before_round"}, {"mojang_textures:batch_uuids_provider:round", []string{}, 0},
{"mojang_textures:batch_uuids_provider:after_round"}, // This event will be not emitted, but we emit it to ensure, that RecordTimer will not be called
{"mojang_textures:batch_uuids_provider:result", []string{}, []*mojang.ProfileInfo{}, nil},
}, },
ExpectedCalls: [][]interface{}{ ExpectedCalls: [][]interface{}{
{"RecordTimer", "mojang_textures.usernames.round_time", mock.AnythingOfType("time.Duration")}, {"UpdateGauge", "mojang_textures.usernames.iteration_size", int64(0)},
{"UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)},
// Should not call RecordTimer
}, },
}, },
} }

View File

@ -1,6 +1,7 @@
package mojangtextures package mojangtextures
import ( import (
"context"
"strings" "strings"
"sync" "sync"
"time" "time"
@ -9,131 +10,234 @@ import (
) )
type jobResult struct { type jobResult struct {
profile *mojang.ProfileInfo Profile *mojang.ProfileInfo
error error Error error
} }
type jobItem struct { type job struct {
username string Username string
respondChan chan *jobResult RespondChan chan *jobResult
} }
type jobsQueue struct { type jobsQueue struct {
lock sync.Mutex lock sync.Mutex
items []*jobItem items []*job
} }
func (s *jobsQueue) New() *jobsQueue { func newJobsQueue() *jobsQueue {
s.items = []*jobItem{} return &jobsQueue{
return s items: []*job{},
}
func (s *jobsQueue) Enqueue(t *jobItem) {
s.lock.Lock()
defer s.lock.Unlock()
s.items = append(s.items, t)
}
func (s *jobsQueue) Dequeue(n int) []*jobItem {
s.lock.Lock()
defer s.lock.Unlock()
if n > s.size() {
n = s.size()
} }
items := s.items[0:n]
s.items = s.items[n:len(s.items)]
return items
} }
func (s *jobsQueue) Size() int { func (s *jobsQueue) Enqueue(job *job) int {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
return s.size() s.items = append(s.items, job)
}
func (s *jobsQueue) size() int {
return len(s.items) return len(s.items)
} }
func (s *jobsQueue) Dequeue(n int) ([]*job, int) {
s.lock.Lock()
defer s.lock.Unlock()
l := len(s.items)
if n > l {
n = l
}
items := s.items[0:n]
s.items = s.items[n:l]
return items, l - n
}
var usernamesToUuids = mojang.UsernamesToUuids var usernamesToUuids = mojang.UsernamesToUuids
var forever = func() bool {
return true type JobsIteration struct {
Jobs []*job
Queue int
c chan struct{}
}
func (j *JobsIteration) Done() {
if j.c != nil {
close(j.c)
}
}
type BatchUuidsProviderStrategy interface {
Queue(job *job)
GetJobs(abort context.Context) <-chan *JobsIteration
}
type PeriodicStrategy struct {
Delay time.Duration
Batch int
queue *jobsQueue
done chan struct{}
}
func NewPeriodicStrategy(delay time.Duration, batch int) *PeriodicStrategy {
return &PeriodicStrategy{
Delay: delay,
Batch: batch,
queue: newJobsQueue(),
}
}
func (ctx *PeriodicStrategy) Queue(job *job) {
ctx.queue.Enqueue(job)
}
func (ctx *PeriodicStrategy) GetJobs(abort context.Context) <-chan *JobsIteration {
ch := make(chan *JobsIteration)
go func() {
for {
select {
case <-abort.Done():
close(ch)
return
case <-time.After(ctx.Delay):
jobs, queueLen := ctx.queue.Dequeue(ctx.Batch)
jobDoneChan := make(chan struct{})
ch <- &JobsIteration{jobs, queueLen, jobDoneChan}
<-jobDoneChan
}
}
}()
return ch
}
type FullBusStrategy struct {
Delay time.Duration
Batch int
queue *jobsQueue
busIsFull chan bool
}
func NewFullBusStrategy(delay time.Duration, batch int) *FullBusStrategy {
return &FullBusStrategy{
Delay: delay,
Batch: batch,
queue: newJobsQueue(),
busIsFull: make(chan bool),
}
}
func (ctx *FullBusStrategy) Queue(job *job) {
n := ctx.queue.Enqueue(job)
if n % ctx.Batch == 0 {
ctx.busIsFull <- true
}
}
// Формально, это описание логики водителя маршрутки xD
func (ctx *FullBusStrategy) GetJobs(abort context.Context) <-chan *JobsIteration {
ch := make(chan *JobsIteration)
go func() {
for {
t := time.NewTimer(ctx.Delay)
select {
case <-abort.Done():
close(ch)
return
case <-t.C:
ctx.sendJobs(ch)
case <-ctx.busIsFull:
t.Stop()
ctx.sendJobs(ch)
}
}
}()
return ch
}
func (ctx *FullBusStrategy) sendJobs(ch chan *JobsIteration) {
jobs, queueLen := ctx.queue.Dequeue(ctx.Batch)
ch <- &JobsIteration{jobs, queueLen, nil}
} }
type BatchUuidsProvider struct { type BatchUuidsProvider struct {
Emitter context context.Context
emitter Emitter
IterationDelay time.Duration strategy BatchUuidsProviderStrategy
IterationSize int
onFirstCall sync.Once onFirstCall sync.Once
queue jobsQueue }
func NewBatchUuidsProvider(
context context.Context,
strategy BatchUuidsProviderStrategy,
emitter Emitter,
) *BatchUuidsProvider {
return &BatchUuidsProvider{
context: context,
emitter: emitter,
strategy: strategy,
}
} }
func (ctx *BatchUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) { func (ctx *BatchUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo, error) {
ctx.onFirstCall.Do(func() { ctx.onFirstCall.Do(ctx.startQueue)
ctx.queue.New()
ctx.startQueue()
})
resultChan := make(chan *jobResult) resultChan := make(chan *jobResult)
ctx.queue.Enqueue(&jobItem{username, resultChan}) ctx.strategy.Queue(&job{username, resultChan})
ctx.Emit("mojang_textures:batch_uuids_provider:queued", username) ctx.emitter.Emit("mojang_textures:batch_uuids_provider:queued", username)
result := <-resultChan result := <-resultChan
return result.profile, result.error return result.Profile, result.Error
} }
func (ctx *BatchUuidsProvider) startQueue() { func (ctx *BatchUuidsProvider) startQueue() {
go func() { go func() {
time.Sleep(ctx.IterationDelay) jobsChan := ctx.strategy.GetJobs(ctx.context)
for forever() { for {
ctx.Emit("mojang_textures:batch_uuids_provider:before_round") select {
ctx.queueRound() case <-ctx.context.Done():
ctx.Emit("mojang_textures:batch_uuids_provider:after_round") return
time.Sleep(ctx.IterationDelay) case iteration := <-jobsChan:
go func() {
ctx.performRequest(iteration)
iteration.Done()
}()
}
} }
}() }()
} }
func (ctx *BatchUuidsProvider) queueRound() { func (ctx *BatchUuidsProvider) performRequest(iteration *JobsIteration) {
queueSize := ctx.queue.Size() usernames := make([]string, len(iteration.Jobs))
jobs := ctx.queue.Dequeue(ctx.IterationSize) for i, job := range iteration.Jobs {
usernames[i] = job.Username
var usernames []string
for _, job := range jobs {
usernames = append(usernames, job.username)
} }
ctx.Emit("mojang_textures:batch_uuids_provider:round", usernames, queueSize-len(jobs)) ctx.emitter.Emit("mojang_textures:batch_uuids_provider:round", usernames, iteration.Queue)
if len(usernames) == 0 { if len(usernames) == 0 {
return return
} }
profiles, err := usernamesToUuids(usernames) profiles, err := usernamesToUuids(usernames)
ctx.Emit("mojang_textures:batch_uuids_provider:result", usernames, profiles, err) ctx.emitter.Emit("mojang_textures:batch_uuids_provider:result", usernames, profiles, err)
for _, job := range jobs { for _, job := range iteration.Jobs {
go func(job *jobItem) { response := &jobResult{}
response := &jobResult{} if err == nil {
if err != nil { // The profiles in the response aren't ordered, so we must search each username over full array
response.error = err for _, profile := range profiles {
} else { if strings.EqualFold(job.Username, profile.Name) {
// The profiles in the response aren't ordered, so we must search each username over full array response.Profile = profile
for _, profile := range profiles { break
if strings.EqualFold(job.username, profile.Name) {
response.profile = profile
break
}
} }
} }
} else {
response.Error = err
}
job.respondChan <- response job.RespondChan <- response
}(job) close(job.RespondChan)
} }
} }

View File

@ -1,64 +1,51 @@
package mojangtextures package mojangtextures
import ( import (
"crypto/rand" "context"
"encoding/base64" "fmt"
"strings" "strconv"
"sync"
"testing" "testing"
"time"
testify "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock" "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/elyby/chrly/api/mojang" "github.com/elyby/chrly/api/mojang"
) )
func TestJobsQueue(t *testing.T) { func TestJobsQueue(t *testing.T) {
createQueue := func() *jobsQueue {
queue := &jobsQueue{}
queue.New()
return queue
}
t.Run("Enqueue", func(t *testing.T) { t.Run("Enqueue", func(t *testing.T) {
assert := testify.New(t) s := newJobsQueue()
require.Equal(t, 1, s.Enqueue(&job{Username: "username1"}))
s := createQueue() require.Equal(t, 2, s.Enqueue(&job{Username: "username2"}))
s.Enqueue(&jobItem{username: "username1"}) require.Equal(t, 3, s.Enqueue(&job{Username: "username3"}))
s.Enqueue(&jobItem{username: "username2"})
s.Enqueue(&jobItem{username: "username3"})
assert.Equal(3, s.Size())
}) })
t.Run("Dequeue", func(t *testing.T) { t.Run("Dequeue", func(t *testing.T) {
assert := testify.New(t) s := newJobsQueue()
s.Enqueue(&job{Username: "username1"})
s.Enqueue(&job{Username: "username2"})
s.Enqueue(&job{Username: "username3"})
s.Enqueue(&job{Username: "username4"})
s.Enqueue(&job{Username: "username5"})
s := createQueue() items, queueLen := s.Dequeue(2)
s.Enqueue(&jobItem{username: "username1"}) require.Len(t, items, 2)
s.Enqueue(&jobItem{username: "username2"}) require.Equal(t, 3, queueLen)
s.Enqueue(&jobItem{username: "username3"}) require.Equal(t, "username1", items[0].Username)
s.Enqueue(&jobItem{username: "username4"}) require.Equal(t, "username2", items[1].Username)
items := s.Dequeue(2) items, queueLen = s.Dequeue(40)
assert.Len(items, 2) require.Len(t, items, 3)
assert.Equal("username1", items[0].username) require.Equal(t, 0, queueLen)
assert.Equal("username2", items[1].username) require.Equal(t, "username3", items[0].Username)
assert.Equal(2, s.Size()) require.Equal(t, "username4", items[1].Username)
require.Equal(t, "username5", items[2].Username)
items = s.Dequeue(40)
assert.Len(items, 2)
assert.Equal("username3", items[0].username)
assert.Equal("username4", items[1].username)
}) })
} }
// This is really stupid test just to get 100% coverage on this package :)
func TestBatchUuidsProvider_forever(t *testing.T) {
testify.True(t, forever())
}
type mojangUsernamesToUuidsRequestMock struct { type mojangUsernamesToUuidsRequestMock struct {
mock.Mock mock.Mock
} }
@ -73,6 +60,37 @@ func (o *mojangUsernamesToUuidsRequestMock) UsernamesToUuids(usernames []string)
return result, args.Error(1) return result, args.Error(1)
} }
type manualStrategy struct {
ch chan *JobsIteration
once sync.Once
lock sync.Mutex
jobs []*job
}
func (m *manualStrategy) Queue(job *job) {
m.lock.Lock()
m.jobs = append(m.jobs, job)
m.lock.Unlock()
}
func (m *manualStrategy) GetJobs(_ context.Context) <-chan *JobsIteration {
m.lock.Lock()
defer m.lock.Unlock()
m.ch = make(chan *JobsIteration)
return m.ch
}
func (m *manualStrategy) Iterate(countJobsToReturn int, countLeftJobsInQueue int) {
m.lock.Lock()
defer m.lock.Unlock()
m.ch <- &JobsIteration{
Jobs: m.jobs[0:countJobsToReturn],
Queue: countLeftJobsInQueue,
}
}
type batchUuidsProviderGetUuidResult struct { type batchUuidsProviderGetUuidResult struct {
Result *mojang.ProfileInfo Result *mojang.ProfileInfo
Error error Error error
@ -81,49 +99,36 @@ type batchUuidsProviderGetUuidResult struct {
type batchUuidsProviderTestSuite struct { type batchUuidsProviderTestSuite struct {
suite.Suite suite.Suite
Provider *BatchUuidsProvider Provider *BatchUuidsProvider
GetUuidAsync func(username string) chan *batchUuidsProviderGetUuidResult
Emitter *mockEmitter Emitter *mockEmitter
Strategy *manualStrategy
MojangApi *mojangUsernamesToUuidsRequestMock MojangApi *mojangUsernamesToUuidsRequestMock
Iterate func() GetUuidAsync func(username string) <-chan *batchUuidsProviderGetUuidResult
done func() stop context.CancelFunc
iterateChan chan bool
} }
func (suite *batchUuidsProviderTestSuite) SetupTest() { func (suite *batchUuidsProviderTestSuite) SetupTest() {
suite.Emitter = &mockEmitter{} suite.Emitter = &mockEmitter{}
suite.Strategy = &manualStrategy{}
ctx, stop := context.WithCancel(context.Background())
suite.stop = stop
suite.MojangApi = &mojangUsernamesToUuidsRequestMock{}
usernamesToUuids = suite.MojangApi.UsernamesToUuids
suite.Provider = &BatchUuidsProvider{ suite.Provider = NewBatchUuidsProvider(ctx, suite.Strategy, suite.Emitter)
Emitter: suite.Emitter,
IterationDelay: 0,
IterationSize: 10,
}
suite.iterateChan = make(chan bool) suite.GetUuidAsync = func(username string) <-chan *batchUuidsProviderGetUuidResult {
forever = func() bool { s := make(chan struct{})
return <-suite.iterateChan
}
suite.Iterate = func() {
suite.iterateChan <- true
}
suite.done = func() {
suite.iterateChan <- false
}
suite.GetUuidAsync = func(username string) chan *batchUuidsProviderGetUuidResult {
s := make(chan bool)
// This dirty hack ensures, that the username will be queued before we return control to the caller. // This dirty hack ensures, that the username will be queued before we return control to the caller.
// It's needed to keep expected calls order and prevent cases when iteration happens before all usernames // It's needed to keep expected calls order and prevent cases when iteration happens before
// will be queued. // all usernames will be queued.
suite.Emitter.On("Emit", suite.Emitter.On("Emit",
"mojang_textures:batch_uuids_provider:queued", "mojang_textures:batch_uuids_provider:queued",
username, username,
).Once().Run(func(args mock.Arguments) { ).Once().Run(func(args mock.Arguments) {
s <- true close(s)
}) })
c := make(chan *batchUuidsProviderGetUuidResult) c := make(chan *batchUuidsProviderGetUuidResult)
@ -139,13 +144,10 @@ func (suite *batchUuidsProviderTestSuite) SetupTest() {
return c return c
} }
suite.MojangApi = &mojangUsernamesToUuidsRequestMock{}
usernamesToUuids = suite.MojangApi.UsernamesToUuids
} }
func (suite *batchUuidsProviderTestSuite) TearDownTest() { func (suite *batchUuidsProviderTestSuite) TearDownTest() {
suite.done() suite.stop()
suite.Emitter.AssertExpectations(suite.T()) suite.Emitter.AssertExpectations(suite.T())
suite.MojangApi.AssertExpectations(suite.T()) suite.MojangApi.AssertExpectations(suite.T())
} }
@ -154,37 +156,14 @@ func TestBatchUuidsProvider(t *testing.T) {
suite.Run(t, new(batchUuidsProviderTestSuite)) suite.Run(t, new(batchUuidsProviderTestSuite))
} }
func (suite *batchUuidsProviderTestSuite) TestGetUuidForOneUsername() { func (suite *batchUuidsProviderTestSuite) TestGetUuidForFewUsernames() {
expectedUsernames := []string{"username"}
expectedResult := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username"}
expectedResponse := []*mojang.ProfileInfo{expectedResult}
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:before_round").Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", expectedUsernames, 0).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", expectedUsernames, expectedResponse, nil).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:after_round").Once()
suite.MojangApi.On("UsernamesToUuids", expectedUsernames).Once().Return([]*mojang.ProfileInfo{expectedResult}, nil)
resultChan := suite.GetUuidAsync("username")
suite.Iterate()
result := <-resultChan
suite.Assert().Equal(expectedResult, result.Result)
suite.Assert().Nil(result.Error)
}
func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() {
expectedUsernames := []string{"username1", "username2"} expectedUsernames := []string{"username1", "username2"}
expectedResult1 := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username1"} expectedResult1 := &mojang.ProfileInfo{Id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Name: "username1"}
expectedResult2 := &mojang.ProfileInfo{Id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Name: "username2"} expectedResult2 := &mojang.ProfileInfo{Id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Name: "username2"}
expectedResponse := []*mojang.ProfileInfo{expectedResult1, expectedResult2} expectedResponse := []*mojang.ProfileInfo{expectedResult1, expectedResult2}
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:before_round").Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", expectedUsernames, 0).Once() suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", expectedUsernames, 0).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", expectedUsernames, expectedResponse, nil).Once() suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", expectedUsernames, expectedResponse, nil).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:after_round").Once()
suite.MojangApi.On("UsernamesToUuids", expectedUsernames).Once().Return([]*mojang.ProfileInfo{ suite.MojangApi.On("UsernamesToUuids", expectedUsernames).Once().Return([]*mojang.ProfileInfo{
expectedResult1, expectedResult1,
@ -194,7 +173,7 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() {
resultChan1 := suite.GetUuidAsync("username1") resultChan1 := suite.GetUuidAsync("username1")
resultChan2 := suite.GetUuidAsync("username2") resultChan2 := suite.GetUuidAsync("username2")
suite.Iterate() suite.Strategy.Iterate(2, 0)
result1 := <-resultChan1 result1 := <-resultChan1
suite.Assert().Equal(expectedResult1, result1.Result) suite.Assert().Equal(expectedResult1, result1.Result)
@ -205,78 +184,40 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernames() {
suite.Assert().Nil(result2.Error) suite.Assert().Nil(result2.Error)
} }
func (suite *batchUuidsProviderTestSuite) TestGetUuidForMoreThan10Usernames() { func (suite *batchUuidsProviderTestSuite) TestShouldNotSendRequestWhenNoJobsAreReturned() {
usernames := make([]string, 12) //noinspection GoPreferNilSlice
for i := 0; i < cap(usernames); i++ { emptyUsernames := []string{}
usernames[i] = randStr(8) done := make(chan struct{})
} suite.Emitter.On("Emit",
"mojang_textures:batch_uuids_provider:round",
emptyUsernames,
1,
).Once().Run(func(args mock.Arguments) {
close(done)
})
// In this test we're not testing response, so always return an empty resultset _ = suite.GetUuidAsync("username") // Schedule one username to run the queue
expectedResponse := []*mojang.ProfileInfo{}
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:before_round").Twice() suite.Strategy.Iterate(0, 1) // Return no jobs and indicate that there is one job in queue
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", usernames[0:10], 2).Once() <-done
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", usernames[0:10], expectedResponse, nil).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", usernames[10:12], 0).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", usernames[10:12], expectedResponse, nil).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:after_round").Twice()
suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return(expectedResponse, nil)
suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return(expectedResponse, nil)
channels := make([]chan *batchUuidsProviderGetUuidResult, len(usernames))
for i, username := range usernames {
channels[i] = suite.GetUuidAsync(username)
}
suite.Iterate()
suite.Iterate()
for _, channel := range channels {
<-channel
}
} }
func (suite *batchUuidsProviderTestSuite) TestDoNothingWhenNoTasks() { // Test written for multiple usernames to ensure that the error
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:before_round").Times(3) // will be returned for each iteration group
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", []string{"username"}, 0).Once() func (suite *batchUuidsProviderTestSuite) TestGetUuidForFewUsernamesWithAnError() {
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", []string{"username"}, mock.Anything, nil).Once()
var nilStringSlice []string
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", nilStringSlice, 0).Twice()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:after_round").Times(3)
suite.MojangApi.On("UsernamesToUuids", []string{"username"}).Once().Return([]*mojang.ProfileInfo{}, nil)
// Perform first iteration and await it finishes
resultChan := suite.GetUuidAsync("username")
suite.Iterate()
result := <-resultChan
suite.Assert().Nil(result.Result)
suite.Assert().Nil(result.Error)
// Let it to perform a few more iterations to ensure, that there are no calls to external APIs
suite.Iterate()
suite.Iterate()
}
func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernamesWithAnError() {
expectedUsernames := []string{"username1", "username2"} expectedUsernames := []string{"username1", "username2"}
expectedError := &mojang.TooManyRequestsError{} expectedError := &mojang.TooManyRequestsError{}
var nilProfilesResponse []*mojang.ProfileInfo var nilProfilesResponse []*mojang.ProfileInfo
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:before_round").Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", expectedUsernames, 0).Once() suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:round", expectedUsernames, 0).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", expectedUsernames, nilProfilesResponse, expectedError).Once() suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:result", expectedUsernames, nilProfilesResponse, expectedError).Once()
suite.Emitter.On("Emit", "mojang_textures:batch_uuids_provider:after_round").Once()
suite.MojangApi.On("UsernamesToUuids", expectedUsernames).Once().Return(nil, expectedError) suite.MojangApi.On("UsernamesToUuids", expectedUsernames).Once().Return(nil, expectedError)
resultChan1 := suite.GetUuidAsync("username1") resultChan1 := suite.GetUuidAsync("username1")
resultChan2 := suite.GetUuidAsync("username2") resultChan2 := suite.GetUuidAsync("username2")
suite.Iterate() suite.Strategy.Iterate(2, 0)
result1 := <-resultChan1 result1 := <-resultChan1
suite.Assert().Nil(result1.Result) suite.Assert().Nil(result1.Result)
@ -287,14 +228,213 @@ func (suite *batchUuidsProviderTestSuite) TestGetUuidForTwoUsernamesWithAnError(
suite.Assert().Equal(expectedError, result2.Error) suite.Assert().Equal(expectedError, result2.Error)
} }
var replacer = strings.NewReplacer("-", "_", "=", "") func TestPeriodicStrategy(t *testing.T) {
t.Run("should return first job only after duration", func(t *testing.T) {
d := 20 * time.Millisecond
strategy := NewPeriodicStrategy(d, 10)
j := &job{}
strategy.Queue(j)
// https://stackoverflow.com/a/50581165 ctx, cancel := context.WithCancel(context.Background())
func randStr(len int) string { startedAt := time.Now()
buff := make([]byte, len) ch := strategy.GetJobs(ctx)
_, _ = rand.Read(buff) iteration := <-ch
str := replacer.Replace(base64.URLEncoding.EncodeToString(buff)) durationBeforeResult := time.Now().Sub(startedAt)
require.True(t, durationBeforeResult >= d)
require.True(t, durationBeforeResult < d*2)
// Base 64 can be longer than len require.Equal(t, []*job{j}, iteration.Jobs)
return str[:len] require.Equal(t, 0, iteration.Queue)
cancel()
})
t.Run("should return the configured batch size", func(t *testing.T) {
strategy := NewPeriodicStrategy(0, 10)
jobs := make([]*job, 15)
for i := 0; i < 15; i++ {
jobs[i] = &job{Username: strconv.Itoa(i)}
strategy.Queue(jobs[i])
}
ctx, cancel := context.WithCancel(context.Background())
ch := strategy.GetJobs(ctx)
iteration := <-ch
require.Len(t, iteration.Jobs, 10)
require.Equal(t, jobs[0:10], iteration.Jobs)
require.Equal(t, 5, iteration.Queue)
cancel()
})
t.Run("should not return the next iteration until the previous one is finished", func(t *testing.T) {
strategy := NewPeriodicStrategy(0, 10)
strategy.Queue(&job{})
ctx, cancel := context.WithCancel(context.Background())
ch := strategy.GetJobs(ctx)
iteration := <-ch
require.Len(t, iteration.Jobs, 1)
require.Equal(t, 0, iteration.Queue)
time.Sleep(time.Millisecond) // Let strategy's internal loop to work (if the implementation is broken)
select {
case <-ch:
require.Fail(t, "the previous iteration isn't marked as done")
default:
// ok
}
iteration.Done()
time.Sleep(time.Millisecond) // Let strategy's internal loop to work
select {
case iteration = <-ch:
// ok
default:
require.Fail(t, "iteration should be provided")
}
require.Empty(t, iteration.Jobs)
require.Equal(t, 0, iteration.Queue)
iteration.Done()
cancel()
})
t.Run("each iteration should be returned only after the configured duration", func(t *testing.T) {
d := 5 * time.Millisecond
strategy := NewPeriodicStrategy(d, 10)
ctx, cancel := context.WithCancel(context.Background())
ch := strategy.GetJobs(ctx)
for i := 0; i < 3; i++ {
startedAt := time.Now()
iteration := <-ch
durationBeforeResult := time.Now().Sub(startedAt)
require.True(t, durationBeforeResult >= d)
require.True(t, durationBeforeResult < d*2)
require.Empty(t, iteration.Jobs)
require.Equal(t, 0, iteration.Queue)
// Sleep for at least doubled duration before calling Done() to check,
// that this duration isn't included into the next iteration time
time.Sleep(d * 2)
iteration.Done()
}
cancel()
})
}
func TestFullBusStrategy(t *testing.T) {
t.Run("should provide iteration immediately when the batch size exceeded", func(t *testing.T) {
jobs := make([]*job, 10)
for i := 0; i < 10; i++ {
jobs[i] = &job{}
}
d := 20 * time.Millisecond
strategy := NewFullBusStrategy(d, 10)
ctx, cancel := context.WithCancel(context.Background())
ch := strategy.GetJobs(ctx)
done := make(chan struct{})
go func() {
defer close(done)
select {
case iteration := <-ch:
require.Len(t, iteration.Jobs, 10)
require.Equal(t, 0, iteration.Queue)
case <-time.After(d):
require.Fail(t, "iteration should be provided immediately")
}
}()
for _, j := range jobs {
strategy.Queue(j)
}
<-done
cancel()
})
t.Run("should provide iteration after duration if batch size isn't exceeded", func(t *testing.T) {
jobs := make([]*job, 9)
for i := 0; i < 9; i++ {
jobs[i] = &job{}
}
d := 20 * time.Millisecond
strategy := NewFullBusStrategy(d, 10)
ctx, cancel := context.WithCancel(context.Background())
startedAt := time.Now()
ch := strategy.GetJobs(ctx)
done := make(chan struct{})
go func() {
defer close(done)
iteration := <-ch
duration := time.Now().Sub(startedAt)
require.True(t, duration >= d, fmt.Sprintf("has %d, expected %d", duration, d))
require.True(t, duration < d*2)
require.Equal(t, jobs, iteration.Jobs)
require.Equal(t, 0, iteration.Queue)
}()
for _, j := range jobs {
strategy.Queue(j)
}
<-done
cancel()
})
t.Run("should provide iteration as soon as the bus is full, without waiting for the previous iteration to finish", func(t *testing.T) {
d := 20 * time.Millisecond
strategy := NewFullBusStrategy(d, 10)
ctx, cancel := context.WithCancel(context.Background())
ch := strategy.GetJobs(ctx)
done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 3; i++ {
time.Sleep(5 * time.Millisecond) // See comment below
select {
case iteration := <-ch:
require.Len(t, iteration.Jobs, 10)
// Don't assert iteration.Queue length since it might be unstable
// Don't call iteration.Done()
case <-time.After(d):
t.Fatalf("iteration should be provided as soon as the bus is full")
}
}
// Scheduled 31 tasks. 3 iterations should be performed immediately
// and should be executed only after timeout. The timeout above is used
// to increase overall time to ensure, that timer resets on every iteration
startedAt := time.Now()
iteration := <-ch
duration := time.Now().Sub(startedAt)
require.True(t, duration >= d)
require.True(t, duration < d*2)
require.Len(t, iteration.Jobs, 1)
require.Equal(t, 0, iteration.Queue)
}()
for i := 0; i < 31; i++ {
strategy.Queue(&job{})
}
<-done
cancel()
})
} }