From 6d6d0e4b79fc17ba25f0cbb24e39b88b5dd8d0c4 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Thu, 3 Oct 2019 01:00:27 +0300 Subject: [PATCH] Decrease queue batch size. Log all 400 response from the Mojang's API. Resolves #10. --- CHANGELOG.md | 3 +++ api/mojang/queue/queue.go | 8 +++++++- api/mojang/queue/queue_test.go | 32 +++++++++++++++++--------------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b9a8c0..0545e6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] - xxxx-xx-xx +### Changed +- Mojang's textures queue batch size [reduced to 10](https://wiki.vg/index.php?title=Mojang_API&type=revision&diff=14964&oldid=14954). +- 400 BadRequest errors from the Mojang's API are now logged. ## [4.2.2] - 2019-06-19 ### Fixed diff --git a/api/mojang/queue/queue.go b/api/mojang/queue/queue.go index ff9900d..3eb4d8e 100644 --- a/api/mojang/queue/queue.go +++ b/api/mojang/queue/queue.go @@ -114,7 +114,7 @@ func (ctx *JobsQueue) queueRound() { } queueSize := ctx.queue.Size() - jobs := ctx.queue.Dequeue(100) + jobs := ctx.queue.Dequeue(10) ctx.Logger.UpdateGauge("mojang_textures.usernames.iteration_size", int64(len(jobs))) ctx.Logger.UpdateGauge("mojang_textures.usernames.queue_size", int64(queueSize-len(jobs))) var usernames []string @@ -184,6 +184,12 @@ func (ctx *JobsQueue) handleResponseError(err error, threadName string) { case mojang.ResponseError: if _, ok := err.(*mojang.TooManyRequestsError); ok { ctx.Logger.Warning(":name: Got 429 Too Many Requests :err", wd.NameParam(threadName), wd.ErrParam(err)) + return + } + + if _, ok := err.(*mojang.BadRequestError); ok { + ctx.Logger.Warning(":name: Got 400 Bad Request :err", wd.NameParam(threadName), wd.ErrParam(err)) + return } return diff --git a/api/mojang/queue/queue_test.go b/api/mojang/queue/queue_test.go index 8864df2..6c1e5d6 100644 --- a/api/mojang/queue/queue_test.go +++ b/api/mojang/queue/queue_test.go @@ -258,30 +258,30 @@ func (suite *queueTestSuite) TestReceiveTexturesForUsernameWithCachedUnknownUuid suite.Assert().Nil(<-resultChan) } -func (suite *queueTestSuite) TestReceiveTexturesForMoreThan100Usernames() { - usernames := make([]string, 120) - for i := 0; i < 120; i++ { +func (suite *queueTestSuite) TestReceiveTexturesForMoreThan10Usernames() { + usernames := make([]string, 12) + for i := 0; i < cap(usernames); i++ { usernames[i] = randStr(8) } - suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(120) - suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(120) - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(100)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(20)).Once() - suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(20)).Once() + suite.Logger.On("IncCounter", "mojang_textures.request", int64(1)).Times(12) + suite.Logger.On("IncCounter", "mojang_textures.usernames.queued", int64(1)).Times(12) + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(10)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.iteration_size", int64(2)).Once() + suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(2)).Once() suite.Logger.On("UpdateGauge", "mojang_textures.usernames.queue_size", int64(0)).Once() suite.Logger.On("RecordTimer", "mojang_textures.usernames.round_time", mock.Anything).Twice() - suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(120) - suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(120) + suite.Logger.On("IncCounter", "mojang_textures.usernames.uuid_miss", int64(1)).Times(12) + suite.Logger.On("RecordTimer", "mojang_textures.result_time", mock.Anything).Times(12) - suite.Storage.On("GetUuid", mock.Anything).Times(120).Return("", &ValueNotFound{}) - suite.Storage.On("StoreUuid", mock.Anything, "").Times(120).Return(nil) // should be called with "" if username is not compared to uuid + suite.Storage.On("GetUuid", mock.Anything).Times(12).Return("", &ValueNotFound{}) + suite.Storage.On("StoreUuid", mock.Anything, "").Times(12).Return(nil) // should be called with "" if username is not compared to uuid // Storage.GetTextures and Storage.SetTextures shouldn't be called - suite.MojangApi.On("UsernamesToUuids", usernames[0:100]).Once().Return([]*mojang.ProfileInfo{}, nil) - suite.MojangApi.On("UsernamesToUuids", usernames[100:120]).Once().Return([]*mojang.ProfileInfo{}, nil) + suite.MojangApi.On("UsernamesToUuids", usernames[0:10]).Once().Return([]*mojang.ProfileInfo{}, nil) + suite.MojangApi.On("UsernamesToUuids", usernames[10:12]).Once().Return([]*mojang.ProfileInfo{}, nil) - channels := make([]chan *mojang.SignedTexturesResponse, 120) + channels := make([]chan *mojang.SignedTexturesResponse, 12) for i, username := range usernames { channels[i] = suite.Queue.GetTexturesForUsername(username) } @@ -417,6 +417,7 @@ func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromU suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) + suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{}) @@ -453,6 +454,7 @@ func (suite *queueTestSuite) TestShouldNotLogErrorWhenExpectedErrorReturnedFromU suite.Logger.On("UpdateGauge", mock.Anything, mock.Anything) suite.Logger.On("RecordTimer", mock.Anything, mock.Anything) suite.Logger.On("Debug", ":name: Got response error :err", mock.Anything, mock.Anything).Times(len(expectedErrors)) + suite.Logger.On("Warning", ":name: Got 400 Bad Request :err", mock.Anything, mock.Anything).Once() suite.Logger.On("Warning", ":name: Got 429 Too Many Requests :err", mock.Anything, mock.Anything).Once() suite.Storage.On("GetUuid", "maksimkurb").Return("", &ValueNotFound{})