From 5a0c10c1a1750f172878cb2bf35afb5680b7b907 Mon Sep 17 00:00:00 2001 From: ErickSkrauch Date: Fri, 3 Jan 2020 00:51:57 +0300 Subject: [PATCH] Implemented worker command --- .travis.yml | 2 +- bootstrap/bootstrap.go | 50 +++++-- cmd/root.go | 6 +- cmd/serve.go | 31 +--- cmd/version.go | 10 +- cmd/worker.go | 45 ++++++ http/uuids_worker.go | 89 +++++++++++ http/uuids_worker_test.go | 157 ++++++++++++++++++++ mojangtextures/mojang_textures.go | 6 +- mojangtextures/mojang_textures_test.go | 2 +- mojangtextures/remote_api_uuids_provider.go | 4 +- script/mocks | 4 - version/version.go | 14 ++ 13 files changed, 367 insertions(+), 53 deletions(-) create mode 100644 cmd/worker.go create mode 100644 http/uuids_worker.go create mode 100644 http/uuids_worker_test.go delete mode 100755 script/mocks create mode 100644 version/version.go diff --git a/.travis.yml b/.travis.yml index b91a909..339b236 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ jobs: env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o release/chrly - -ldflags '-extldflags "-static" -X github.com/elyby/chrly/bootstrap.version=$APP_VERSION' + -ldflags '-extldflags "-static" -X github.com/elyby/chrly/version.version=$APP_VERSION -X github.com/elyby/chrly/version.commit=$TRAVIS_COMMIT' main.go - docker build -t elyby/chrly:$DOCKER_TAG . - docker push elyby/chrly:$DOCKER_TAG diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index d8774ee..4455e17 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -1,7 +1,9 @@ package bootstrap import ( + "net/url" "os" + "time" "github.com/getsentry/raven-go" "github.com/mono83/slf/rays" @@ -9,24 +11,23 @@ import ( "github.com/mono83/slf/recievers/statsd" "github.com/mono83/slf/recievers/writer" "github.com/mono83/slf/wd" + "github.com/spf13/viper" + + "github.com/elyby/chrly/mojangtextures" + "github.com/elyby/chrly/version" ) -var version = "" - -func GetVersion() string { - return version -} - func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) { wd.AddReceiver(writer.New(writer.Options{ - Marker: false, + Marker: false, TimeFormat: "15:04:05.000", })) + if statsdAddr != "" { hostname, _ := os.Hostname() statsdReceiver, err := statsd.NewReceiver(statsd.Config{ - Address: statsdAddr, - Prefix: "ely.skinsystem." + hostname + ".app.", + Address: statsdAddr, + Prefix: "ely.skinsystem." + hostname + ".app.", FlushEvery: 1, }) @@ -45,7 +46,7 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) { ravenClient.SetEnvironment("production") ravenClient.SetDefaultLoggerName("sentry-watchdog-receiver") - programVersion := GetVersion() + programVersion := version.Version() if programVersion != "" { raven.SetRelease(programVersion) } @@ -62,3 +63,32 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) { return wd.New("", "").WithParams(rays.Host), nil } + +func init() { + viper.SetDefault("queue.loop_delay", 2*time.Second+500*time.Millisecond) + viper.SetDefault("queue.batch_size", 10) +} + +func CreateMojangUUIDsProvider(logger wd.Watchdog) (mojangtextures.UUIDsProvider, error) { + var uuidsProvider mojangtextures.UUIDsProvider + preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver") + if preferredUuidsProvider == "remote" { + remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url")) + if err != nil { + return nil, err + } + + uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{ + Url: *remoteUrl, + Logger: logger, + } + } else { + uuidsProvider = &mojangtextures.BatchUuidsProvider{ + IterationDelay: viper.GetDuration("queue.loop_delay"), + IterationSize: viper.GetInt("queue.batch_size"), + Logger: logger, + } + } + + return uuidsProvider, nil +} diff --git a/cmd/root.go b/cmd/root.go index 5b973bd..038161c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,16 +5,16 @@ import ( "os" "strings" - "github.com/elyby/chrly/bootstrap" - "github.com/spf13/cobra" "github.com/spf13/viper" + + "github.com/elyby/chrly/version" ) var RootCmd = &cobra.Command{ Use: "chrly", Short: "Implementation of Minecraft skins system server", - Version: bootstrap.GetVersion(), + Version: version.Version(), } // Execute adds all child commands to the root command and sets flags appropriately. diff --git a/cmd/serve.go b/cmd/serve.go index 481e219..da05120 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -3,8 +3,6 @@ package cmd import ( "fmt" "log" - "net/url" - "time" "github.com/mono83/slf/wd" "github.com/spf13/cobra" @@ -19,7 +17,7 @@ import ( var serveCmd = &cobra.Command{ Use: "serve", - Short: "Starts http handler for the skins system", + Short: "Starts HTTP handler for the skins system", Run: func(cmd *cobra.Command, args []string) { // TODO: this is a mess, need to organize this code somehow to make services initialization more compact logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn")) @@ -55,32 +53,17 @@ var serveCmd = &cobra.Command{ return } - var uuidsProvider mojangtextures.UuidsProvider - preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver") - if preferredUuidsProvider == "remote" { - remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url")) - if err != nil { - logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) - return - } - - uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{ - Url: *remoteUrl, - Logger: logger, - } - } else { - uuidsProvider = &mojangtextures.BatchUuidsProvider{ - IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond, - IterationSize: viper.GetInt("queue.batch_size"), - Logger: logger, - } + uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger) + if err != nil { + logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) + return } texturesStorage := mojangtextures.NewInMemoryTexturesStorage() texturesStorage.Start() mojangTexturesProvider := &mojangtextures.Provider{ Logger: logger, - UuidsProvider: uuidsProvider, + UUIDsProvider: uuidsProvider, TexturesProvider: &mojangtextures.MojangApiTexturesProvider{ Logger: logger, }, @@ -115,6 +98,4 @@ func init() { viper.SetDefault("storage.redis.poll", 10) viper.SetDefault("storage.filesystem.basePath", "data") viper.SetDefault("storage.filesystem.capesDirName", "capes") - viper.SetDefault("queue.loop_delay", 2_500) - viper.SetDefault("queue.batch_size", 10) } diff --git a/cmd/version.go b/cmd/version.go index e1196fe..59cd038 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -2,18 +2,20 @@ package cmd import ( "fmt" + "runtime" "github.com/spf13/cobra" - "github.com/elyby/chrly/bootstrap" - "runtime" + + "github.com/elyby/chrly/version" ) var versionCmd = &cobra.Command{ Use: "version", Short: "Show the Chrly version information", Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Version: %s\n", bootstrap.GetVersion()) - fmt.Printf("Go version: %s\n", runtime.Version()) + fmt.Printf("Version: %s\n", version.Version()) + fmt.Printf("Commit: %s\n", version.Commit()) + fmt.Printf("Go version: %s\n", runtime.Version()) fmt.Printf("OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) }, } diff --git a/cmd/worker.go b/cmd/worker.go new file mode 100644 index 0000000..13c1adb --- /dev/null +++ b/cmd/worker.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/mono83/slf/wd" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/elyby/chrly/bootstrap" + "github.com/elyby/chrly/http" +) + +var workerCmd = &cobra.Command{ + Use: "worker", + Short: "Starts HTTP handler for the Mojang usernames to UUIDs worker", + Run: func(cmd *cobra.Command, args []string) { + logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn")) + if err != nil { + log.Fatal(fmt.Printf("Cannot initialize logger: %v", err)) + } + logger.Info("Logger successfully initialized") + + uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger) + if err != nil { + logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err)) + return + } + + cfg := &http.UUIDsWorker{ + ListenSpec: fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")), + UUIDsProvider: uuidsProvider, + Logger: logger, + } + + if err := cfg.Run(); err != nil { + logger.Error(fmt.Sprintf("Error in main(): %v", err)) + } + }, +} + +func init() { + RootCmd.AddCommand(workerCmd) +} diff --git a/http/uuids_worker.go b/http/uuids_worker.go new file mode 100644 index 0000000..5d08df5 --- /dev/null +++ b/http/uuids_worker.go @@ -0,0 +1,89 @@ +package http + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/mono83/slf/wd" + + "github.com/elyby/chrly/api/mojang" + "github.com/elyby/chrly/mojangtextures" +) + +type UuidsProvider interface { + GetUuid(username string) (*mojang.ProfileInfo, error) +} + +type UUIDsWorker struct { + ListenSpec string + + UUIDsProvider mojangtextures.UUIDsProvider + Logger wd.Watchdog +} + +func (ctx *UUIDsWorker) Run() error { + ctx.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", ctx.ListenSpec)) + + listener, err := net.Listen("tcp", ctx.ListenSpec) + if err != nil { + return err + } + + server := &http.Server{ + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, // TODO: should I adjust this values? + MaxHeaderBytes: 1 << 16, + Handler: ctx.CreateHandler(), + } + + // noinspection GoUnhandledErrorResult + go server.Serve(listener) + + s := waitForSignal() + ctx.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s)) + + return nil +} + +func (ctx *UUIDsWorker) CreateHandler() http.Handler { + router := mux.NewRouter().StrictSlash(true) + router.NotFoundHandler = http.HandlerFunc(NotFound) + + router.Handle("/api/worker/mojang-uuid/{username}", http.HandlerFunc(ctx.GetUUID)).Methods("GET") + + return router +} + +func (ctx *UUIDsWorker) GetUUID(response http.ResponseWriter, request *http.Request) { + username := parseUsername(mux.Vars(request)["username"]) + profile, err := ctx.UUIDsProvider.GetUuid(username) + if err != nil { + if _, ok := err.(*mojang.TooManyRequestsError); ok { + ctx.Logger.Warning("Got 429 Too Many Requests") + response.WriteHeader(http.StatusTooManyRequests) + return + } + + ctx.Logger.Warning("Got non success response: :err", wd.ErrParam(err)) + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusInternalServerError) + result, _ := json.Marshal(map[string]interface{}{ + "provider": err.Error(), + }) + _, _ = response.Write(result) + return + } + + if profile == nil { + response.WriteHeader(http.StatusNoContent) + return + } + + response.Header().Set("Content-Type", "application/json") + responseData, _ := json.Marshal(profile) + _, _ = response.Write(responseData) +} diff --git a/http/uuids_worker_test.go b/http/uuids_worker_test.go new file mode 100644 index 0000000..ea0e64e --- /dev/null +++ b/http/uuids_worker_test.go @@ -0,0 +1,157 @@ +package http + +import ( + "errors" + "github.com/elyby/chrly/api/mojang" + "github.com/elyby/chrly/tests" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +/*************** + * Setup mocks * + ***************/ + +type uuidsProviderMock struct { + mock.Mock +} + +func (m *uuidsProviderMock) GetUuid(username string) (*mojang.ProfileInfo, error) { + args := m.Called(username) + var result *mojang.ProfileInfo + if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok { + result = casted + } + + return result, args.Error(1) +} + +type uuidsWorkerTestSuite struct { + suite.Suite + + App *UUIDsWorker + + UuidsProvider *uuidsProviderMock + Logger *tests.WdMock +} + +/******************** + * Setup test suite * + ********************/ + +func (suite *uuidsWorkerTestSuite) SetupTest() { + suite.UuidsProvider = &uuidsProviderMock{} + suite.Logger = &tests.WdMock{} + + suite.App = &UUIDsWorker{ + UUIDsProvider: suite.UuidsProvider, + Logger: suite.Logger, + } +} + +func (suite *uuidsWorkerTestSuite) TearDownTest() { + suite.UuidsProvider.AssertExpectations(suite.T()) + suite.Logger.AssertExpectations(suite.T()) +} + +func (suite *uuidsWorkerTestSuite) RunSubTest(name string, subTest func()) { + suite.SetupTest() + suite.Run(name, subTest) + suite.TearDownTest() +} + +/************* + * Run tests * + *************/ + +func TestUUIDsWorker(t *testing.T) { + suite.Run(t, new(uuidsWorkerTestSuite)) +} + +type uuidsWorkerTestCase struct { + Name string + BeforeTest func(suite *uuidsWorkerTestSuite) + AfterTest func(suite *uuidsWorkerTestSuite, response *http.Response) +} + +/************************ + * Get UUID tests cases * + ************************/ + +var getUuidTestsCases = []*uuidsWorkerTestCase{ + { + Name: "Success provider response", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(&mojang.ProfileInfo{ + Id: "0fcc38620f1845f3a54e1b523c1bd1c7", + Name: "mock_username", + }, nil) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(200, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "id": "0fcc38620f1845f3a54e1b523c1bd1c7", + "name": "mock_username" + }`, string(body)) + }, + }, + { + Name: "Receive empty response from UUIDs provider", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, nil) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(204, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Assert().Empty(body) + }, + }, + { + Name: "Receive error from UUIDs provider", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, errors.New("this is an error")) + suite.Logger.On("Warning", "Got non success response: :err", mock.Anything).Times(1) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(500, response.StatusCode) + suite.Equal("application/json", response.Header.Get("Content-Type")) + body, _ := ioutil.ReadAll(response.Body) + suite.JSONEq(`{ + "provider": "this is an error" + }`, string(body)) + }, + }, + { + Name: "Receive Too Many Requests from UUIDs provider", + BeforeTest: func(suite *uuidsWorkerTestSuite) { + suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, &mojang.TooManyRequestsError{}) + suite.Logger.On("Warning", "Got 429 Too Many Requests").Times(1) + }, + AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) { + suite.Equal(429, response.StatusCode) + body, _ := ioutil.ReadAll(response.Body) + suite.Empty(body) + }, + }, +} + +func (suite *uuidsWorkerTestSuite) TestGetUUID() { + for _, testCase := range getUuidTestsCases { + suite.RunSubTest(testCase.Name, func() { + testCase.BeforeTest(suite) + + req := httptest.NewRequest("GET", "http://chrly/api/worker/mojang-uuid/mock_username", nil) + w := httptest.NewRecorder() + + suite.App.CreateHandler().ServeHTTP(w, req) + + testCase.AfterTest(suite, w.Result()) + }) + } +} diff --git a/mojangtextures/mojang_textures.go b/mojangtextures/mojang_textures.go index 33919c6..2212cd7 100644 --- a/mojangtextures/mojang_textures.go +++ b/mojangtextures/mojang_textures.go @@ -69,7 +69,7 @@ func (c *broadcaster) BroadcastAndRemove(username string, result *broadcastResul // https://help.mojang.com/customer/portal/articles/928638 var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`) -type UuidsProvider interface { +type UUIDsProvider interface { GetUuid(username string) (*mojang.ProfileInfo, error) } @@ -78,7 +78,7 @@ type TexturesProvider interface { } type Provider struct { - UuidsProvider + UUIDsProvider TexturesProvider Storage Logger wd.Watchdog @@ -139,7 +139,7 @@ func (ctx *Provider) getResultAndBroadcast(username string, uuid string) { func (ctx *Provider) getResult(username string, uuid string) *broadcastResult { if uuid == "" { - profile, err := ctx.UuidsProvider.GetUuid(username) + profile, err := ctx.UUIDsProvider.GetUuid(username) if err != nil { ctx.handleMojangApiResponseError(err, "usernames") return &broadcastResult{nil, err} diff --git a/mojangtextures/mojang_textures_test.go b/mojangtextures/mojang_textures_test.go index fabdee6..e9c2ce2 100644 --- a/mojangtextures/mojang_textures_test.go +++ b/mojangtextures/mojang_textures_test.go @@ -158,7 +158,7 @@ func (suite *providerTestSuite) SetupTest() { suite.Logger = &mocks.WdMock{} suite.Provider = &Provider{ - UuidsProvider: suite.UuidsProvider, + UUIDsProvider: suite.UuidsProvider, TexturesProvider: suite.TexturesProvider, Storage: suite.Storage, Logger: suite.Logger, diff --git a/mojangtextures/remote_api_uuids_provider.go b/mojangtextures/remote_api_uuids_provider.go index 0980176..4d05b49 100644 --- a/mojangtextures/remote_api_uuids_provider.go +++ b/mojangtextures/remote_api_uuids_provider.go @@ -2,6 +2,7 @@ package mojangtextures import ( "encoding/json" + "github.com/elyby/chrly/version" "io/ioutil" "net/http" . "net/url" @@ -11,7 +12,6 @@ import ( "github.com/mono83/slf/wd" "github.com/elyby/chrly/api/mojang" - "github.com/elyby/chrly/bootstrap" ) var HttpClient = &http.Client{ @@ -34,7 +34,7 @@ func (ctx *RemoteApiUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo request, _ := http.NewRequest("GET", url.String(), nil) request.Header.Add("Accept", "application/json") // Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint - request.Header.Add("User-Agent", "Chrly/"+bootstrap.GetVersion()) + request.Header.Add("User-Agent", "Chrly/"+version.Version()) start := time.Now() response, err := HttpClient.Do(request) diff --git a/script/mocks b/script/mocks deleted file mode 100755 index 66a61ef..0000000 --- a/script/mocks +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -mockgen -source=interfaces/repositories.go -destination=interfaces/mock_interfaces/mock_interfaces.go -mockgen -source=interfaces/auth.go -destination=interfaces/mock_interfaces/mock_auth.go diff --git a/version/version.go b/version/version.go new file mode 100644 index 0000000..89e6def --- /dev/null +++ b/version/version.go @@ -0,0 +1,14 @@ +package version + +var ( + version = "" + commit = "" +) + +func Version() string { + return version +} + +func Commit() string { + return commit +}