Implemented worker command

.travis.yml

@@ -31,7 +31,7 @@ jobs:
         env CGO_ENABLED=0 GOOS=linux GOARCH=amd64
         go build
         -o release/chrly
-        -ldflags '-extldflags "-static" -X github.com/elyby/chrly/bootstrap.version=$APP_VERSION'
+        -ldflags '-extldflags "-static" -X github.com/elyby/chrly/version.version=$APP_VERSION -X github.com/elyby/chrly/version.commit=$TRAVIS_COMMIT'
         main.go
       - docker build -t elyby/chrly:$DOCKER_TAG .
       - docker push elyby/chrly:$DOCKER_TAG

bootstrap/bootstrap.go

@@ -1,7 +1,9 @@
 package bootstrap
 
 import (
+	"net/url"
 	"os"
+	"time"
 
 	"github.com/getsentry/raven-go"
 	"github.com/mono83/slf/rays"
@@ -9,24 +11,23 @@ import (
 	"github.com/mono83/slf/recievers/statsd"
 	"github.com/mono83/slf/recievers/writer"
 	"github.com/mono83/slf/wd"
+	"github.com/spf13/viper"
+
+	"github.com/elyby/chrly/mojangtextures"
+	"github.com/elyby/chrly/version"
 )
 
-var version = ""
-
-func GetVersion() string {
-	return version
-}
-
 func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) {
 	wd.AddReceiver(writer.New(writer.Options{
-		Marker: false,
+		Marker:     false,
 		TimeFormat: "15:04:05.000",
 	}))
+
 	if statsdAddr != "" {
 		hostname, _ := os.Hostname()
 		statsdReceiver, err := statsd.NewReceiver(statsd.Config{
-			Address: statsdAddr,
+			Address:    statsdAddr,
-			Prefix: "ely.skinsystem." + hostname + ".app.",
+			Prefix:     "ely.skinsystem." + hostname + ".app.",
 			FlushEvery: 1,
 		})
 
@@ -45,7 +46,7 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) {
 
 		ravenClient.SetEnvironment("production")
 		ravenClient.SetDefaultLoggerName("sentry-watchdog-receiver")
-		programVersion := GetVersion()
+		programVersion := version.Version()
 		if programVersion != "" {
 			raven.SetRelease(programVersion)
 		}
@@ -62,3 +63,32 @@ func CreateLogger(statsdAddr string, sentryAddr string) (wd.Watchdog, error) {
 
 	return wd.New("", "").WithParams(rays.Host), nil
 }
+
+func init() {
+	viper.SetDefault("queue.loop_delay", 2*time.Second+500*time.Millisecond)
+	viper.SetDefault("queue.batch_size", 10)
+}
+
+func CreateMojangUUIDsProvider(logger wd.Watchdog) (mojangtextures.UUIDsProvider, error) {
+	var uuidsProvider mojangtextures.UUIDsProvider
+	preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver")
+	if preferredUuidsProvider == "remote" {
+		remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url"))
+		if err != nil {
+			return nil, err
+		}
+
+		uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{
+			Url:    *remoteUrl,
+			Logger: logger,
+		}
+	} else {
+		uuidsProvider = &mojangtextures.BatchUuidsProvider{
+			IterationDelay: viper.GetDuration("queue.loop_delay"),
+			IterationSize:  viper.GetInt("queue.batch_size"),
+			Logger:         logger,
+		}
+	}
+
+	return uuidsProvider, nil
+}

cmd/root.go

@@ -5,16 +5,16 @@ import (
 	"os"
 	"strings"
 
-	"github.com/elyby/chrly/bootstrap"
-
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/elyby/chrly/version"
 )
 
 var RootCmd = &cobra.Command{
 	Use:     "chrly",
 	Short:   "Implementation of Minecraft skins system server",
-	Version: bootstrap.GetVersion(),
+	Version: version.Version(),
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.

cmd/serve.go

@@ -3,8 +3,6 @@ package cmd
 import (
 	"fmt"
 	"log"
-	"net/url"
-	"time"
 
 	"github.com/mono83/slf/wd"
 	"github.com/spf13/cobra"
@@ -19,7 +17,7 @@ import (
 
 var serveCmd = &cobra.Command{
 	Use:   "serve",
-	Short: "Starts http handler for the skins system",
+	Short: "Starts HTTP handler for the skins system",
 	Run: func(cmd *cobra.Command, args []string) {
 		// TODO: this is a mess, need to organize this code somehow to make services initialization more compact
 		logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn"))
@@ -55,32 +53,17 @@ var serveCmd = &cobra.Command{
 			return
 		}
 
-		var uuidsProvider mojangtextures.UuidsProvider
+		uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger)
-		preferredUuidsProvider := viper.GetString("mojang_textures.uuids_provider.driver")
+		if err != nil {
-		if preferredUuidsProvider == "remote" {
+			logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err))
-			remoteUrl, err := url.Parse(viper.GetString("mojang_textures.uuids_provider.url"))
+			return
-			if err != nil {
-				logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err))
-				return
-			}
-
-			uuidsProvider = &mojangtextures.RemoteApiUuidsProvider{
-				Url:    *remoteUrl,
-				Logger: logger,
-			}
-		} else {
-			uuidsProvider = &mojangtextures.BatchUuidsProvider{
-				IterationDelay: time.Duration(viper.GetInt("queue.loop_delay")) * time.Millisecond,
-				IterationSize:  viper.GetInt("queue.batch_size"),
-				Logger:         logger,
-			}
 		}
 
 		texturesStorage := mojangtextures.NewInMemoryTexturesStorage()
 		texturesStorage.Start()
 		mojangTexturesProvider := &mojangtextures.Provider{
 			Logger:        logger,
-			UuidsProvider: uuidsProvider,
+			UUIDsProvider: uuidsProvider,
 			TexturesProvider: &mojangtextures.MojangApiTexturesProvider{
 				Logger: logger,
 			},
@@ -115,6 +98,4 @@ func init() {
 	viper.SetDefault("storage.redis.poll", 10)
 	viper.SetDefault("storage.filesystem.basePath", "data")
 	viper.SetDefault("storage.filesystem.capesDirName", "capes")
-	viper.SetDefault("queue.loop_delay", 2_500)
-	viper.SetDefault("queue.batch_size", 10)
 }

cmd/version.go

@@ -2,18 +2,20 @@ package cmd
 
 import (
 	"fmt"
+	"runtime"
 
 	"github.com/spf13/cobra"
-	"github.com/elyby/chrly/bootstrap"
+
-	"runtime"
+	"github.com/elyby/chrly/version"
 )
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Show the Chrly version information",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Printf("Version:    %s\n",    bootstrap.GetVersion())
+		fmt.Printf("Version:    %s\n", version.Version())
-		fmt.Printf("Go version: %s\n",    runtime.Version())
+		fmt.Printf("Commit:     %s\n", version.Commit())
+		fmt.Printf("Go version: %s\n", runtime.Version())
 		fmt.Printf("OS/Arch:    %s/%s\n", runtime.GOOS, runtime.GOARCH)
 	},
 }

cmd/worker.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/mono83/slf/wd"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/elyby/chrly/bootstrap"
+	"github.com/elyby/chrly/http"
+)
+
+var workerCmd = &cobra.Command{
+	Use:   "worker",
+	Short: "Starts HTTP handler for the Mojang usernames to UUIDs worker",
+	Run: func(cmd *cobra.Command, args []string) {
+		logger, err := bootstrap.CreateLogger(viper.GetString("statsd.addr"), viper.GetString("sentry.dsn"))
+		if err != nil {
+			log.Fatal(fmt.Printf("Cannot initialize logger: %v", err))
+		}
+		logger.Info("Logger successfully initialized")
+
+		uuidsProvider, err := bootstrap.CreateMojangUUIDsProvider(logger)
+		if err != nil {
+			logger.Emergency("Unable to parse remote url :err", wd.ErrParam(err))
+			return
+		}
+
+		cfg := &http.UUIDsWorker{
+			ListenSpec:    fmt.Sprintf("%s:%d", viper.GetString("server.host"), viper.GetInt("server.port")),
+			UUIDsProvider: uuidsProvider,
+			Logger:        logger,
+		}
+
+		if err := cfg.Run(); err != nil {
+			logger.Error(fmt.Sprintf("Error in main(): %v", err))
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(workerCmd)
+}

http/uuids_worker.go

@@ -0,0 +1,89 @@
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/mono83/slf/wd"
+
+	"github.com/elyby/chrly/api/mojang"
+	"github.com/elyby/chrly/mojangtextures"
+)
+
+type UuidsProvider interface {
+	GetUuid(username string) (*mojang.ProfileInfo, error)
+}
+
+type UUIDsWorker struct {
+	ListenSpec string
+
+	UUIDsProvider mojangtextures.UUIDsProvider
+	Logger        wd.Watchdog
+}
+
+func (ctx *UUIDsWorker) Run() error {
+	ctx.Logger.Info(fmt.Sprintf("Starting, HTTP on: %s\n", ctx.ListenSpec))
+
+	listener, err := net.Listen("tcp", ctx.ListenSpec)
+	if err != nil {
+		return err
+	}
+
+	server := &http.Server{
+		ReadTimeout:    60 * time.Second,
+		WriteTimeout:   60 * time.Second, // TODO: should I adjust this values?
+		MaxHeaderBytes: 1 << 16,
+		Handler:        ctx.CreateHandler(),
+	}
+
+	// noinspection GoUnhandledErrorResult
+	go server.Serve(listener)
+
+	s := waitForSignal()
+	ctx.Logger.Info(fmt.Sprintf("Got signal: %v, exiting.", s))
+
+	return nil
+}
+
+func (ctx *UUIDsWorker) CreateHandler() http.Handler {
+	router := mux.NewRouter().StrictSlash(true)
+	router.NotFoundHandler = http.HandlerFunc(NotFound)
+
+	router.Handle("/api/worker/mojang-uuid/{username}", http.HandlerFunc(ctx.GetUUID)).Methods("GET")
+
+	return router
+}
+
+func (ctx *UUIDsWorker) GetUUID(response http.ResponseWriter, request *http.Request) {
+	username := parseUsername(mux.Vars(request)["username"])
+	profile, err := ctx.UUIDsProvider.GetUuid(username)
+	if err != nil {
+		if _, ok := err.(*mojang.TooManyRequestsError); ok {
+			ctx.Logger.Warning("Got 429 Too Many Requests")
+			response.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+
+		ctx.Logger.Warning("Got non success response: :err", wd.ErrParam(err))
+		response.Header().Set("Content-Type", "application/json")
+		response.WriteHeader(http.StatusInternalServerError)
+		result, _ := json.Marshal(map[string]interface{}{
+			"provider": err.Error(),
+		})
+		_, _ = response.Write(result)
+		return
+	}
+
+	if profile == nil {
+		response.WriteHeader(http.StatusNoContent)
+		return
+	}
+
+	response.Header().Set("Content-Type", "application/json")
+	responseData, _ := json.Marshal(profile)
+	_, _ = response.Write(responseData)
+}

http/uuids_worker_test.go

@@ -0,0 +1,157 @@
+package http
+
+import (
+	"errors"
+	"github.com/elyby/chrly/api/mojang"
+	"github.com/elyby/chrly/tests"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+/***************
+ * Setup mocks *
+ ***************/
+
+type uuidsProviderMock struct {
+	mock.Mock
+}
+
+func (m *uuidsProviderMock) GetUuid(username string) (*mojang.ProfileInfo, error) {
+	args := m.Called(username)
+	var result *mojang.ProfileInfo
+	if casted, ok := args.Get(0).(*mojang.ProfileInfo); ok {
+		result = casted
+	}
+
+	return result, args.Error(1)
+}
+
+type uuidsWorkerTestSuite struct {
+	suite.Suite
+
+	App *UUIDsWorker
+
+	UuidsProvider *uuidsProviderMock
+	Logger        *tests.WdMock
+}
+
+/********************
+ * Setup test suite *
+ ********************/
+
+func (suite *uuidsWorkerTestSuite) SetupTest() {
+	suite.UuidsProvider = &uuidsProviderMock{}
+	suite.Logger = &tests.WdMock{}
+
+	suite.App = &UUIDsWorker{
+		UUIDsProvider: suite.UuidsProvider,
+		Logger:        suite.Logger,
+	}
+}
+
+func (suite *uuidsWorkerTestSuite) TearDownTest() {
+	suite.UuidsProvider.AssertExpectations(suite.T())
+	suite.Logger.AssertExpectations(suite.T())
+}
+
+func (suite *uuidsWorkerTestSuite) RunSubTest(name string, subTest func()) {
+	suite.SetupTest()
+	suite.Run(name, subTest)
+	suite.TearDownTest()
+}
+
+/*************
+ * Run tests *
+ *************/
+
+func TestUUIDsWorker(t *testing.T) {
+	suite.Run(t, new(uuidsWorkerTestSuite))
+}
+
+type uuidsWorkerTestCase struct {
+	Name       string
+	BeforeTest func(suite *uuidsWorkerTestSuite)
+	AfterTest  func(suite *uuidsWorkerTestSuite, response *http.Response)
+}
+
+/************************
+ * Get UUID tests cases *
+ ************************/
+
+var getUuidTestsCases = []*uuidsWorkerTestCase{
+	{
+		Name: "Success provider response",
+		BeforeTest: func(suite *uuidsWorkerTestSuite) {
+			suite.UuidsProvider.On("GetUuid", "mock_username").Return(&mojang.ProfileInfo{
+				Id:   "0fcc38620f1845f3a54e1b523c1bd1c7",
+				Name: "mock_username",
+			}, nil)
+		},
+		AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
+			suite.Equal(200, response.StatusCode)
+			suite.Equal("application/json", response.Header.Get("Content-Type"))
+			body, _ := ioutil.ReadAll(response.Body)
+			suite.JSONEq(`{
+				"id": "0fcc38620f1845f3a54e1b523c1bd1c7",
+				"name": "mock_username"
+			}`, string(body))
+		},
+	},
+	{
+		Name: "Receive empty response from UUIDs provider",
+		BeforeTest: func(suite *uuidsWorkerTestSuite) {
+			suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, nil)
+		},
+		AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
+			suite.Equal(204, response.StatusCode)
+			body, _ := ioutil.ReadAll(response.Body)
+			suite.Assert().Empty(body)
+		},
+	},
+	{
+		Name: "Receive error from UUIDs provider",
+		BeforeTest: func(suite *uuidsWorkerTestSuite) {
+			suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, errors.New("this is an error"))
+			suite.Logger.On("Warning", "Got non success response: :err", mock.Anything).Times(1)
+		},
+		AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
+			suite.Equal(500, response.StatusCode)
+			suite.Equal("application/json", response.Header.Get("Content-Type"))
+			body, _ := ioutil.ReadAll(response.Body)
+			suite.JSONEq(`{
+				"provider": "this is an error"
+			}`, string(body))
+		},
+	},
+	{
+		Name: "Receive Too Many Requests from UUIDs provider",
+		BeforeTest: func(suite *uuidsWorkerTestSuite) {
+			suite.UuidsProvider.On("GetUuid", "mock_username").Return(nil, &mojang.TooManyRequestsError{})
+			suite.Logger.On("Warning", "Got 429 Too Many Requests").Times(1)
+		},
+		AfterTest: func(suite *uuidsWorkerTestSuite, response *http.Response) {
+			suite.Equal(429, response.StatusCode)
+			body, _ := ioutil.ReadAll(response.Body)
+			suite.Empty(body)
+		},
+	},
+}
+
+func (suite *uuidsWorkerTestSuite) TestGetUUID() {
+	for _, testCase := range getUuidTestsCases {
+		suite.RunSubTest(testCase.Name, func() {
+			testCase.BeforeTest(suite)
+
+			req := httptest.NewRequest("GET", "http://chrly/api/worker/mojang-uuid/mock_username", nil)
+			w := httptest.NewRecorder()
+
+			suite.App.CreateHandler().ServeHTTP(w, req)
+
+			testCase.AfterTest(suite, w.Result())
+		})
+	}
+}

mojangtextures/mojang_textures.go

@@ -69,7 +69,7 @@ func (c *broadcaster) BroadcastAndRemove(username string, result *broadcastResul
 // https://help.mojang.com/customer/portal/articles/928638
 var allowedUsernamesRegex = regexp.MustCompile(`^[\w_]{3,16}$`)
 
-type UuidsProvider interface {
+type UUIDsProvider interface {
 	GetUuid(username string) (*mojang.ProfileInfo, error)
 }
 
@@ -78,7 +78,7 @@ type TexturesProvider interface {
 }
 
 type Provider struct {
-	UuidsProvider
+	UUIDsProvider
 	TexturesProvider
 	Storage
 	Logger wd.Watchdog
@@ -139,7 +139,7 @@ func (ctx *Provider) getResultAndBroadcast(username string, uuid string) {
 
 func (ctx *Provider) getResult(username string, uuid string) *broadcastResult {
 	if uuid == "" {
-		profile, err := ctx.UuidsProvider.GetUuid(username)
+		profile, err := ctx.UUIDsProvider.GetUuid(username)
 		if err != nil {
 			ctx.handleMojangApiResponseError(err, "usernames")
 			return &broadcastResult{nil, err}

mojangtextures/mojang_textures_test.go

@@ -158,7 +158,7 @@ func (suite *providerTestSuite) SetupTest() {
 	suite.Logger = &mocks.WdMock{}
 
 	suite.Provider = &Provider{
-		UuidsProvider:    suite.UuidsProvider,
+		UUIDsProvider:    suite.UuidsProvider,
 		TexturesProvider: suite.TexturesProvider,
 		Storage:          suite.Storage,
 		Logger:           suite.Logger,

mojangtextures/remote_api_uuids_provider.go

@@ -2,6 +2,7 @@ package mojangtextures
 
 import (
 	"encoding/json"
+	"github.com/elyby/chrly/version"
 	"io/ioutil"
 	"net/http"
 	. "net/url"
@@ -11,7 +12,6 @@ import (
 	"github.com/mono83/slf/wd"
 
 	"github.com/elyby/chrly/api/mojang"
-	"github.com/elyby/chrly/bootstrap"
 )
 
 var HttpClient = &http.Client{
@@ -34,7 +34,7 @@ func (ctx *RemoteApiUuidsProvider) GetUuid(username string) (*mojang.ProfileInfo
 	request, _ := http.NewRequest("GET", url.String(), nil)
 	request.Header.Add("Accept", "application/json")
 	// Change default User-Agent to allow specify "Username -> UUID at time" Mojang's api endpoint
-	request.Header.Add("User-Agent", "Chrly/"+bootstrap.GetVersion())
+	request.Header.Add("User-Agent", "Chrly/"+version.Version())
 
 	start := time.Now()
 	response, err := HttpClient.Do(request)

script/mocks

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-mockgen -source=interfaces/repositories.go -destination=interfaces/mock_interfaces/mock_interfaces.go
-mockgen -source=interfaces/auth.go -destination=interfaces/mock_interfaces/mock_auth.go

version/version.go

@@ -0,0 +1,14 @@
+package version
+
+var (
+	version = ""
+	commit  = ""
+)
+
+func Version() string {
+	return version
+}
+
+func Commit() string {
+	return commit
+}