diff --git a/.gitignore b/.gitignore index daf913b..97539af 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ _testmain.go *.exe *.test *.prof +http_to_nsqd \ No newline at end of file diff --git a/README.md b/README.md index bf0d814..2e2d141 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,34 @@ -# log_to_nsq +# http_to_nsq Publish http request to nsqd with custom topic. (for record logs) + +## Installation + +``` +$ go get github.com/0neSe7en/http_to_nsqd +``` + +## About + +An util for collect logs via HTTP. Send logs to different urls, and `http_to_nsq` will publish logs to different topics. + +## Usage + +- `host`(optional) - HTTP Address to listen (default: 127.0.0.1:3000) +- `nsqd` - NSQD Address for publish (default: 127.0.0.1:4150) +- `urltopic` - URL and Topic mapping. +- `param` - Param in GET or POST (default: result) + +## Example + +Start `http_to_nsq` with following command. + +``` +$ http_to_nsqd --urltopic /log:log +$ curl "http://127.0.0.1:3000/log?result=%7B%22message%22:%20%22it%20is%20a%20log%20for%20test%22%7D" +// {"message":{"message": "it is a log for test"},"timestamp":"2016-12-26T06:13:21.043Z","ip":"127.0.0.1"} +// This log is published to the "log" topic. +``` + +# License + +MIT \ No newline at end of file diff --git a/app.go b/app.go new file mode 100644 index 0000000..fed6f26 --- /dev/null +++ b/app.go @@ -0,0 +1,30 @@ +package main + +import ( + "github.com/jessevdk/go-flags" + "os" + "github.com/labstack/echo" + "github.com/labstack/echo/middleware" +) + +var Opts struct { + Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"` + HTTPAddress string `long:"host" description:"HTTP Address to listen" required:"true" default:"127.0.0.1:3000"` + NSQDAddress string `long:"nsqd" description:"NSQD Address for publish" required:"true" default:"127.0.0.1:4150"` + TopicMapping map[string]string `long:"urltopic" description:"URL and Topic mapping." required:"true"` + Param string `long:"param" description:"Param in GET or POST" required:"true" default:"result"` +} + +func main() { + _, err := flags.Parse(&Opts) + if err != nil { + os.Exit(1) + } + e := echo.New() + InitNsq(Opts.NSQDAddress) + e.Use(middleware.GzipWithConfig(middleware.GzipConfig{Level: 6})) + e.Use(middleware.Recover()) + e.POST("/*", Postlog, Validate, NsqPublish) + e.GET("/*", Getlog, Validate, NsqPublish) + e.Logger.Fatal(e.Start(Opts.HTTPAddress)) +} diff --git a/handler.go b/handler.go new file mode 100644 index 0000000..51b0dc9 --- /dev/null +++ b/handler.go @@ -0,0 +1,39 @@ +package main + +import ( +"encoding/json" +"github.com/labstack/echo" +"net/http" +) + +func Postlog(c echo.Context) error { + result := c.FormValue(Opts.Param) + if result == "" { + return c.NoContent(http.StatusBadRequest) + } + c.Set("logMessage", result) + return c.Blob(http.StatusOK, "image/gif", Gif) +} + +func Getlog(c echo.Context) error { + res := "" + result := c.QueryParam(Opts.Param) + if result != "" { + res = result + } else if r := c.QueryParams(); len(r) != 0 { + req := c.Request() + r["referer"] = []string{req.Referer()} + r["UA"] = []string{req.UserAgent()} + resByte, err := json.Marshal(r) + if err != nil { + return c.NoContent(http.StatusBadRequest) + } + res = string(resByte) + } + if res == "" { + return c.NoContent(http.StatusBadRequest) + } + c.Set("logMessage", res) + return c.Blob(http.StatusOK, "image/gif", Gif) +} + diff --git a/middlewares.go b/middlewares.go new file mode 100644 index 0000000..aa0598b --- /dev/null +++ b/middlewares.go @@ -0,0 +1,60 @@ +package main + +import ( + "github.com/labstack/echo" + "github.com/nsqio/go-nsq" + "github.com/labstack/gommon/log" +) + +var producer *nsq.Producer + +func InitNsq(addr string) { + cfg := nsq.NewConfig() + var err error + producer, err = nsq.NewProducer(addr, cfg) + if err != nil { + log.Fatalf("failed to create nsq.Producer - %s", err) + } +} + +func NsqPublish(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + defer func() { + topic, ok := c.Get("topic").(string) + if !ok { + log.Error("topic is not string.") + log.Error(topic) + return + } + messageStr, ok := c.Get("logMessage").(string) + processed := Process(messageStr, c.RealIP()) + message := []byte(processed) + if !ok { + log.Error("message is not string.") + log.Error(message) + return + } + err := producer.Publish(topic, message) + if err != nil { + log.Errorf("nsq send failed. %s", err) + } + }() + return next(c) + } +} + + +func Validate(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + + req := c.Request() + host := req.Host + uri := req.URL.Path + if topic, ok := Opts.TopicMapping[uri]; ok { + c.Set("topic", topic) + return next(c) + } + log.Infof("HOST=%s. 404 not found %s.", host, uri) + return echo.ErrNotFound + } +} \ No newline at end of file diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..785bcf2 --- /dev/null +++ b/utils.go @@ -0,0 +1,10 @@ +package main + +import "time" + +var Gif = []byte{0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x1, 0x0, 0x1, 0x0, 0x80, 0x0, 0x0, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x2c, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x2, 0x2, 0x44, 0x1, 0x0, 0x3b} + +func Process(l string, ip string) string { + return "{\"message\":" + l + ",\"timestamp\":\"" + time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + "\",\"ip\":\"" + ip + "\"}" +} +