init commit.

.gitignore

@@ -22,3 +22,4 @@ _testmain.go
 *.exe
 *.test
 *.prof
+http_to_nsqd

README.md

@@ -1,2 +1,34 @@
-# log_to_nsq
+# http_to_nsq
 Publish http request to nsqd with custom topic. (for record logs)
+
+## Installation
+
+```
+$ go get github.com/0neSe7en/http_to_nsqd
+```
+
+## About
+
+An util for collect logs via HTTP. Send logs to different urls, and `http_to_nsq` will publish logs to different topics.
+
+## Usage
+
+- `host`(optional) - HTTP Address to listen (default: 127.0.0.1:3000)
+- `nsqd` - NSQD Address for publish (default: 127.0.0.1:4150)
+- `urltopic` - URL and Topic mapping.
+- `param` - Param in GET or POST (default: result)
+
+## Example
+
+Start `http_to_nsq` with following command.
+
+```
+$ http_to_nsqd --urltopic /log:log
+$ curl "http://127.0.0.1:3000/log?result=%7B%22message%22:%20%22it%20is%20a%20log%20for%20test%22%7D"
+// {"message":{"message": "it is a log for test"},"timestamp":"2016-12-26T06:13:21.043Z","ip":"127.0.0.1"}
+// This log is published to the "log" topic.
+```
+
+# License
+
+MIT

app.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"github.com/jessevdk/go-flags"
+	"os"
+	"github.com/labstack/echo"
+	"github.com/labstack/echo/middleware"
+)
+
+var Opts struct {
+	Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
+	HTTPAddress string `long:"host" description:"HTTP Address to listen" required:"true" default:"127.0.0.1:3000"`
+	NSQDAddress string `long:"nsqd" description:"NSQD Address for publish" required:"true" default:"127.0.0.1:4150"`
+	TopicMapping map[string]string `long:"urltopic" description:"URL and Topic mapping." required:"true"`
+	Param string `long:"param" description:"Param in GET or POST" required:"true" default:"result"`
+}
+
+func main() {
+	_, err := flags.Parse(&Opts)
+	if err != nil {
+		os.Exit(1)
+	}
+	e := echo.New()
+	InitNsq(Opts.NSQDAddress)
+	e.Use(middleware.GzipWithConfig(middleware.GzipConfig{Level: 6}))
+	e.Use(middleware.Recover())
+	e.POST("/*", Postlog, Validate, NsqPublish)
+	e.GET("/*", Getlog, Validate, NsqPublish)
+	e.Logger.Fatal(e.Start(Opts.HTTPAddress))
+}

handler.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+"encoding/json"
+"github.com/labstack/echo"
+"net/http"
+)
+
+func Postlog(c echo.Context) error {
+	result := c.FormValue(Opts.Param)
+	if result == "" {
+		return c.NoContent(http.StatusBadRequest)
+	}
+	c.Set("logMessage", result)
+	return c.Blob(http.StatusOK, "image/gif", Gif)
+}
+
+func Getlog(c echo.Context) error {
+	res := ""
+	result := c.QueryParam(Opts.Param)
+	if result != "" {
+		res = result
+	} else if r := c.QueryParams(); len(r) != 0 {
+		req := c.Request()
+		r["referer"] = []string{req.Referer()}
+		r["UA"] = []string{req.UserAgent()}
+		resByte, err := json.Marshal(r)
+		if err != nil {
+			return c.NoContent(http.StatusBadRequest)
+		}
+		res = string(resByte)
+	}
+	if res == "" {
+		return c.NoContent(http.StatusBadRequest)
+	}
+	c.Set("logMessage", res)
+	return c.Blob(http.StatusOK, "image/gif", Gif)
+}
+

middlewares.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"github.com/labstack/echo"
+	"github.com/nsqio/go-nsq"
+	"github.com/labstack/gommon/log"
+)
+
+var producer *nsq.Producer
+
+func InitNsq(addr string) {
+	cfg := nsq.NewConfig()
+	var err error
+	producer, err = nsq.NewProducer(addr, cfg)
+	if err != nil {
+		log.Fatalf("failed to create nsq.Producer - %s", err)
+	}
+}
+
+func NsqPublish(next echo.HandlerFunc) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		defer func() {
+			topic, ok := c.Get("topic").(string)
+			if !ok {
+				log.Error("topic is not string.")
+				log.Error(topic)
+				return
+			}
+			messageStr, ok := c.Get("logMessage").(string)
+			processed := Process(messageStr, c.RealIP())
+			message := []byte(processed)
+			if !ok {
+				log.Error("message is not string.")
+				log.Error(message)
+				return
+			}
+			err := producer.Publish(topic, message)
+			if err != nil {
+				log.Errorf("nsq send failed. %s", err)
+			}
+		}()
+		return next(c)
+	}
+}
+
+
+func Validate(next echo.HandlerFunc) echo.HandlerFunc {
+	return func(c echo.Context) error {
+
+		req := c.Request()
+		host := req.Host
+		uri := req.URL.Path
+		if topic, ok := Opts.TopicMapping[uri]; ok {
+			c.Set("topic", topic)
+			return next(c)
+		}
+		log.Infof("HOST=%s. 404 not found %s.", host, uri)
+		return echo.ErrNotFound
+	}
+}

utils.go

@@ -0,0 +1,10 @@
+package main
+
+import "time"
+
+var Gif = []byte{0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x1, 0x0, 0x1, 0x0, 0x80, 0x0, 0x0, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x2c, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x2, 0x2, 0x44, 0x1, 0x0, 0x3b}
+
+func Process(l string, ip string) string {
+	return "{\"message\":" + l + ",\"timestamp\":\"" + time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + "\",\"ip\":\"" + ip + "\"}"
+}
+