documentation, refactoring and lot of stuff

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "docs/themes/zdoc"]
+	path = docs/themes/zdoc
+	url = https://github.com/zzossig/hugo-theme-zdoc.git

README.md

@@ -1 +1,39 @@
-# DripLane
+# Introduction
+
+Driplane allows you to create an automatic alerting system or start automated tasks triggered by events.
+You can keep under control a source as Twitter, a file, a RSS feed or a website.
+It includes a mini language to define the filtering pipelines (the rules) and contains a JS plugin system. 
+
+With driplane you can create several rules starting from one or more streams, filter the content in the pipeline and launch task or send e-mail if some events occurred.
+
+The documentation can be found [HERE](https://matrix86.github.io/driplane/)
+
+## How it works
+
+The user can define one or more rules. Each rules contain a source (`feeder`), who cares of get the information and send updates (`Message`) through the pipeline, and several `filters`.
+The task of the filters is to propagate or stop the `Message` to the next filter in the pipeline, or change the `Message` received before to propagate it. If the condition is verified, the `Message` will be propagated.
+
+## Use cases
+
+Using driplane is it possible to:
+
+ * keep track of keywords or users on Twitter, receive the new tweets or quoted tweet from them, search for URLs or particular strings and send it to a Telegram or Slack channel through a webhook.
+ * keep track of a RSS feed or a website, and download and store on file all the new changes to them.
+
+The rules and the JS plugins allows you to create more complex custom logics.
+
+## Usage
+
+```
+Usage of driplane:
+  -config string
+    	Set configuration file.
+  -debug
+    	Enable debug logs.
+  -help
+    	This help.
+  -js string
+    	Path of the js plugins.
+  -rules string
+    	Path of the rules' directory.
+```

cmd/driplane/main.go

@@ -2,9 +2,12 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"github.com/Matrix86/driplane/utils"
+	"github.com/evilsocket/islazy/tui"
 	"os"
 	"os/signal"
+	"runtime"
 	"syscall"
 
 	"github.com/Matrix86/driplane/core"
@@ -21,6 +24,7 @@ var (
 	configFile string
 )
 
+// Signal stops feeders on SIGINT or SIGTERM signal interception
 func Signal(o *core.Orchestrator) {
 
 	sChan := make(chan os.Signal, 1)
@@ -45,6 +49,12 @@ func main() {
 	flag.BoolVar(&debugFlag, "debug", false, "Enable debug logs.")
 	flag.Parse()
 
+	appName := fmt.Sprintf("%s v%s", core.Name, core.Version)
+	appBuild := fmt.Sprintf("(built for %s %s with %s)", runtime.GOOS, runtime.GOARCH, runtime.Version())
+	appAuthor := fmt.Sprintf("Author: %s", core.Author)
+
+	fmt.Printf("%s %s\n%s\n", tui.Bold(appName), tui.Dim(appBuild), tui.Dim(appAuthor))
+
 	if helpFlag {
 		flag.Usage()
 		return

config.yaml.example

@@ -1,8 +1,9 @@
 general:
-  rules_path: ""
-  js_path: "" # if empty rules_path is used as base
   log_path: ""
-  debug: true
+  rules_path: "rules"
+  js_path: "js"
+  templates_path: "templates"
+  debug: false
 
 twitter:
   consumerKey: "",

core/configuration.go

@@ -10,13 +10,15 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+// Configuration contains all the configs read by yaml file
 type Configuration struct {
 	sync.RWMutex
 
 	FilePath string
 	flat     map[string]string
 }
 
+// LoadConfiguration create a Configuration struct from a filename
 func LoadConfiguration(path string) (*Configuration, error) {
 	configuration := &Configuration{
 		FilePath: path,
@@ -66,6 +68,7 @@ func (c *Configuration) flatMap(m map[interface{}]interface{}) map[string]string
 	return flatten
 }
 
+// Get returns the config value with that name, if it exists
 func (c *Configuration) Get(name string) string {
 	c.RLock()
 	defer c.RUnlock()
@@ -76,6 +79,7 @@ func (c *Configuration) Get(name string) string {
 	return c.flat[name]
 }
 
+// Set insert a new config in the Configuration struct
 func (c *Configuration) Set(name string, value string) error {
 	c.Lock()
 	defer c.Unlock()
@@ -84,6 +88,7 @@ func (c *Configuration) Set(name string, value string) error {
 	return nil
 }
 
+// GetConfig returns the complete configuration in a flatten way
 func (c *Configuration) GetConfig() map[string]string {
 	c.RLock()
 	defer c.RUnlock()

core/orchestrator.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 )
 
+// Orchestrator handles the pipelines and rules
 type Orchestrator struct {
 	asts   map[string]*AST
 	config *Configuration
@@ -15,6 +16,7 @@ type Orchestrator struct {
 	sync.Mutex
 }
 
+// NewOrchestrator create a new instance of the Orchestrator
 func NewOrchestrator(asts map[string]*AST, config *Configuration) (Orchestrator, error) {
 	o := Orchestrator{}
 
@@ -35,6 +37,7 @@ func NewOrchestrator(asts map[string]*AST, config *Configuration) (Orchestrator,
 	return o, nil
 }
 
+// StartFeeders opens the gates
 func (o *Orchestrator) StartFeeders() {
 	o.Lock()
 	defer o.Unlock()
@@ -49,6 +52,7 @@ func (o *Orchestrator) StartFeeders() {
 	}
 }
 
+// HasRunningFeeder return true if one or more feeders are running
 func (o *Orchestrator) HasRunningFeeder() bool {
 	rs := RuleSetInstance()
 	for _, rulename := range rs.feedRules {
@@ -60,12 +64,14 @@ func (o *Orchestrator) HasRunningFeeder() bool {
 	return false
 }
 
+// WaitFeeders waits until all the feeders are stopped
 func (o *Orchestrator) WaitFeeders() {
 	log.Debug("Waiting")
 	o.waitFeeder.Wait()
 	log.Debug("Stop waiting")
 }
 
+// StopFeeders closes the gates
 func (o *Orchestrator) StopFeeders() {
 	o.Lock()
 	defer o.Unlock()

core/pipe_rule.go

@@ -16,25 +16,28 @@ var (
 	once     sync.Once
 )
 
+// Ruleset identifies a set of rules
 type Ruleset struct {
 	rules map[string]*PipeRule
 
 	feedRules []string
 	bus       bus.Bus
-	lastId    int32
+	lastID    int32
 }
 
+// RuleSetInstance is the singleton for the Ruleset object
 func RuleSetInstance() *Ruleset {
 	once.Do(func() {
 		instance = &Ruleset{
 			rules:  make(map[string]*PipeRule),
 			bus:    bus.New(),
-			lastId: 0,
+			lastID: 0,
 		}
 	})
 	return instance
 }
 
+// AddRule appends a new rule to the set
 func (r *Ruleset) AddRule(node *RuleNode, config *Configuration) error {
 	if node == nil || node.Identifier == "" {
 		return fmt.Errorf("Ruleset.AddRule: rules without name are not supported")
@@ -58,8 +61,10 @@ func (r *Ruleset) AddRule(node *RuleNode, config *Configuration) error {
 	return nil
 }
 
+// INode for the nodes generalization
 type INode interface{}
 
+// PipeRule identifies a single rule
 type PipeRule struct {
 	Name      string
 	HasFeeder bool
@@ -121,11 +126,11 @@ func (p *PipeRule) newFilter(fn *FilterNode) (filters.Filter, error) {
 	}
 
 	rs := RuleSetInstance()
-	f, err := filters.NewFilter(p.Name, fn.Name+"filter", params, rs.bus, rs.lastId+1, fn.Neg)
+	f, err := filters.NewFilter(p.Name, fn.Name+"filter", params, rs.bus, rs.lastID+1, fn.Neg)
 	if err != nil {
 		return nil, err
 	}
-	rs.lastId++
+	rs.lastID++
 
 	return f, nil
 }
@@ -196,6 +201,7 @@ func (p *PipeRule) addNode(node *Node, prev string) error {
 	return nil
 }
 
+// NewPipeRule creates and returns a PipeRule struct
 func NewPipeRule(node *RuleNode, config *Configuration) (*PipeRule, error) {
 	rule := &PipeRule{}
 	rule.Name = node.Identifier
@@ -234,12 +240,12 @@ func NewPipeRule(node *RuleNode, config *Configuration) (*PipeRule, error) {
 		}
 
 		rs := RuleSetInstance()
-		f, err := feeders.NewFeeder(node.Feeder.Name+"feeder", params, rs.bus, rs.lastId+1)
+		f, err := feeders.NewFeeder(node.Feeder.Name+"feeder", params, rs.bus, rs.lastID+1)
 		if err != nil {
 			log.Error("piperule.NewRule: %s", err)
 			return nil, err
 		}
-		rs.lastId++
+		rs.lastID++
 
 		rule.HasFeeder = true
 		rule.nodes = append(rule.nodes, f)

core/rule_parser.go

@@ -12,7 +12,6 @@ var ruleLexer = lexer.Must(lexer.Regexp(
 	`(?m)` +
 		`(\s+)` +
 		`|(^[#].*$)` +
-		`|(?P<Keyword>(?i)FEEDER|SET)` +
 		`|(?P<Ident>[a-zA-Z][a-zA-Z_\d]*)` +
 		//`|(?P<String>"(?:[^"]*("")?)*")` +
 		`|(?P<String>"(?:\\.|[^\"])*")` +
@@ -21,28 +20,33 @@ var ruleLexer = lexer.Must(lexer.Regexp(
 		`|(?P<Operators>!)`,
 ))
 
+// AST defines a set of Rules
 type AST struct {
 	Rules []*RuleNode `@@*`
 }
 
+// RuleNode defines the first part of the Rule
 type RuleNode struct {
 	Identifier string      `@Ident "="">"`
 	Feeder     *FeederNode `( @@`
 	First      *Node       `| @@ ) ";"`
 }
 
+// Node identifies a Filter or a RuleCall
 type Node struct {
 	//Action   *ActionNode `( @@ `
 	Filter   *FilterNode `( @@`
 	RuleCall *RuleCall   `| @@)`
 }
 
+// FeederNode identifies the Feeder in the rule
 type FeederNode struct {
 	Name   string   `"<" @Ident`
 	Params []*Param `(":" @@ ("," @@)*)? ">"`
 	Next   *Node    `("|" @@)?`
 }
 
+// FilterNode identifies the Filter in the rule
 type FilterNode struct {
 	Neg    bool     `@("!")?`
 	Name   string   `@Ident`
@@ -56,25 +60,30 @@ type FilterNode struct {
 //	Next   *Node    `("|" @@)?`
 //}
 
+// RuleCall identifies the Call nodes in the rule
 type RuleCall struct {
 	Name string `"@" @Ident`
 	Next *Node  `("|" @@)?`
 }
 
+// Param identifies the parameters accepted by nodes
 type Param struct {
 	Name  string `@Ident "="`
 	Value *Value `@@`
 }
 
+// Value identifies a String or a Number
 type Value struct {
 	String *string  `  @String`
 	Number *float64 `| @Float`
 }
 
+// Parser handles the parsing of the rules
 type Parser struct {
 	handle *participle.Parser
 }
 
+// NewParser creates a new Parser struct
 func NewParser() (*Parser, error) {
 	var err error
 	parser := &Parser{}
@@ -90,6 +99,7 @@ func NewParser() (*Parser, error) {
 	return parser, nil
 }
 
+// ParseFile returns the AST of the input file
 func (p *Parser) ParseFile(filename string) (*AST, error) {
 	ast := &AST{}
 	content, err := ioutil.ReadFile(filename)

core/version.go

@@ -1,7 +1,8 @@
 package core
 
+// Info on driplane
 const (
 	Name    = "driplane"
-	Version = "0.3"
+	Version = "0.8"
 	Author  = "Gianluca Braga aka Matrix86"
 )