Download rules and unarchive them (#1)

* download and unarchive

* add scheduler

* fix headers

* add gh actions

* fix gh action

* fix typo

* fix

* update example config

* remove .github

* rename example config

* review

* update config

* add healthcheck

* review

* review

* updates

* remove unused config values

* add sample config into dockerfile

* review

* review

* fix

* fix lint

* update error logging

.licensei.toml

@@ -5,4 +5,21 @@ approved = [
     "bsd-2-clause",
     "mpl-2.0",
     "isc"
-]
+]
+
+[header]
+ignoreFiles = ["*mock_*.go"]
+template = """// Copyright © :YEAR: Cisco Systems, Inc. and its affiliates.
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License."""

Dockerfile

@@ -7,24 +7,37 @@ RUN apk add --update --no-cache gcc g++ git ca-certificates build-base
 COPY . /build
 WORKDIR /build
 
+ARG VERSION
 ARG BUILD_TIMESTAMP
-ARG IMAGE_VERSION
+ARG COMMIT_HASH
+ARG TARGETOS
+ARG TARGETARCH
 
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
-    CGO_ENABLED=0 \
+    GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \
     go build \
-    -ldflags="-s -w" \
+    -ldflags="-s -w \
+        -X 'github.com/openclarity/yara-rule-server/pkg/version.Version=${VERSION}' \
+        -X 'github.com/openclarity/yara-rule-server/pkg/version.CommitHash=${COMMIT_HASH}' \
+        -X 'github.com/openclarity/yara-rule-server/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}'" \
     -o bin/yara-rule-server main.go
 
-FROM alpine:3.18
+
+
+FROM alpine:3.17
 
 RUN apk upgrade
 RUN apk add util-linux
+RUN apk add --update yara --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing
+RUN mkdir /etc/yara-rule-server
 
 WORKDIR /app
 
-COPY --from=builder /build/bin/yara-rule-server ./yara-rule-server
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /build/bin/yara-rule-server ./yara-rule-server
+COPY config.example.yaml /etc/yara-rule-server/config.yaml
+
+WORKDIR /opt
 
 ENTRYPOINT ["/app/yara-rule-server"]

Makefile

@@ -34,20 +34,16 @@ build: ## Build Yara Rule Server
 .PHONY: docker
 docker: ## Build Yara Rule Server docker image
 	@(echo "Building Yara Rule Server docker image [${DOCKER_IMAGE}:${DOCKER_TAG}] ..." )
-	@(docker build -t ${DOCKER_IMAGE}:${DOCKER_TAG} . \
+	@(docker build --build-arg VERSION=${VERSION} \
 		--build-arg BUILD_TIMESTAMP=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") \
-		--build-arg IMAGE_VERSION=${VERSION})
+		--build-arg COMMIT_HASH=$(shell git rev-parse HEAD) \
+		-t ${DOCKER_IMAGE}:${DOCKER_TAG} .)
 
 .PHONY: docker-push
 docker-push: docker ## Build Yara Rule Server docker image and push it to remote
 	@(echo "Pushing Yara Rule Server docker image [${DOCKER_IMAGE}:${DOCKER_TAG}] ..." )
 	@(docker push ${DOCKER_IMAGE}:${DOCKER_TAG})
 
-.PHONY: api
-api: ## Generating API code
-	@(echo "Generating API code ..." )
-	@(cd api; ./generate.sh)
-
 .PHONY: test
 test: ## Run Unit Tests
 	@(go test ./pkg/...)

cmd/root.go

@@ -16,29 +16,35 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
+	"github.com/Portshift/go-utils/healthz"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
 	"github.com/openclarity/yara-rule-server/pkg/config"
+	"github.com/openclarity/yara-rule-server/pkg/fileserver"
+	"github.com/openclarity/yara-rule-server/pkg/rules"
+	"github.com/openclarity/yara-rule-server/pkg/version"
 )
 
-var (
-	cfgFile string
-	cfg     *config.Config
-	logger  *logrus.Entry
-)
+const ExecutableName = "yara-rule-server"
+
+var cfgFile string
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "yara-rule-server",
 	Short: "Yara rule server",
-	Long:  `Yara rule server downloads Yara rules, compiles them, and provides compiled rules as a file server`,
-	Run: func(cmd *cobra.Command, args []string) {
-		run(cmd, args)
-	},
+	Long:  "Yara rule server downloads Yara rules, compiles them, and provides compiled rules as a file server",
+	Version: fmt.Sprintf("Version: %s \nCommit: %s\nBuild Time: %s",
+		version.Version, version.CommitHash, version.BuildTimestamp),
+	SilenceUsage: true,
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
@@ -51,41 +57,91 @@ func Execute() {
 }
 
 func init() {
-	cobra.OnInitialize(
-		initConfig,
-		initLogger,
-	)
-	// Here you will define your flags and configuration settings.
-	// Cobra supports persistent flags, which, if defined here,
-	// will be global for your application.
-
-	// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.yara-rules-server.yaml)")
-
-	// Cobra also supports local flags, which will only run
-	// when this action is called directly.
-	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
-}
+	cmdRun := cobra.Command{
+		Use:     "run",
+		Run:     run,
+		Short:   "Starts the server",
+		Long:    "Starts the Yara Rule server",
+		Example: ExecutableName + " run",
+	}
+	cmdRun.PersistentFlags().StringVar(&cfgFile,
+		"config",
+		"",
+		"config file (default is /etc/yara-rules-server/config.yaml)")
 
-// initConfig reads in config file and ENV variables if set.
-func initConfig() {
-	cfg = config.LoadConfig(cfgFile)
-	if cfg.LogLevel == "" {
-		cfg.LogLevel = "info"
+	cmdVersion := cobra.Command{
+		Use:     "version",
+		Run:     versionCommand,
+		Short:   "Displays the version",
+		Long:    "Displays the version of the VMClarity API server",
+		Example: ExecutableName + " version",
 	}
-	fmt.Printf("Configuration: %+v", cfg)
+
+	rootCmd.AddCommand(&cmdRun)
+	rootCmd.AddCommand(&cmdVersion)
 }
 
-func initLogger() {
-	log := logrus.New()
+func initLogger(cfg *config.Config) *logrus.Entry {
+	logger := logrus.New()
 	if level, err := logrus.ParseLevel(cfg.LogLevel); err != nil {
-		log.SetLevel(level)
+		logger.SetLevel(level)
 	}
 	if cfg.EnableJSONLog {
-		log.SetFormatter(&logrus.JSONFormatter{})
+		logger.SetFormatter(&logrus.JSONFormatter{})
 	}
-	logger = log.WithField("app", "yara-rule-server")
+
+	return logger.WithField("app", "yara-rule-server")
 }
 
 func run(cmd *cobra.Command, args []string) {
-	logger.Infoln("running command")
+	cfg := config.LoadConfig(cfgFile)
+	fmt.Printf("config %+v", cfg)
+	logger := initLogger(cfg)
+
+	healthServer := healthz.NewHealthServer(cfg.HealthCheckAddressAddress)
+	healthServer.Start()
+	healthServer.SetIsReady(false)
+
+	// First we need to download and compile rules before starting the server.
+	if err := rules.DownloadAndCompile(cfg, logger); err != nil {
+		logger.Fatalf("Falied to compile YARA rules: %v", err)
+	}
+
+	// Start listening for OS signals to make sure that we can gracefully exit
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	// Start scheduler
+	s, err := rules.ScheduledDownload(cfg, logger)
+	if err != nil {
+		logger.Fatalf("Failed to start download scheduler: %v", err)
+	}
+	logger.Infoln("Yara rule download scheduler has been started.")
+
+	// Start file server
+	srv := fileserver.Start(cfg, logger)
+	logger.Infoln("Yara rule file server has been started.")
+
+	healthServer.SetIsReady(true)
+
+	<-ctx.Done()
+
+	logger.Infoln("Stopping yara rule download scheduler...")
+	s.Stop()
+
+	logger.Infoln("Stopping yara rule file server...")
+	shutdownCtx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		logger.Fatalf("Server shutdown failed: %v", err)
+	}
+}
+
+// Command to display the version.
+func versionCommand(_ *cobra.Command, _ []string) {
+	fmt.Printf("Version: %s \nCommit: %s\nBuild Time: %s",
+		version.Version, version.CommitHash, version.BuildTimestamp)
 }

config.example.yaml

@@ -0,0 +1,9 @@
+enable_json_log: true
+rule_update_schedule: "0 0 * * *"
+rule_sources:
+  - name: "base"
+    url: "https://github.com/Yara-Rules/rules/archive/refs/heads/master.zip"
+    exclude_regex: ".*index.*.yar|.*/utils/.*|.*/deprecated/.*|.*index_.*|.*MALW_AZORULT.yar"
+  - name: "magic"
+    url: "https://github.com/securitymagic/yara/archive/refs/heads/main.zip"
+    exclude_regex: ".*index.*.yar"

go.mod

@@ -3,23 +3,28 @@ module github.com/openclarity/yara-rule-server
 go 1.20
 
 require (
+	github.com/Portshift/go-utils v0.0.0-20220421083203-89265d8a6487
+	github.com/go-co-op/gocron v1.32.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.16.0
 )
 
 require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect