Merge pull request #1023 from crawlab-team/develop

Develop

.gitignore

@@ -1,128 +1,9 @@
 .idea/
 .vscode/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-# node_modules
-node_modules/
-
-# egg-info
-*.egg-info
-
-# .DS_Store
 .DS_Store
-
-.docks
-.docs
-
+node_modules/
 logs/
 tmp/
 _book/
-.idea
 *.lock
-
-backend/spiders
-spiders/*.zip
-
 vendor/

Dockerfile

@@ -16,9 +16,6 @@ WORKDIR /app
 RUN rm /app/.npmrc
 
 # install frontend
-#RUN npm config set unsafe-perm true
-#RUN npm install -g yarn && yarn install
-
 RUN yarn install && yarn run build:docker
 
 # images
@@ -33,18 +30,26 @@ ENV CRAWLAB_IS_DOCKER Y
 # install packages
 RUN chmod 777 /tmp \
 	&& apt-get update \
-	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init cloc \
+	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
+
+# install python
+RUN apt-get install -y python3 python3-pip \
 	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
 	&& ln -s /usr/bin/python3 /usr/local/bin/python
 
+# install golang
+RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \
+	&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
+	&& ln -s /usr/local/go/bin/go /usr/local/bin/go
+
 # install seaweedfs
-RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \
+RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \
   && tar -zxf linux_amd64.tar.gz \
   && cp weed /usr/local/bin
 
 # install backend
 RUN pip install scrapy pymongo bs4 requests
-RUN pip install crawlab-sdk==0.6.b20210729-1634
+RUN pip install crawlab-sdk==0.6.b20211024-1207
 
 # add files
 COPY ./backend/conf /app/backend/conf

Dockerfile.cn

@@ -16,9 +16,6 @@ WORKDIR /app
 #RUN rm /app/.npmrc
 
 # install frontend
-#RUN npm config set unsafe-perm true
-#RUN npm install -g yarn && yarn install
-
 RUN yarn install && yarn run build:docker
 
 # images
@@ -33,18 +30,26 @@ ENV CRAWLAB_IS_DOCKER Y
 # install packages
 RUN chmod 777 /tmp \
 	&& apt-get update \
-	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init cloc \
+	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
+
+# install python
+RUN apt-get install -y python3 python3-pip \
 	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
 	&& ln -s /usr/bin/python3 /usr/local/bin/python
 
+# install golang
+RUN curl -OL https://storage.googleapis.com/golang/go1.16.7.linux-amd64.tar.gz \
+	&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
+	&& ln -s /usr/local/go/bin/go /usr/local/bin/go
+
 # install seaweedfs
-RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \
+RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \
   && tar -zxf linux_amd64.tar.gz \
   && cp weed /usr/local/bin
 
 # install backend
 RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20210729-1634
+RUN pip install crawlab-sdk==0.6.b20211024-1207
 
 # add files
 COPY ./backend/conf /app/backend/conf

Dockerfile.local

@@ -16,9 +16,6 @@ WORKDIR /app
 #RUN rm /app/.npmrc
 
 # install frontend
-#RUN npm config set unsafe-perm true
-#RUN npm install -g yarn && yarn install
-
 RUN yarn install && yarn run build:docker
 
 # images
@@ -33,18 +30,26 @@ ENV CRAWLAB_IS_DOCKER Y
 # install packages
 RUN chmod 777 /tmp \
 	&& apt-get update \
-	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip nginx wget dumb-init cloc \
+	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate nginx wget dumb-init cloc
+
+# install python
+RUN apt-get install -y python3 python3-pip \
 	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
 	&& ln -s /usr/bin/python3 /usr/local/bin/python
 
+# install golang
+RUN curl -OL https://golang.org/dl/go1.16.7.linux-amd64.tar.gz \
+	&& tar -C /usr/local -xvf go1.16.7.linux-amd64.tar.gz \
+	&& ln -s /usr/local/go/bin/go /usr/local/bin/go
+
 # install seaweedfs
-RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \
+RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.76/linux_amd64.tar.gz \
   && tar -zxf linux_amd64.tar.gz \
   && cp weed /usr/local/bin
 
 # install backend
 RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20210729-1634
+RUN pip install crawlab-sdk==0.6.b20211024-1207
 
 # add files
 COPY ./backend/conf /app/backend/conf

backend/cmd/master.go

@@ -2,30 +2,11 @@ package cmd
 
 import (
 	"crawlab/apps"
-	"fmt"
-	"github.com/crawlab-team/crawlab-core/entity"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var (
-	runOnMaster       bool
-	masterConfigPath  string
-	masterGrpcAddress string
-	masterGrpcAuthKey string
 )
 
 func init() {
 	rootCmd.AddCommand(masterCmd)
-
-	masterCmd.PersistentFlags().StringVarP(&masterConfigPath, "config-path", "c", "", "Config path of master node")
-	_ = viper.BindPFlag("configPath", masterCmd.PersistentFlags().Lookup("configPath"))
-
-	masterCmd.PersistentFlags().StringVarP(&masterGrpcAddress, "grpc-address", "g", "", "gRPC address of master node")
-	_ = viper.BindPFlag("grpcAddress", masterCmd.PersistentFlags().Lookup("grpcAddress"))
-
-	masterCmd.PersistentFlags().StringVarP(&masterGrpcAuthKey, "grpc-auth-key", "a", "", "gRPC auth key of master node")
-	_ = viper.BindPFlag("grpcAuthKey", masterCmd.PersistentFlags().Lookup("grpcAuthKey"))
 }
 
 var masterCmd = &cobra.Command{
@@ -37,23 +18,6 @@ which runs api and assign tasks to worker nodes`,
 	Run: func(cmd *cobra.Command, args []string) {
 		// options
 		var opts []apps.MasterOption
-		if masterConfigPath != "" {
-			opts = append(opts, apps.WithMasterConfigPath(masterConfigPath))
-			viper.Set("config.path", masterConfigPath)
-		}
-		opts = append(opts, apps.WithRunOnMaster(runOnMaster))
-		if masterGrpcAddress != "" {
-			address, err := entity.NewAddressFromString(masterGrpcAddress)
-			if err != nil {
-				fmt.Println(fmt.Sprintf("invalid grpc-address: %s", masterGrpcAddress))
-			}
-			opts = append(opts, apps.WithMasterGrpcAddress(address))
-			viper.Set("grpc.address", masterGrpcAddress)
-			viper.Set("grpc.server.address", masterGrpcAddress)
-		}
-		if masterGrpcAuthKey != "" {
-			viper.Set("grpc.authKey", masterGrpcAuthKey)
-		}
 
 		// app
 		master := apps.NewMaster(opts...)

backend/cmd/root.go

@@ -50,6 +50,7 @@ func initConfig() {
 	replacer := strings.NewReplacer(".", "_")
 	viper.SetEnvKeyReplacer(replacer)
 
+	// read config file
 	if err := viper.ReadInConfig(); err == nil {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
 	}

backend/cmd/worker.go

@@ -2,29 +2,11 @@ package cmd
 
 import (
 	"crawlab/apps"
-	"fmt"
-	"github.com/crawlab-team/crawlab-core/entity"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var (
-	workerConfigPath  string
-	workerGrpcAddress string
-	workerGrpcAuthKey string
 )
 
 func init() {
 	rootCmd.AddCommand(workerCmd)
-
-	workerCmd.PersistentFlags().StringVarP(&workerConfigPath, "config-path", "c", "", "Config path of worker node")
-	_ = viper.BindPFlag("configPath", workerCmd.PersistentFlags().Lookup("configPath"))
-
-	workerCmd.PersistentFlags().StringVarP(&workerGrpcAddress, "grpc-address", "g", "", "gRPC address of worker node")
-	_ = viper.BindPFlag("grpcAddress", workerCmd.PersistentFlags().Lookup("grpcAddress"))
-
-	workerCmd.PersistentFlags().StringVarP(&workerGrpcAuthKey, "grpc-auth-key", "a", "", "gRPC auth key of worker node")
-	_ = viper.BindPFlag("grpcAuthKey", workerCmd.PersistentFlags().Lookup("grpcAuthKey"))
 }
 
 var workerCmd = &cobra.Command{
@@ -37,22 +19,6 @@ assigned by the master node`,
 	Run: func(cmd *cobra.Command, args []string) {
 		// options
 		var opts []apps.WorkerOption
-		if workerConfigPath != "" {
-			opts = append(opts, apps.WithWorkerConfigPath(workerConfigPath))
-			viper.Set("config.path", workerConfigPath)
-		}
-		if workerGrpcAddress != "" {
-			address, err := entity.NewAddressFromString(workerGrpcAddress)
-			if err != nil {
-				fmt.Println(fmt.Sprintf("invalid grpc-address: %s", workerGrpcAddress))
-				return
-			}
-			opts = append(opts, apps.WithWorkerGrpcAddress(address))
-			viper.Set("grpc.address", workerGrpcAddress)
-		}
-		if workerGrpcAuthKey != "" {
-			viper.Set("grpc.authKey", workerGrpcAuthKey)
-		}
 
 		// app
 		master := apps.NewWorker(opts...)

backend/go.mod

@@ -4,7 +4,7 @@ go 1.15
 
 require (
 	github.com/apex/log v1.9.0
-	github.com/crawlab-team/crawlab-core v0.6.0-beta.20210811.1634
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211113.2050
 	github.com/crawlab-team/go-trace v0.1.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/spf13/cobra v1.1.3