From 8aa2ccf3ee1bd17b8239ae1bff0ea50073df9c23 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Wed, 10 Jun 2020 10:39:18 +0800 Subject: [PATCH 01/80] WIP for vas sample applications --- applications/vas-sample-app/application.go | 86 ++++++++++++ applications/vas-sample-app/eaaInterface.go | 148 ++++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 applications/vas-sample-app/application.go create mode 100644 applications/vas-sample-app/eaaInterface.go diff --git a/applications/vas-sample-app/application.go b/applications/vas-sample-app/application.go new file mode 100644 index 00000000..57844334 --- /dev/null +++ b/applications/vas-sample-app/application.go @@ -0,0 +1,86 @@ +package main + +import ( + "github.com/levigross/grequests" + "encoding/json" + "log" + "strings" + "time" +) + +type Source struct { + URI string `json:"uri"` + Type string `json:"type"` +} + +type Destination struct { + Path string `json:"path"` + Type string `json:"type"` + Format string `json:"format"` +} + + +type VASPostBody struct { + SRCE Source `json:"source"` + DEST Destination `json:"destination"` +} + +type VASInstanceStatus struct { + AvgFps float64 `json:"avg_fps"` + ElapsedTime float64 `json:"elapsed_time"` + ID int `json:"id"` + StartTime float64 `json:"start_time"` + State string `json:"state"` +} + + +func main () { + + // Interface: GET /pipelines for Return supported pipelines + log.Println("GET /pipelines for Return supported pipelines") + resp, err := grequests.Get("http://localhost:8080/pipelines", nil) + if err != nil { + log.Fatalln ("Get Failed with: ", err) + } + log.Println(resp.String()) + log.Println("Completed Get Pipelines .......") + + // Interface: POST /pipelines/{name}/{version} for Start new pipeline instance + log.Println("Starting POST Request .......") + var src Source + src.URI = "https://github.com/intel-iot-devkit/sample-videos/blob/master/bottle-detection.mp4?raw=true" + src.Type = "uri" + var dst Destination + dst.Path = "/tmp/results.txt" + dst.Type = "file" + dst.Format = "json-lines" + var vasPost = VASPostBody{SRCE: src, DEST: dst} + var vasPostJson, _ = json.Marshal(vasPost) + var vasRequestOption = &grequests.RequestOptions{} + vasRequestOption.JSON = string(vasPostJson) + resp, err = grequests.Post("http://localhost:8080/pipelines/object_detection/1",vasRequestOption) + if err != nil { + log.Fatalln ("Post Failed with: ", err) + } + log.Println("Pipeline Instance Created:", resp.String()) + var instance = strings.TrimSpace(resp.String()) + + // Interface: GET /pipelines/{name}/{version}/{instance_id}/status for Return pipeline instance status. + // Loop status query until instantance completed. + var statusResp VASInstanceStatus + for { + resp, err := grequests.Get("http://localhost:8080/pipelines/object_detection/1/"+instance+"/status", nil) + if err != nil { + log.Fatalln ("Get Failed with: ", err) + } + log.Println(resp.String()) + resp.JSON(&statusResp) + if statusResp.State == "COMPLETED" { + break + } + time.Sleep(time.Duration(10)*time.Second) + + } + + +} diff --git a/applications/vas-sample-app/eaaInterface.go b/applications/vas-sample-app/eaaInterface.go new file mode 100644 index 00000000..ed986b49 --- /dev/null +++ b/applications/vas-sample-app/eaaInterface.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2020 Intel Corporation + +package main + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "log" + "net/http" + "time" + + "github.com/gorilla/websocket" + "github.com/otcshare/edgeapps/applications/sample-app/common" + "github.com/pkg/errors" +) + +// subscribeConsumer sends a consumer subscription POST request to the appliance +// Subscribe to a set of notification types, +// optionally specifying the exact producer by URN. +func serviceSubscription(client *http.Client, + notifs []common.NotificationDescriptor, path string) error { + + address := "https://" + common.Cfg.EdgeNodeEndpoint + + "/subscriptions/" + path + + payload, err := json.Marshal(notifs) + if err != nil { + return errors.Wrap(err, "Couldn't marshal notifications") + } + + req, err := http.NewRequest("POST", address, bytes.NewBuffer(payload)) + if err != nil { + return errors.Wrap(err, "Couldn't create POST request for ") + } + + respPost, err := client.Do(req) + if err != nil { + return errors.Wrap(err, "Couldn't send POST request to "+address) + } + defer func() { + if err := respPost.Body.Close(); err != nil { + log.Println("Failed to close response body") + } + }() + return nil +} + +// unsubscribeConsumer sends a consumer subscription POST request +// to the appliance +// Unsubscribe from a particular or entire namespace of producers. +func unsubscribeConsumer(c *http.Client, notifs []common.NotificationDescriptor, + path string) error { + + address := "https://" + common.Cfg.EdgeNodeEndpoint + + "/subscriptions/" + path + + payload, err := json.Marshal(notifs) + if err != nil { + return errors.Wrap(err, "Couldn't marshal notifications") + } + + req, err := http.NewRequest("DELETE", address, bytes.NewBuffer(payload)) + if err != nil { + return errors.Wrap(err, + fmt.Sprintf("%s %s", "Couldn't create DELETE request for", address)) + } + + respPost, err := c.Do(req) + if err != nil { + return errors.Wrap(err, "Couldn't send POST request to "+address) + } + defer func() { + if err := respPost.Body.Close(); err != nil { + log.Println("Failed to close response body") + } + }() + + return nil +} + +// connectConsumer sends a consumer notifications GET request to the appliance +// Connect to a secure WebSocket to receive streaming notifications +// (see reference for notification to consumer for schema of messages received +// on websocket). +func connectConsumer(client *http.Client) (*websocket.Conn, error) { + + transport, ok := client.Transport.(*http.Transport) + if !ok { + return nil, errors.New("HTTP client doens't have http.Transport") + } + + socket := &websocket.Dialer{ + TLSClientConfig: &tls.Config{ + RootCAs: transport.TLSClientConfig.RootCAs, + Certificates: []tls.Certificate{ + transport.TLSClientConfig.Certificates[0]}, + ServerName: common.Cfg.EaaCommonName, + }, + } + + hostHeader := http.Header{} + hostHeader.Add("Host", common.Cfg.Namespace+":"+common.Cfg.ConsumerAppID) + + conn, resp, err := socket.Dial("wss://"+common.Cfg.EdgeNodeEndpoint+ + "/notifications", hostHeader) + if err != nil { + return nil, errors.Wrap(err, "Couldn't dial to wss") + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Println("Failed to close response body") + } + }() + + return conn, nil +} + +// getMsgFromConn retrieves a message from a connection and parses +// it to a notification struct +func getMessages(conn *websocket.Conn) { + + err := conn.SetReadDeadline(time.Now().Add(time.Second * + time.Duration(common.Cfg.ConsumerTimeout))) + if err != nil { + log.Println("Failed to set read dead line") + } + + for { + var resp common.NotificationToConsumer + _, message, err := conn.ReadMessage() + if err != nil { + fmt.Println("Stopped reading messages") + return + } + output := bytes.NewReader(message) + err = json.NewDecoder(output).Decode(&resp) + if err != nil { + log.Println("Cannot decode a message") + } else { + fmt.Printf("Received notification:\n Name: %v\n Version: %v\n"+ + " Payload: %v\n URN: %v\n", + resp.Name, resp.Version, string(resp.Payload), resp.URN) + } + } +} From dd46f438687be01d4afceef25a0ee26823aa0c21 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Tue, 16 Jun 2020 23:52:11 +0800 Subject: [PATCH 02/80] add docker file and k8s yaml for vas sample app --- applications/go.mod | 12 -- applications/go.sum | 6 - applications/vas-sample-app/Dockerfile | 26 +++ applications/vas-sample-app/build-image.sh | 10 + .../vas-sample-app/{ => cmd}/application.go | 32 ++- .../vas-sample-app/cmd/eaaInterface.go | 101 ++++++++++ applications/vas-sample-app/cmd/main.go | 189 ++++++++++++++++++ applications/vas-sample-app/eaaInterface.go | 148 -------------- applications/vas-sample-app/start.sh | 7 + applications/vas-sample-app/vas-cons-app.yaml | 24 +++ 10 files changed, 371 insertions(+), 184 deletions(-) delete mode 100644 applications/go.mod delete mode 100644 applications/go.sum create mode 100644 applications/vas-sample-app/Dockerfile create mode 100755 applications/vas-sample-app/build-image.sh rename applications/vas-sample-app/{ => cmd}/application.go (72%) create mode 100644 applications/vas-sample-app/cmd/eaaInterface.go create mode 100644 applications/vas-sample-app/cmd/main.go delete mode 100644 applications/vas-sample-app/eaaInterface.go create mode 100755 applications/vas-sample-app/start.sh create mode 100644 applications/vas-sample-app/vas-cons-app.yaml diff --git a/applications/go.mod b/applications/go.mod deleted file mode 100644 index 823ab17b..00000000 --- a/applications/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright (c) 2020 Intel Corporation - -module sample-app - -go 1.14.2 - -require ( - github.com/gorilla/websocket v1.4.2 - github.com/otcshare/edgeapps v0.0.0-20200513043854-a3d6e4a2e67d - github.com/pkg/errors v0.9.1 -) diff --git a/applications/go.sum b/applications/go.sum deleted file mode 100644 index 64abd7cc..00000000 --- a/applications/go.sum +++ /dev/null @@ -1,6 +0,0 @@ -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/otcshare/edgeapps v0.0.0-20200513043854-a3d6e4a2e67d h1:Uzx9ed8O51XOdsvaL1r6EvnRul4evUQAQj47GCnA5Xw= -github.com/otcshare/edgeapps v0.0.0-20200513043854-a3d6e4a2e67d/go.mod h1:Jk67iWGIVe9yzeyzMWKBtEA6dB/rXtFZRAdD8gp0yTM= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/applications/vas-sample-app/Dockerfile b/applications/vas-sample-app/Dockerfile new file mode 100644 index 00000000..ca349e71 --- /dev/null +++ b/applications/vas-sample-app/Dockerfile @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +FROM centos:7.6.1810 + +ENV http_proxy=$http_proxy +ENV https_proxy=$https_proxy +ENV no_proxy=$no_proxy,eaa.openness,media.openness + +ENV GOPATH=/root/go +ENV GOROOT=/usr/local/go +ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH + +RUN yum install -y wget + +# Install Go +RUN cd /tmp && \ + wget https://dl.google.com/go/go1.12.4.linux-amd64.tar.gz && \ + tar -xvf go1.12.4.linux-amd64.tar.gz && \ + mv go /usr/local && \ + rm -rf /tmp/go* + + +COPY start.sh ./ +COPY cmd/ ./ +ENTRYPOINT ["./start.sh"] diff --git a/applications/vas-sample-app/build-image.sh b/applications/vas-sample-app/build-image.sh new file mode 100755 index 00000000..98742c47 --- /dev/null +++ b/applications/vas-sample-app/build-image.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# PDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +sudo docker build \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + --build-arg no_proxy=$no_proxy \ + -t vas-cons-app:1.0 . diff --git a/applications/vas-sample-app/application.go b/applications/vas-sample-app/cmd/application.go similarity index 72% rename from applications/vas-sample-app/application.go rename to applications/vas-sample-app/cmd/application.go index 57844334..792ea697 100644 --- a/applications/vas-sample-app/application.go +++ b/applications/vas-sample-app/cmd/application.go @@ -33,20 +33,12 @@ type VASInstanceStatus struct { State string `json:"state"` } - -func main () { - - // Interface: GET /pipelines for Return supported pipelines - log.Println("GET /pipelines for Return supported pipelines") - resp, err := grequests.Get("http://localhost:8080/pipelines", nil) - if err != nil { - log.Fatalln ("Get Failed with: ", err) - } - log.Println(resp.String()) - log.Println("Completed Get Pipelines .......") +func postVAServingRequest(vaEndPoint string, vaPipeline string) error { // Interface: POST /pipelines/{name}/{version} for Start new pipeline instance - log.Println("Starting POST Request .......") + var endPointStr string + endPointStr = vaEndPoint+"/pipelines/"+vaPipeline + log.Println("Starting POST Request:"+endPointStr) var src Source src.URI = "https://github.com/intel-iot-devkit/sample-videos/blob/master/bottle-detection.mp4?raw=true" src.Type = "uri" @@ -58,20 +50,24 @@ func main () { var vasPostJson, _ = json.Marshal(vasPost) var vasRequestOption = &grequests.RequestOptions{} vasRequestOption.JSON = string(vasPostJson) - resp, err = grequests.Post("http://localhost:8080/pipelines/object_detection/1",vasRequestOption) + //resp, err = grequests.Post("http://localhost:8080/pipelines/object_detection/1",vasRequestOption) + resp, err := grequests.Post(endPointStr,vasRequestOption) if err != nil { log.Fatalln ("Post Failed with: ", err) + return err } log.Println("Pipeline Instance Created:", resp.String()) var instance = strings.TrimSpace(resp.String()) - + // Interface: GET /pipelines/{name}/{version}/{instance_id}/status for Return pipeline instance status. // Loop status query until instantance completed. - var statusResp VASInstanceStatus + var statusResp VASInstanceStatus for { - resp, err := grequests.Get("http://localhost:8080/pipelines/object_detection/1/"+instance+"/status", nil) + //resp, err := grequests.Get("http://localhost:8080/pipelines/object_detection/1/"+instance+"/status", nil) + resp, err := grequests.Get(endPointStr+"/"+instance+"/status", nil) if err != nil { log.Fatalln ("Get Failed with: ", err) + return err } log.Println(resp.String()) resp.JSON(&statusResp) @@ -79,8 +75,8 @@ func main () { break } time.Sleep(time.Duration(10)*time.Second) - - } + } + return nil } diff --git a/applications/vas-sample-app/cmd/eaaInterface.go b/applications/vas-sample-app/cmd/eaaInterface.go new file mode 100644 index 00000000..9a424e66 --- /dev/null +++ b/applications/vas-sample-app/cmd/eaaInterface.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2020 Intel Corporation + +package main + +import "encoding/json" + +// NotificationDescriptor describes a type used in EAA API +type NotificationDescriptor struct { + // Name of notification + Name string `json:"name,omitempty"` + // Version of notification + Version string `json:"version,omitempty"` + // Human readable description of notification + Description string `json:"description,omitempty"` +} + +// NotificationToConsumer describes a type used in EAA API +type NotificationToConsumer struct { + // Name of notification + Name string `json:"name,omitempty"` + // Version of notification + Version string `json:"version,omitempty"` + // The payload can be any JSON object with a name + // and version-specific schema. + Payload json.RawMessage `json:"payload,omitempty"` + // URN of the producer + URN URN `json:"producer,omitempty"` +} + +// ServiceList JSON struct +type ServiceList struct { + Services []Service `json:"services,omitempty"` +} + +// Service JSON struct +type Service struct { + URN *URN `json:"urn,omitempty"` + Description string `json:"description,omitempty"` + EndpointURI string `json:"endpoint_uri,omitempty"` + Status string `json:"status,omitempty"` + Notifications []NotificationDescriptor `json:"notifications,omitempty"` + Info json.RawMessage `json:"info,omitempty"` +} + +// SubscriptionList JSON struct +type SubscriptionList struct { + Subscriptions []Subscription `json:"subscriptions,omitempty"` +} + +// Subscription describes a type used in EAA API +type Subscription struct { + + // The name of the producer app. The unique ID is optional for + // subscribing and if not given will subscribe to any producer in the + // namespace. + URN *URN `json:"urn,omitempty"` + + // The list of all notification types registered by all producers in + // this namespace. + Notifications []NotificationDescriptor `json:"notifications,omitempty"` +} + +// URN describes a type used in EAA API +type URN struct { + + // The per-namespace unique portion of the URN that when appended to + // the namespace with a separator forms the complete URN. + ID string `json:"id,omitempty"` + + // The non-unique portion of the URN that identifies the class excluding + // a trailing separator. + Namespace string `json:"namespace,omitempty"` +} + +// AuthCredentials defines a response for a request to obtain authentication +// credentials. These credentials may be used to further communicate with +// endpoint(s) that are protected by a form of authentication. +// +// Any strings that are annotated as "PEM-encoded" implies that encoding format +// is used, with any newlines indicated with `\n` characters. Most languages +// provide encoders that correctly marshal this out. For more information, +// see the RFC here: https://tools.ietf.org/html/rfc7468 +type AuthCredentials struct { + ID string `json:"id,omitempty"` + Certificate string `json:"certificate,omitempty"` + CaChain []string `json:"ca_chain,omitempty"` + CaPool []string `json:"ca_pool,omitempty"` +} + +// AuthIdentity defines a request to obtain authentication credentials. These +// credentials would be used to further communicate with endpoint(s) that are +// protected by a form of authentication. +// +// Any strings that are annotated as "PEM-encoded" implies that encoding format +// is used, with any newlines indicated with `\n` characters. Most languages +// provide encoders that correctly marshal this out. For more information, +// see the RFC here: https://tools.ietf.org/html/rfc7468 +type AuthIdentity struct { + Csr string `json:"csr,omitempty"` +} diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go new file mode 100644 index 00000000..6a9deb64 --- /dev/null +++ b/applications/vas-sample-app/cmd/main.go @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2020 Intel Corporation + +package main + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "log" + "net/http" + +) + +// Connectivity constants +const ( + EAAServerName = "eaa.openness" + EAAServerPort = "443" + EAAServPort = "80" + EAACommonName = "eaa.openness" +) + +var myURN URN + + +// VasConfig describes VAS JSON config file +type VasConfig struct { + Acceleration string `json:"Acceleration"` + Framework string `json:"Framework"` + Pipelines []string `json:"Pipelines"` +} + +func authenticate(prvKey *ecdsa.PrivateKey) (*x509.CertPool, tls.Certificate) { + certTemplate := x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: "media:consumer", + Organization: []string{"Intel Corporation"}, + }, + SignatureAlgorithm: x509.ECDSAWithSHA256, + EmailAddresses: []string{"hello@openness.org"}, + } + + log.Println("CSR creating certificate") + conCsrBytes, err := x509.CreateCertificateRequest(rand.Reader, + &certTemplate, prvKey) + if err != nil { + log.Fatal(err) + } + + csrMem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", + Bytes: conCsrBytes}) + + conID := AuthIdentity{ + Csr: string(csrMem), + } + + reqBody, err := json.Marshal(conID) + if err != nil { + log.Fatal(err) + } + log.Println("CSR POST /auth") + resp, err := http.Post("http://"+EAAServerName+":"+EAAServPort+"/auth", + "", bytes.NewBuffer(reqBody)) + if err != nil { + log.Fatal(err) + } + + var conCreds AuthCredentials + err = json.NewDecoder(resp.Body).Decode(&conCreds) + if err != nil { + log.Fatal(err) + } + + x509Encoded, err := x509.MarshalECPrivateKey(prvKey) + if err != nil { + log.Fatal(err) + } + + pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", + Bytes: x509Encoded}) + conCert, err := tls.X509KeyPair([]byte(conCreds.Certificate), + pemEncoded) + if err != nil { + log.Fatal(err) + } + + conCertPool := x509.NewCertPool() + for _, cert := range conCreds.CaPool { + ok := conCertPool.AppendCertsFromPEM([]byte(cert)) + if !ok { + log.Fatal("Error: failed to append cert") + } + } + + return conCertPool, conCert +} + +func discoverServices(client *http.Client) (ServiceList, error) { + var servList = ServiceList{} + + // Consumer discover services + req, err := http.NewRequest("GET", + "https://"+EAAServerName+":"+EAAServerPort+"/services", nil) + if err != nil { + log.Println("Service-discovery request creation failed:", err) + return servList, err + } + + resp, err := client.Do(req) + if err != nil { + log.Println("Service-discovery request failed:", err) + return servList, err + } + + // TODO check if service list is empty -> handle & exit program + + err = json.NewDecoder(resp.Body).Decode(&servList) + if err != nil { + log.Println("Service-list decode failed:", err) + return servList, err + } + + err = resp.Body.Close() + if err != nil { + return servList, err + } + + return servList, nil +} + +func main() { + log.Println("Video-analytics-service Consumer Started") + + myURN = URN{ + ID: "consumer", + Namespace: "media", + } + + // Authentication (CSR) + log.Println("CSR Started") + conPriv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + log.Fatal(err) + } + certPool, cert := authenticate(conPriv) + + // HTTPS client + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + Certificates: []tls.Certificate{cert}, + ServerName: EAACommonName, + }, + }, + Timeout: 0, + } + + // Service Discovery + log.Println("Service Discovery Started") + servList, err := discoverServices(client) + if err != nil { + log.Fatal(err) + return + } + var vasInfo VasConfig + for _, s := range servList.Services { + log.Println("Discoverd serive:") + log.Println(" URN.ID: ", s.URN.ID) + log.Println(" URN.Namespace:", s.URN.Namespace) + log.Println(" Description: ", s.Description) + log.Println(" EndpointURI: ", s.EndpointURI) + // Subscribe to all services related to my Namespace + if myURN.Namespace == s.URN.Namespace { + // Service Request to VA-Serving + err := json.Unmarshal(s.Info, &vasInfo) + if err != nil { + log.Println(err) + } + postVAServingRequest(s.EndpointURI, vasInfo.Pipelines[0]) + } + } +} diff --git a/applications/vas-sample-app/eaaInterface.go b/applications/vas-sample-app/eaaInterface.go deleted file mode 100644 index ed986b49..00000000 --- a/applications/vas-sample-app/eaaInterface.go +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright (c) 2020 Intel Corporation - -package main - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "fmt" - "log" - "net/http" - "time" - - "github.com/gorilla/websocket" - "github.com/otcshare/edgeapps/applications/sample-app/common" - "github.com/pkg/errors" -) - -// subscribeConsumer sends a consumer subscription POST request to the appliance -// Subscribe to a set of notification types, -// optionally specifying the exact producer by URN. -func serviceSubscription(client *http.Client, - notifs []common.NotificationDescriptor, path string) error { - - address := "https://" + common.Cfg.EdgeNodeEndpoint + - "/subscriptions/" + path - - payload, err := json.Marshal(notifs) - if err != nil { - return errors.Wrap(err, "Couldn't marshal notifications") - } - - req, err := http.NewRequest("POST", address, bytes.NewBuffer(payload)) - if err != nil { - return errors.Wrap(err, "Couldn't create POST request for ") - } - - respPost, err := client.Do(req) - if err != nil { - return errors.Wrap(err, "Couldn't send POST request to "+address) - } - defer func() { - if err := respPost.Body.Close(); err != nil { - log.Println("Failed to close response body") - } - }() - return nil -} - -// unsubscribeConsumer sends a consumer subscription POST request -// to the appliance -// Unsubscribe from a particular or entire namespace of producers. -func unsubscribeConsumer(c *http.Client, notifs []common.NotificationDescriptor, - path string) error { - - address := "https://" + common.Cfg.EdgeNodeEndpoint + - "/subscriptions/" + path - - payload, err := json.Marshal(notifs) - if err != nil { - return errors.Wrap(err, "Couldn't marshal notifications") - } - - req, err := http.NewRequest("DELETE", address, bytes.NewBuffer(payload)) - if err != nil { - return errors.Wrap(err, - fmt.Sprintf("%s %s", "Couldn't create DELETE request for", address)) - } - - respPost, err := c.Do(req) - if err != nil { - return errors.Wrap(err, "Couldn't send POST request to "+address) - } - defer func() { - if err := respPost.Body.Close(); err != nil { - log.Println("Failed to close response body") - } - }() - - return nil -} - -// connectConsumer sends a consumer notifications GET request to the appliance -// Connect to a secure WebSocket to receive streaming notifications -// (see reference for notification to consumer for schema of messages received -// on websocket). -func connectConsumer(client *http.Client) (*websocket.Conn, error) { - - transport, ok := client.Transport.(*http.Transport) - if !ok { - return nil, errors.New("HTTP client doens't have http.Transport") - } - - socket := &websocket.Dialer{ - TLSClientConfig: &tls.Config{ - RootCAs: transport.TLSClientConfig.RootCAs, - Certificates: []tls.Certificate{ - transport.TLSClientConfig.Certificates[0]}, - ServerName: common.Cfg.EaaCommonName, - }, - } - - hostHeader := http.Header{} - hostHeader.Add("Host", common.Cfg.Namespace+":"+common.Cfg.ConsumerAppID) - - conn, resp, err := socket.Dial("wss://"+common.Cfg.EdgeNodeEndpoint+ - "/notifications", hostHeader) - if err != nil { - return nil, errors.Wrap(err, "Couldn't dial to wss") - } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Println("Failed to close response body") - } - }() - - return conn, nil -} - -// getMsgFromConn retrieves a message from a connection and parses -// it to a notification struct -func getMessages(conn *websocket.Conn) { - - err := conn.SetReadDeadline(time.Now().Add(time.Second * - time.Duration(common.Cfg.ConsumerTimeout))) - if err != nil { - log.Println("Failed to set read dead line") - } - - for { - var resp common.NotificationToConsumer - _, message, err := conn.ReadMessage() - if err != nil { - fmt.Println("Stopped reading messages") - return - } - output := bytes.NewReader(message) - err = json.NewDecoder(output).Decode(&resp) - if err != nil { - log.Println("Cannot decode a message") - } else { - fmt.Printf("Received notification:\n Name: %v\n Version: %v\n"+ - " Payload: %v\n URN: %v\n", - resp.Name, resp.Version, string(resp.Payload), resp.URN) - } - } -} diff --git a/applications/vas-sample-app/start.sh b/applications/vas-sample-app/start.sh new file mode 100755 index 00000000..c2c7916e --- /dev/null +++ b/applications/vas-sample-app/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +go run main.go application.go eaa_interface.go +fg diff --git a/applications/vas-sample-app/vas-cons-app.yaml b/applications/vas-sample-app/vas-cons-app.yaml new file mode 100644 index 00000000..0a96d782 --- /dev/null +++ b/applications/vas-sample-app/vas-cons-app.yaml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- +apiVersion: v1 +kind: Pod +metadata: + name: vas-cons-app + namespace: default + labels: + name: vas-cons-app +spec: + containers: + - name: vas-cons-app + image: vas-cons-app:1.0 + imagePullPolicy: Never + volumeMounts: + - name: tmp + mountPath: /tmp + + volumes: + - name: tmp + hostPath: + path: /tmp From 33b86ffb8315712904719c81890eae01c50ff646 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Thu, 18 Jun 2020 17:47:10 +0800 Subject: [PATCH 03/80] add helmchart --- .../vas-sample-app/{ => build}/Dockerfile | 5 +- .../vas-sample-app/{ => build}/build-image.sh | 2 +- .../vas-sample-app/{ => build}/start.sh | 0 .../deployments/helm/Chart.yaml | 7 ++ .../deployments/helm/templates/_helpers.tpl | 36 ++++++ .../helm/templates/cdn_network_policy.yaml | 40 +++++++ .../helm/templates/cdn_nginx_deploy.yaml | 107 ++++++++++++++++++ .../helm/templates/cdn_nginx_pv.yaml | 33 ++++++ .../helm/templates/cdn_nginx_pvc.yaml | 23 ++++ .../helm/templates/cdn_nginx_service.yaml | 43 +++++++ .../helm/templates/cdn_secret.yaml | 25 ++++ .../deployments/helm/templates/configmap.yaml | 15 +++ .../helm/templates/vas-cons-app.yaml | 24 ++++ .../deployments/helm/values.yaml | 8 ++ .../{ => deployments/yaml}/vas-cons-app.yaml | 0 15 files changed, 364 insertions(+), 4 deletions(-) rename applications/vas-sample-app/{ => build}/Dockerfile (93%) rename applications/vas-sample-app/{ => build}/build-image.sh (84%) rename applications/vas-sample-app/{ => build}/start.sh (100%) create mode 100644 applications/vas-sample-app/deployments/helm/Chart.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/_helpers.tpl create mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/configmap.yaml create mode 100644 applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml create mode 100644 applications/vas-sample-app/deployments/helm/values.yaml rename applications/vas-sample-app/{ => deployments/yaml}/vas-cons-app.yaml (100%) diff --git a/applications/vas-sample-app/Dockerfile b/applications/vas-sample-app/build/Dockerfile similarity index 93% rename from applications/vas-sample-app/Dockerfile rename to applications/vas-sample-app/build/Dockerfile index ca349e71..04b45515 100644 --- a/applications/vas-sample-app/Dockerfile +++ b/applications/vas-sample-app/build/Dockerfile @@ -20,7 +20,6 @@ RUN cd /tmp && \ mv go /usr/local && \ rm -rf /tmp/go* - -COPY start.sh ./ -COPY cmd/ ./ +COPY ./build/start.sh ./ +COPY ./cmd ./ ENTRYPOINT ["./start.sh"] diff --git a/applications/vas-sample-app/build-image.sh b/applications/vas-sample-app/build/build-image.sh similarity index 84% rename from applications/vas-sample-app/build-image.sh rename to applications/vas-sample-app/build/build-image.sh index 98742c47..b9de9955 100755 --- a/applications/vas-sample-app/build-image.sh +++ b/applications/vas-sample-app/build/build-image.sh @@ -3,7 +3,7 @@ # PDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -sudo docker build \ +sudo docker build -f ./build/Dockerfile \ --build-arg http_proxy=$http_proxy \ --build-arg https_proxy=$https_proxy \ --build-arg no_proxy=$no_proxy \ diff --git a/applications/vas-sample-app/start.sh b/applications/vas-sample-app/build/start.sh similarity index 100% rename from applications/vas-sample-app/start.sh rename to applications/vas-sample-app/build/start.sh diff --git a/applications/vas-sample-app/deployments/helm/Chart.yaml b/applications/vas-sample-app/deployments/helm/Chart.yaml new file mode 100644 index 00000000..62874aa5 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/Chart.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +apiVersion: v2 +description: VAS sample +name: vas-sample +version: 0.0.1 diff --git a/applications/vas-sample-app/deployments/helm/templates/_helpers.tpl b/applications/vas-sample-app/deployments/helm/templates/_helpers.tpl new file mode 100644 index 00000000..23bc9d37 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/_helpers.tpl @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nginxcdn.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "nginxcdn.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nginxcdn.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + + diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml new file mode 100644 index 00000000..614d4748 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +{{- if not .Values.deployment.sriov }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "nginxcdn.fullname" . }}-policy + namespace: default +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ template "nginxcdn.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: +{{- if .Values.numNodes }} +{{- range $ipIdx := until ( int .Values.numNodes ) }} + - ipBlock: + cidr: 192.168.{{ add $ipIdx 1 }}.0/24 +{{- end }} +{{- else }} + - ipBlock: + cidr: 192.168.1.0/24 +{{- end }} + ports: + {{- if kindIs "map" .Values.nginx.http.server }} + - protocol: TCP + port: {{ default 80 .Values.nginx.http.server.port }} + - protocol: TCP + port: {{ default 443 .Values.nginx.http.server.sslport }} + {{- else }} + - protocol: TCP + port: 80 + - protocol: TCP + port: 443 + {{- end }} +{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml new file mode 100644 index 00000000..dd6f0919 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +{{- $release_name := include "nginxcdn.fullname" . }} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nginxcdn.fullname" . }}-deployment +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ template "nginxcdn.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "nginxcdn.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Values.deployment.sriov }} + annotations: + k8s.v1.cni.cncf.io/networks: sriov-openness +{{- end }} + spec: + tolerations: + - key: node-role.kube-ovn/master + effect: NoSchedule +{{- if not ( eq .Values.nginx.http.server.ext_ssl_certificate true ) }} + initContainers: + - name: openssl + image: emberstack/openssl:latest + command: [ "openssl", "req", "-x509", "-nodes", "-days", "36500", "-newkey", "rsa:2048", "-keyout", "/root/cdnssl/nginx.key", "-out", "/root/cdnssl/nginx.crt", "-subj", "/CN=cdn.openness" ] + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 1Gi + cpu: 1 + volumeMounts: + - name: cdn-ssl + mountPath: /root/cdnssl +{{- end }} + containers: + - name: cdn-nginx + image: {{ if .Values.deployment.image }} {{- .Values.deployment.image -}} {{ else }} nginx:1.16.1 {{ end }} + imagePullPolicy: IfNotPresent + command: [ "nginx", "-g", "daemon off;"] + securityContext: + runAsUser: 0 + privileged: true + resources: + limits: +{{- if .Values.deployment.hugepages2M }} + hugepages-2Mi: {{ .Values.deployment.hugepages2M }} +{{- end }} + memory: {{ if .Values.deployment.memory }} {{- .Values.deployment.memory -}} {{ else }} 10Gi {{ end }} + cpu: {{ if .Values.deployment.cpuLimit }} {{- .Values.deployment.cpuLimit -}} {{ else if .Values.nginx.worker_processes }} {{- (.Values.nginx.worker_processes | add1 | int) -}} {{ else }} 5 {{ end }} +{{- if .Values.deployment.sriov }} + intel.com/intel_sriov_netdevice: "1" +{{- end }} + volumeMounts: +{{- range $index, $val := .Values.nginx.http.proxy_cache }} + - mountPath: /cdn/cache{{ $index }} + name: cdn-cache{{ $index }} +{{- end }} + - mountPath: /etc/nginx/nginx.conf + name: cdn-conf + subPath: nginx.conf + - mountPath: /etc/nginx/ssl + name: cdn-ssl + volumes: +{{- range $index, $val := .Values.nginx.http.proxy_cache }} + - name: cdn-cache{{ $index }} +{{- if $.Values.CDNNode }} + persistentVolumeClaim: + claimName: {{ $release_name }}-pvc{{ $index }} +{{- else }} + hostPath: + path: {{ $val.hostPath }} + type: DirectoryOrCreate +{{- end }} +{{- end }} + - name: cdn-conf + configMap: + name: {{ template "nginxcdn.fullname" . }}-config + items: + - key: nginxconf + path: nginx.conf +{{- if eq .Values.nginx.http.server.ext_ssl_certificate true }} + - name: cdn-ssl + secret: + secretName: {{ template "nginxcdn.fullname" . }}-secret +{{- else }} + - name: cdn-ssl + hostPath: + path: /etc/openness/cdn/ssl + type: DirectoryOrCreate +{{- end }} +{{- if or .Values.deployment.sriov .Values.SSDDisks }} + nodeSelector: +{{- if .Values.deployment.sriov }} + feature.node.kubernetes.io/network-sriov.capable: 'true' +{{- end }} +{{- if .Values.SSDDisks }} + feature.node.kubernetes.io/storage-nonrotationaldisk: 'true' +{{- end }} +{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml new file mode 100644 index 00000000..ced4996b --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +{{- $release_name := include "nginxcdn.fullname" . }} + +{{- if .Values.CDNNode }} +{{- range $index, $val := .Values.nginx.http.proxy_cache }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ $release_name }}-pv{{ $index }} + labels: + type: local +spec: + capacity: + storage: {{ upper $val.max_size }}i + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: local-storage + local: + path: {{ $val.hostPath }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - {{ $.Values.CDNNode }} +--- +{{- end }} +{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml new file mode 100644 index 00000000..fe4d1095 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +{{- $release_name := include "nginxcdn.fullname" . }} + +{{- if .Values.CDNNode }} +{{- range $index, $val := .Values.nginx.http.proxy_cache }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $release_name }}-pvc{{ $index }} +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-storage + resources: + requests: + storage: {{ upper $val.max_size }}i + volumeName: {{ $release_name }}-pv{{ $index }} +--- +{{- end }} +{{- end }} + diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml new file mode 100644 index 00000000..876c9576 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +{{- if .Values.CDNService }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "nginxcdn.fullname" . }}-service +spec: +{{- if .Values.CDNService.type }} + type: {{ .Values.CDNService.type }} +{{- end }} + selector: + app.kubernetes.io/name: {{ template "nginxcdn.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - name: http +{{- if kindIs "map" .Values.nginx.http.server }} + port: {{ default "80" .Values.nginx.http.server.port }} +{{- else }} + port: 80 +{{- end }} +{{- if .Values.CDNService.type }} +{{- if eq .Values.CDNService.type "NodePort" }} +{{- if .Values.CDNService.httpNodePort }} + nodePort: {{ .Values.CDNService.httpNodePort }} +{{- end }} +{{- end }} +{{- end }} + - name: https +{{- if kindIs "map" .Values.nginx.http.server }} + port: {{ default "443" .Values.nginx.http.server.sslport }} +{{- else }} + port: 443 +{{- end }} +{{- if .Values.CDNService.type }} +{{- if eq .Values.CDNService.type "NodePort" }} +{{- if .Values.CDNService.httpsNodePort }} + nodePort: {{ .Values.CDNService.httpsNodePort }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml new file mode 100644 index 00000000..ea011fe8 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +{{- if kindIs "map" .Values.nginx.http.server }} +{{- if eq ( .Values.nginx.http.server.ext_ssl_certificate ) true }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "nginxcdn.fullname" . }}-secret +type: Opaque +data: +{{- if .Values.nginx.http.server.ssl_cert_file }} + nginx.crt: + {{ ( .Values.nginx.http.server.ssl_cert_file | b64enc ) | quote }} +{{- else }} + {{ fail "ssl certificate file missing" }} +{{- end }} +{{- if .Values.nginx.http.server.ssl_key_file }} + nginx.key: + {{ ( .Values.nginx.http.server.ssl_key_file | b64enc ) | quote }} +{{- else }} + {{ fail "ssl certificate key missing" }} +{{- end }} +{{- end }} +{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/configmap.yaml b/applications/vas-sample-app/deployments/helm/templates/configmap.yaml new file mode 100644 index 00000000..1bff71b4 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/configmap.yaml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "nginxcdn.fullname" . }}-config + labels: + helm.sh/chart: {{ template "nginxcdn.chart" . }} + app.kubernetes.io/name: {{ template "nginxcdn.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +data: + nginxconf: + {{ tpl (.Files.Get "nginx.conf") . | quote }} diff --git a/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml b/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml new file mode 100644 index 00000000..189ac297 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Release.Name }}-vas-cons-app + namespace: default + labels: + name: {{ .Release.Name }}-vas-cons-app +spec: + containers: + - name: {{ .Release.Name }}-vas-cons-app + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.pullPolicy }} + volumeMounts: + - name: tmp + mountPath: /tmp + + volumes: + - name: tmp + hostPath: + path: /tmp diff --git a/applications/vas-sample-app/deployments/helm/values.yaml b/applications/vas-sample-app/deployments/helm/values.yaml new file mode 100644 index 00000000..a44ed5d1 --- /dev/null +++ b/applications/vas-sample-app/deployments/helm/values.yaml @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- +image: + repository: vas-cons-app + tag: 1.0 +pullPolicy: Never diff --git a/applications/vas-sample-app/vas-cons-app.yaml b/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml similarity index 100% rename from applications/vas-sample-app/vas-cons-app.yaml rename to applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml From d50e4f09f587c23f5d367d0b6af54d8d55ccb8ce Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Thu, 18 Jun 2020 17:48:26 +0800 Subject: [PATCH 04/80] helm charts --- .../deployments/helm/templates/_helpers.tpl | 36 ------ .../helm/templates/cdn_network_policy.yaml | 40 ------- .../helm/templates/cdn_nginx_deploy.yaml | 107 ------------------ .../helm/templates/cdn_nginx_pv.yaml | 33 ------ .../helm/templates/cdn_nginx_pvc.yaml | 23 ---- .../helm/templates/cdn_nginx_service.yaml | 43 ------- .../helm/templates/cdn_secret.yaml | 25 ---- .../deployments/helm/templates/configmap.yaml | 15 --- 8 files changed, 322 deletions(-) delete mode 100644 applications/vas-sample-app/deployments/helm/templates/_helpers.tpl delete mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml delete mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml delete mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml delete mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml delete mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml delete mode 100644 applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml delete mode 100644 applications/vas-sample-app/deployments/helm/templates/configmap.yaml diff --git a/applications/vas-sample-app/deployments/helm/templates/_helpers.tpl b/applications/vas-sample-app/deployments/helm/templates/_helpers.tpl deleted file mode 100644 index 23bc9d37..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/_helpers.tpl +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "nginxcdn.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Expand the name of the chart. -*/}} -{{- define "nginxcdn.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "nginxcdn.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - - diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml deleted file mode 100644 index 614d4748..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/cdn_network_policy.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -{{- if not .Values.deployment.sriov }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ include "nginxcdn.fullname" . }}-policy - namespace: default -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: {{ template "nginxcdn.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - policyTypes: - - Ingress - ingress: - - from: -{{- if .Values.numNodes }} -{{- range $ipIdx := until ( int .Values.numNodes ) }} - - ipBlock: - cidr: 192.168.{{ add $ipIdx 1 }}.0/24 -{{- end }} -{{- else }} - - ipBlock: - cidr: 192.168.1.0/24 -{{- end }} - ports: - {{- if kindIs "map" .Values.nginx.http.server }} - - protocol: TCP - port: {{ default 80 .Values.nginx.http.server.port }} - - protocol: TCP - port: {{ default 443 .Values.nginx.http.server.sslport }} - {{- else }} - - protocol: TCP - port: 80 - - protocol: TCP - port: 443 - {{- end }} -{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml deleted file mode 100644 index dd6f0919..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_deploy.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -{{- $release_name := include "nginxcdn.fullname" . }} - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "nginxcdn.fullname" . }}-deployment -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: {{ template "nginxcdn.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ template "nginxcdn.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} -{{- if .Values.deployment.sriov }} - annotations: - k8s.v1.cni.cncf.io/networks: sriov-openness -{{- end }} - spec: - tolerations: - - key: node-role.kube-ovn/master - effect: NoSchedule -{{- if not ( eq .Values.nginx.http.server.ext_ssl_certificate true ) }} - initContainers: - - name: openssl - image: emberstack/openssl:latest - command: [ "openssl", "req", "-x509", "-nodes", "-days", "36500", "-newkey", "rsa:2048", "-keyout", "/root/cdnssl/nginx.key", "-out", "/root/cdnssl/nginx.crt", "-subj", "/CN=cdn.openness" ] - imagePullPolicy: IfNotPresent - resources: - limits: - memory: 1Gi - cpu: 1 - volumeMounts: - - name: cdn-ssl - mountPath: /root/cdnssl -{{- end }} - containers: - - name: cdn-nginx - image: {{ if .Values.deployment.image }} {{- .Values.deployment.image -}} {{ else }} nginx:1.16.1 {{ end }} - imagePullPolicy: IfNotPresent - command: [ "nginx", "-g", "daemon off;"] - securityContext: - runAsUser: 0 - privileged: true - resources: - limits: -{{- if .Values.deployment.hugepages2M }} - hugepages-2Mi: {{ .Values.deployment.hugepages2M }} -{{- end }} - memory: {{ if .Values.deployment.memory }} {{- .Values.deployment.memory -}} {{ else }} 10Gi {{ end }} - cpu: {{ if .Values.deployment.cpuLimit }} {{- .Values.deployment.cpuLimit -}} {{ else if .Values.nginx.worker_processes }} {{- (.Values.nginx.worker_processes | add1 | int) -}} {{ else }} 5 {{ end }} -{{- if .Values.deployment.sriov }} - intel.com/intel_sriov_netdevice: "1" -{{- end }} - volumeMounts: -{{- range $index, $val := .Values.nginx.http.proxy_cache }} - - mountPath: /cdn/cache{{ $index }} - name: cdn-cache{{ $index }} -{{- end }} - - mountPath: /etc/nginx/nginx.conf - name: cdn-conf - subPath: nginx.conf - - mountPath: /etc/nginx/ssl - name: cdn-ssl - volumes: -{{- range $index, $val := .Values.nginx.http.proxy_cache }} - - name: cdn-cache{{ $index }} -{{- if $.Values.CDNNode }} - persistentVolumeClaim: - claimName: {{ $release_name }}-pvc{{ $index }} -{{- else }} - hostPath: - path: {{ $val.hostPath }} - type: DirectoryOrCreate -{{- end }} -{{- end }} - - name: cdn-conf - configMap: - name: {{ template "nginxcdn.fullname" . }}-config - items: - - key: nginxconf - path: nginx.conf -{{- if eq .Values.nginx.http.server.ext_ssl_certificate true }} - - name: cdn-ssl - secret: - secretName: {{ template "nginxcdn.fullname" . }}-secret -{{- else }} - - name: cdn-ssl - hostPath: - path: /etc/openness/cdn/ssl - type: DirectoryOrCreate -{{- end }} -{{- if or .Values.deployment.sriov .Values.SSDDisks }} - nodeSelector: -{{- if .Values.deployment.sriov }} - feature.node.kubernetes.io/network-sriov.capable: 'true' -{{- end }} -{{- if .Values.SSDDisks }} - feature.node.kubernetes.io/storage-nonrotationaldisk: 'true' -{{- end }} -{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml deleted file mode 100644 index ced4996b..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pv.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -{{- $release_name := include "nginxcdn.fullname" . }} - -{{- if .Values.CDNNode }} -{{- range $index, $val := .Values.nginx.http.proxy_cache }} -apiVersion: v1 -kind: PersistentVolume -metadata: - name: {{ $release_name }}-pv{{ $index }} - labels: - type: local -spec: - capacity: - storage: {{ upper $val.max_size }}i - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - storageClassName: local-storage - local: - path: {{ $val.hostPath }} - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - {{ $.Values.CDNNode }} ---- -{{- end }} -{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml deleted file mode 100644 index fe4d1095..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_pvc.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -{{- $release_name := include "nginxcdn.fullname" . }} - -{{- if .Values.CDNNode }} -{{- range $index, $val := .Values.nginx.http.proxy_cache }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ $release_name }}-pvc{{ $index }} -spec: - accessModes: - - ReadWriteOnce - storageClassName: local-storage - resources: - requests: - storage: {{ upper $val.max_size }}i - volumeName: {{ $release_name }}-pv{{ $index }} ---- -{{- end }} -{{- end }} - diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml deleted file mode 100644 index 876c9576..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/cdn_nginx_service.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -{{- if .Values.CDNService }} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "nginxcdn.fullname" . }}-service -spec: -{{- if .Values.CDNService.type }} - type: {{ .Values.CDNService.type }} -{{- end }} - selector: - app.kubernetes.io/name: {{ template "nginxcdn.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - ports: - - name: http -{{- if kindIs "map" .Values.nginx.http.server }} - port: {{ default "80" .Values.nginx.http.server.port }} -{{- else }} - port: 80 -{{- end }} -{{- if .Values.CDNService.type }} -{{- if eq .Values.CDNService.type "NodePort" }} -{{- if .Values.CDNService.httpNodePort }} - nodePort: {{ .Values.CDNService.httpNodePort }} -{{- end }} -{{- end }} -{{- end }} - - name: https -{{- if kindIs "map" .Values.nginx.http.server }} - port: {{ default "443" .Values.nginx.http.server.sslport }} -{{- else }} - port: 443 -{{- end }} -{{- if .Values.CDNService.type }} -{{- if eq .Values.CDNService.type "NodePort" }} -{{- if .Values.CDNService.httpsNodePort }} - nodePort: {{ .Values.CDNService.httpsNodePort }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml b/applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml deleted file mode 100644 index ea011fe8..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/cdn_secret.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -{{- if kindIs "map" .Values.nginx.http.server }} -{{- if eq ( .Values.nginx.http.server.ext_ssl_certificate ) true }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "nginxcdn.fullname" . }}-secret -type: Opaque -data: -{{- if .Values.nginx.http.server.ssl_cert_file }} - nginx.crt: - {{ ( .Values.nginx.http.server.ssl_cert_file | b64enc ) | quote }} -{{- else }} - {{ fail "ssl certificate file missing" }} -{{- end }} -{{- if .Values.nginx.http.server.ssl_key_file }} - nginx.key: - {{ ( .Values.nginx.http.server.ssl_key_file | b64enc ) | quote }} -{{- else }} - {{ fail "ssl certificate key missing" }} -{{- end }} -{{- end }} -{{- end }} diff --git a/applications/vas-sample-app/deployments/helm/templates/configmap.yaml b/applications/vas-sample-app/deployments/helm/templates/configmap.yaml deleted file mode 100644 index 1bff71b4..00000000 --- a/applications/vas-sample-app/deployments/helm/templates/configmap.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "nginxcdn.fullname" . }}-config - labels: - helm.sh/chart: {{ template "nginxcdn.chart" . }} - app.kubernetes.io/name: {{ template "nginxcdn.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} -data: - nginxconf: - {{ tpl (.Files.Get "nginx.conf") . | quote }} From f43a5bd9a7174d964761ed101288c35f6c862cbe Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Wed, 24 Jun 2020 16:32:58 +0800 Subject: [PATCH 05/80] add readme for vas sample application --- applications/vas-sample-app/README.md | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 applications/vas-sample-app/README.md diff --git a/applications/vas-sample-app/README.md b/applications/vas-sample-app/README.md new file mode 100644 index 00000000..9d8d810c --- /dev/null +++ b/applications/vas-sample-app/README.md @@ -0,0 +1,31 @@ +```text +SPDX-License-Identifier: Apache-2.0 +Copyright (c) 2020 Intel Corporation +``` + +# VAS Sample Application in OpenNESS + +This sample application demonstrates VAS consumer sample application deployment and execution on the OpenNESS edge platform with VAS enabled. + + +# Introduction +## Directory Structure +- `/cmd` : Main applications inside. +- `/build` : Building scripts in the folder. +- `/deployments` : Helm Charts and K8s yaml files inside. + + +## Quick Start +To build sample application container image: + +```sh +./build/build-image.sh +``` + +To deploy and test the sample application: + +- Make sure the sample application images built and reachable +- Two method for the deployment on the openness edge node + - Use the command ```kubectl apply``` with the yaml file ```vas-cons-app.yaml``` in the folder - deployments/yaml + - Or use helm charts in the folder - deployments/helm +- Check Results by ```kubectl logs``` From cf6dd205d071d6f83c2bf6200aea0f29a1f2a92c Mon Sep 17 00:00:00 2001 From: nathanmur Date: Wed, 24 Jun 2020 15:36:48 -0400 Subject: [PATCH 06/80] Fixed docker build + tag used in helm --- applications/vas-sample-app/build/Dockerfile | 6 ++++-- applications/vas-sample-app/build/start.sh | 2 +- .../cmd/{eaaInterface.go => eaa_interface.go} | 0 applications/vas-sample-app/deployments/helm/values.yaml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) rename applications/vas-sample-app/cmd/{eaaInterface.go => eaa_interface.go} (100%) diff --git a/applications/vas-sample-app/build/Dockerfile b/applications/vas-sample-app/build/Dockerfile index 04b45515..3a39d5ba 100644 --- a/applications/vas-sample-app/build/Dockerfile +++ b/applications/vas-sample-app/build/Dockerfile @@ -20,6 +20,8 @@ RUN cd /tmp && \ mv go /usr/local && \ rm -rf /tmp/go* -COPY ./build/start.sh ./ -COPY ./cmd ./ +WORKDIR /root + +COPY build/start.sh ./ +COPY cmd/ ./ ENTRYPOINT ["./start.sh"] diff --git a/applications/vas-sample-app/build/start.sh b/applications/vas-sample-app/build/start.sh index c2c7916e..b9b198b2 100755 --- a/applications/vas-sample-app/build/start.sh +++ b/applications/vas-sample-app/build/start.sh @@ -4,4 +4,4 @@ # Copyright (c) 2020 Intel Corporation go run main.go application.go eaa_interface.go -fg + diff --git a/applications/vas-sample-app/cmd/eaaInterface.go b/applications/vas-sample-app/cmd/eaa_interface.go similarity index 100% rename from applications/vas-sample-app/cmd/eaaInterface.go rename to applications/vas-sample-app/cmd/eaa_interface.go diff --git a/applications/vas-sample-app/deployments/helm/values.yaml b/applications/vas-sample-app/deployments/helm/values.yaml index a44ed5d1..65224316 100644 --- a/applications/vas-sample-app/deployments/helm/values.yaml +++ b/applications/vas-sample-app/deployments/helm/values.yaml @@ -4,5 +4,5 @@ --- image: repository: vas-cons-app - tag: 1.0 + tag: "1.0" pullPolicy: Never From 9454c731aa7d6e75d02103dae216bd9a96ff8b77 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Sun, 28 Jun 2020 10:14:28 +0800 Subject: [PATCH 07/80] fix go dep issue --- applications/vas-sample-app/build/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/applications/vas-sample-app/build/Dockerfile b/applications/vas-sample-app/build/Dockerfile index 3a39d5ba..fcc38727 100644 --- a/applications/vas-sample-app/build/Dockerfile +++ b/applications/vas-sample-app/build/Dockerfile @@ -11,7 +11,7 @@ ENV GOPATH=/root/go ENV GOROOT=/usr/local/go ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH -RUN yum install -y wget +RUN yum install -y wget git # Install Go RUN cd /tmp && \ @@ -20,8 +20,11 @@ RUN cd /tmp && \ mv go /usr/local && \ rm -rf /tmp/go* +# Get go dependencies +RUN go get github.com/levigross/grequests + WORKDIR /root COPY build/start.sh ./ COPY cmd/ ./ -ENTRYPOINT ["./start.sh"] +#ENTRYPOINT ["./start.sh"] From 37fe2120cb22bd459840a5983239c272c7e3b853 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Sun, 28 Jun 2020 18:59:44 +0800 Subject: [PATCH 08/80] minor issue in dockerfile --- applications/vas-sample-app/build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/vas-sample-app/build/Dockerfile b/applications/vas-sample-app/build/Dockerfile index fcc38727..8a8eafb2 100644 --- a/applications/vas-sample-app/build/Dockerfile +++ b/applications/vas-sample-app/build/Dockerfile @@ -27,4 +27,4 @@ WORKDIR /root COPY build/start.sh ./ COPY cmd/ ./ -#ENTRYPOINT ["./start.sh"] +ENTRYPOINT ["./start.sh"] From 8accad94ec557ae7e633dd5e98b0c5fb3d450b90 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Sun, 28 Jun 2020 19:23:28 +0800 Subject: [PATCH 09/80] fix no proxy issue --- applications/vas-sample-app/build/Dockerfile | 2 +- applications/vas-sample-app/cmd/main.go | 2 +- .../vas-sample-app/deployments/helm/templates/vas-cons-app.yaml | 1 + applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/applications/vas-sample-app/build/Dockerfile b/applications/vas-sample-app/build/Dockerfile index 8a8eafb2..014cf9fd 100644 --- a/applications/vas-sample-app/build/Dockerfile +++ b/applications/vas-sample-app/build/Dockerfile @@ -5,7 +5,7 @@ FROM centos:7.6.1810 ENV http_proxy=$http_proxy ENV https_proxy=$https_proxy -ENV no_proxy=$no_proxy,eaa.openness,media.openness +ENV no_proxy=$no_proxy,eaa.openness,analytics-ffmpeg,analytics-gstreamer,media.openness ENV GOPATH=/root/go ENV GOROOT=/usr/local/go diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index 6a9deb64..24eb6637 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -139,7 +139,7 @@ func main() { myURN = URN{ ID: "consumer", - Namespace: "media", + Namespace: "openvino", } // Authentication (CSR) diff --git a/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml b/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml index 189ac297..b7c7e62a 100644 --- a/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml +++ b/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml @@ -10,6 +10,7 @@ metadata: labels: name: {{ .Release.Name }}-vas-cons-app spec: + restartPolicy: Never containers: - name: {{ .Release.Name }}-vas-cons-app image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml b/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml index 0a96d782..1b8f1e7b 100644 --- a/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml +++ b/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml @@ -10,6 +10,7 @@ metadata: labels: name: vas-cons-app spec: + restartPolicy: Never containers: - name: vas-cons-app image: vas-cons-app:1.0 From 58e0a7dc12fb60374a46523fe9a0e4d56bab4835 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Tue, 30 Jun 2020 15:42:58 +0800 Subject: [PATCH 10/80] update readme --- applications/vas-sample-app/README.md | 57 ++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/applications/vas-sample-app/README.md b/applications/vas-sample-app/README.md index 9d8d810c..d2a6e717 100644 --- a/applications/vas-sample-app/README.md +++ b/applications/vas-sample-app/README.md @@ -16,16 +16,69 @@ This sample application demonstrates VAS consumer sample application deployment ## Quick Start -To build sample application container image: +### To build sample application container image: ```sh ./build/build-image.sh ``` -To deploy and test the sample application: +After build image, you can directly push the image to docker registry on the controller: + +```sh +docker push :5000/vas-cons-app:1.0 +``` + +Or you can directly build image on the edgenode. + + +### To deploy and test the sample application: - Make sure the sample application images built and reachable - Two method for the deployment on the openness edge node - Use the command ```kubectl apply``` with the yaml file ```vas-cons-app.yaml``` in the folder - deployments/yaml - Or use helm charts in the folder - deployments/helm - Check Results by ```kubectl logs``` +NOTE: If image pushed to docker registry on the controller, you need to edit: ```deployments/helm/values.yaml``` or ```deployments/yaml/vas-cons-app.yaml``` - change ```repository``` to IP address of Docker Registry . + + +### To check test results: + +1. check whether the application pod is successfully deployed. +```sh +# kubectl get pods -A | grep vas-cons-app +default vas-cons-app 0/1 Completed 0 30h +``` + +2. check the pod's logs as below expected results: + +```sh +# kubectl logs vas-cons-app +2020/06/29 01:37:25 Video-analytics-service Consumer Started +2020/06/29 01:37:25 CSR Started +2020/06/29 01:37:25 CSR creating certificate +2020/06/29 01:37:25 CSR POST /auth +2020/06/29 01:37:25 Service Discovery Started +2020/06/29 01:37:25 Discoverd serive: +2020/06/29 01:37:25 URN.ID: producer +2020/06/29 01:37:25 Description: Video Analytics Service +2020/06/29 01:37:25 EndpointURI: http://analytics-gstreamer:8080 +2020/06/29 01:37:25 Starting POST Request:http://analytics-gstreamer:8080/pipelines/emotion_recognition/1 +2020/06/29 01:37:25 Pipeline Instance Created: 1 + +2020/06/29 01:37:25 { + "avg_fps": 0, + "elapsed_time": 0.005330085754394531, + "id": 1, + "start_time": 1593394645.8940227, + "state": "QUEUED" +} + +2020/06/29 01:37:35 { + "avg_fps": 133.54638124371084, + "elapsed_time": 8.90360951423645, + "id": 1, + "start_time": 1593394645.8940227, + "state": "COMPLETED" +} +``` + From f4f8c0ff5c1c97bb01b01600898f5bc1bcfde64b Mon Sep 17 00:00:00 2001 From: Patryk Strusiewicz-Surmacki Date: Thu, 2 Jul 2020 11:54:07 +0200 Subject: [PATCH 11/80] Fixed build of sample-app --- applications/go.mod | 12 ------------ applications/sample-app/Makefile | 3 --- applications/sample-app/go.mod | 8 ++++++++ applications/{ => sample-app}/go.sum | 2 -- 4 files changed, 8 insertions(+), 17 deletions(-) delete mode 100644 applications/go.mod create mode 100644 applications/sample-app/go.mod rename applications/{ => sample-app}/go.sum (59%) diff --git a/applications/go.mod b/applications/go.mod deleted file mode 100644 index 823ab17b..00000000 --- a/applications/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright (c) 2020 Intel Corporation - -module sample-app - -go 1.14.2 - -require ( - github.com/gorilla/websocket v1.4.2 - github.com/otcshare/edgeapps v0.0.0-20200513043854-a3d6e4a2e67d - github.com/pkg/errors v0.9.1 -) diff --git a/applications/sample-app/Makefile b/applications/sample-app/Makefile index d534dba4..449890f9 100644 --- a/applications/sample-app/Makefile +++ b/applications/sample-app/Makefile @@ -6,7 +6,6 @@ export GO111MODULE = on .PHONY: build build-docker consumer consumer-docker producer producer-docker clean lint help TMP_DIR:=$(shell mktemp -d) BUILD_DIR ?=build -CURRENT_DIR:=$(shell pwd) VER:=1.0 @@ -16,7 +15,6 @@ build-docker: consumer-docker producer-docker consumer: export GOSUMDB=off - cd ${CURRENT_DIR}/.. && go mod init sample-app || cd ${CURRENT_DIR} mkdir -p "${BUILD_DIR}" GOOS=linux go build -o "${BUILD_DIR}/consumer" ./simpleEaaConsumer unset GOSUMDB @@ -29,7 +27,6 @@ consumer-docker: consumer producer: export GOSUMDB=off - cd ${CURRENT_DIR}/.. && go mod init sample-app || cd ${CURRENT_DIR} mkdir -p "${BUILD_DIR}" GOOS=linux go build -o "${BUILD_DIR}/producer" ./simpleEaaProducer unset GOSUMDB diff --git a/applications/sample-app/go.mod b/applications/sample-app/go.mod new file mode 100644 index 00000000..0dcc8170 --- /dev/null +++ b/applications/sample-app/go.mod @@ -0,0 +1,8 @@ +module github.com/otcshare/edgeapps/applications/sample-app + +go 1.14.2 + +require ( + github.com/gorilla/websocket v1.4.2 + github.com/pkg/errors v0.9.1 +) diff --git a/applications/go.sum b/applications/sample-app/go.sum similarity index 59% rename from applications/go.sum rename to applications/sample-app/go.sum index 64abd7cc..d27541e7 100644 --- a/applications/go.sum +++ b/applications/sample-app/go.sum @@ -1,6 +1,4 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/otcshare/edgeapps v0.0.0-20200513043854-a3d6e4a2e67d h1:Uzx9ed8O51XOdsvaL1r6EvnRul4evUQAQj47GCnA5Xw= -github.com/otcshare/edgeapps v0.0.0-20200513043854-a3d6e4a2e67d/go.mod h1:Jk67iWGIVe9yzeyzMWKBtEA6dB/rXtFZRAdD8gp0yTM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 78eca643e65b7a255de7add462139e5d7dacd24f Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Tue, 21 Jul 2020 12:46:45 +0800 Subject: [PATCH 12/80] 1/ namespace should be default 2/ add debug message when namespace mismatch --- applications/vas-sample-app/cmd/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index 24eb6637..7dd04735 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -139,7 +139,7 @@ func main() { myURN = URN{ ID: "consumer", - Namespace: "openvino", + Namespace: "default", } // Authentication (CSR) @@ -184,6 +184,8 @@ func main() { log.Println(err) } postVAServingRequest(s.EndpointURI, vasInfo.Pipelines[0]) - } + } else { + log.Println("Namespace mismatch, myURN namespace:",myURN.Namespace) + } } } From 94a0453e4371ec339e1aa1991570ef62b0308b04 Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Wed, 22 Jul 2020 15:42:01 +0800 Subject: [PATCH 13/80] fix checkmax error --- applications/vas-sample-app/cmd/main.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index 7dd04735..05b1a4fc 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -169,8 +169,15 @@ func main() { log.Fatal(err) return } + + sum := 0 var vasInfo VasConfig - for _, s := range servList.Services { + for _, s := range servList.Services { + sum ++ + if sum > 100 { + log.Fatal("abnormal services num") + return + } log.Println("Discoverd serive:") log.Println(" URN.ID: ", s.URN.ID) log.Println(" URN.Namespace:", s.URN.Namespace) @@ -187,5 +194,5 @@ func main() { } else { log.Println("Namespace mismatch, myURN namespace:",myURN.Namespace) } - } + } } From 3587e3655f994cf54f8a48eaadf69469f161eb6d Mon Sep 17 00:00:00 2001 From: Amr Mokhtar Date: Wed, 22 Jul 2020 12:31:52 +0100 Subject: [PATCH 14/80] Fix openvino build + editorials --- applications/cdn-caching/nginx/README.md | 16 ++++++++-------- applications/openvino/consumer/cmd/openvino.go | 9 ++++----- applications/smart-city-app/README.md | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/applications/cdn-caching/nginx/README.md b/applications/cdn-caching/nginx/README.md index 8635ca6e..063729f1 100644 --- a/applications/cdn-caching/nginx/README.md +++ b/applications/cdn-caching/nginx/README.md @@ -2,7 +2,7 @@ SPDX-License-Identifier: Apache-2.0 Copyright (c) 2020 Intel Corporation ``` - + # CDN Application in OpenNESS This sample application demonstrates CDN (Content Delivery Network) deployment and execution on the OpenNESS edge platform. @@ -13,7 +13,7 @@ This sample application demonstrates CDN (Content Delivery Network) deployment a - [Setting up kubeovn interface](#setting-up-kubeovn-interface) - [Setting up the SR-IOV interface](#setting-up-the-sr-iov-interface) - [Deploying the Application](#deploying-the-application) - - [Custom SSL certificate keys for the nginx](#Custom-ssl-certificate-keys-for-the-nginx) + - [Custom SSL certificate keys for the nginx](#custom-ssl-certificate-keys-for-the-nginx) - [Testing using wrk](#testing-using-wrk) @@ -21,7 +21,7 @@ This sample application demonstrates CDN (Content Delivery Network) deployment a NGINX is configured to function as a Content Delivery Network. This section describes the deployment of nginx based CDN on an openNESS platform using helm charts and testing the CDN. ## Onboarding nginx based CDN application -The purpose of this section is to guide the user on the complete process of onboarding the nginx based CDN application. This process will also guide the user on setting up network connection between test machine and the application. +The purpose of this section is to guide the user on the complete process of onboarding the nginx based CDN application. This process will also guide the user on setting up network connection between test machine and the application. ### Prerequisites @@ -71,7 +71,7 @@ Extra setup is not required when using the control interface for data / content ... 0000:86:00.0 | 3c:fd:fe:b2:42:d0 | detached ... - + kubectl interfaceservice attach 0000:86:00.0 ... Interface: 0000:86:00.0 successfully attached @@ -83,7 +83,7 @@ Extra setup is not required when using the control interface for data / content ... ``` 3. To enable the kube-ovn interface, set the helm key "sriov" to false and also set the "numNodes" to the number of available nodes in the cluster. The numNodes is used to generate the network policy resources to be used in the openNESS platform. - + #### Setting up the SR-IOV interface 1. Please refer to the link for setting up the SR-IOV interface: [Setting up SR-IOV for network edge](https://github.com/otcshare/specs/blob/master/doc/enhanced-platform-awareness/openness-sriov-multiple-interfaces.md#sriov-for-network-edge) @@ -95,7 +95,7 @@ Extra setup is not required when using the control interface for data / content Helm is used to deploy the CDN application. Please refer to [Helm website](https://helm.sh/) for further details. Clone the edgeapps repo and the nginx based CDN helm chart is available under "applications/cdn-caching/nginx/". -> **Note:** The values.yaml file contain the key and default values used to customize the templates, during the helm deployment. Comments that explain the roles of the keys and their possible vaues are added in the values.yaml file. The example folder will contain further example values files. +> **Note:** The values.yaml file contain the key and default values used to customize the templates, during the helm deployment. Comments that explain the roles of the keys and their possible vaues are added in the values.yaml file. The example folder will contain further example values files. > **Note:** Provide the proper values for the origin server and the disk storage locations, where the contents need to be cached. @@ -107,7 +107,7 @@ The ssl certificates for the nginx, can be passed in the command line as below, helm install -f --set-file nginx.http.server.ssl_cert_file= --set-file nginx.http.server.ssl_key_file= # example -helm install -f ./examples/sriov_nginx_values.yaml --set-file nginx.http.server.ssl_cert_file=$PWD/nginx.crt --set-file nginx.http.server.ssl_key_file=$PWD/nginx.key openness_cdn ./helm/ +helm install -f ./examples/sriov_nginx_values.yaml --set-file nginx.http.server.ssl_cert_file=$PWD/nginx.crt --set-file nginx.http.server.ssl_key_file=$PWD/nginx.key openness-cdn ./helm/ ``` ### Testing using wrk "wrk" tool can be used for generating multiple http requests. Hence it will be used to test the CDN. @@ -128,7 +128,7 @@ Transfer/sec: 1.09GB > **Note** The wrk tool can be installed in a CentOS traffic host machine using the following steps, ``` yum groupinstall 'Development Tools' - yum install -y openssl-devel git + yum install -y openssl-devel git git clone https://github.com/wg/wrk.git wrk cd wrk make diff --git a/applications/openvino/consumer/cmd/openvino.go b/applications/openvino/consumer/cmd/openvino.go index 8c71296d..21ef5383 100644 --- a/applications/openvino/consumer/cmd/openvino.go +++ b/applications/openvino/consumer/cmd/openvino.go @@ -20,7 +20,6 @@ const ( var cmd *exec.Cmd func callOpenVINO(model string, accl string) { - var err error // validate accelerator type switch accl { @@ -34,7 +33,7 @@ func callOpenVINO(model string, accl string) { // kill already running process if not the first time if cmd != nil { - err = cmd.Process.Kill() + err := cmd.Process.Kill() if err != nil { log.Fatal("Failed to kill OpenVINO process:", err) } @@ -62,18 +61,18 @@ func callOpenVINO(model string, accl string) { go func() { stdout, _ := cmd.StdoutPipe() - if _, err = io.Copy(os.Stdout, stdout); err != nil { + if _, err := io.Copy(os.Stdout, stdout); err != nil { log.Println(err) } }() go func() { stderr, _ := cmd.StderrPipe() - if _, err = io.Copy(os.Stderr, stderr); err != nil { + if _, err := io.Copy(os.Stderr, stderr); err != nil { log.Println(err) } }() - err = cmd.Start() + err := cmd.Start() if err != nil { log.Fatal("Failed to run OpenVINO process:", err) } diff --git a/applications/smart-city-app/README.md b/applications/smart-city-app/README.md index 5968ca4d..d9238457 100644 --- a/applications/smart-city-app/README.md +++ b/applications/smart-city-app/README.md @@ -60,7 +60,7 @@ This mode provide an easy and quick start with executing the application in the 4. Install the application using the Helm chart ```shell - helm install smart-city-app smart-city-app app/deployment/kubernetes/helm/smtc + helm install smart-city-app app/deployment/kubernetes/helm/smtc ``` 5. From a web browser, launch the Smart City web UI at URL `https:///` @@ -110,7 +110,7 @@ Visual Cloud Accelerator Card - Analytics (VCAC-A) is a PCIe add on card compris 7. Install the application using the Helm chart ```shell - helm install smart-city-app smart-city-app app/deployment/kubernetes/helm/smtc + helm install smart-city-app app/deployment/kubernetes/helm/smtc ``` 8. From a web browser, launch the Smart City web UI at URL `https:///` From 52ec06aa84adc7d30d54651aaee7d6f091c4410b Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Fri, 24 Jul 2020 10:36:45 +0800 Subject: [PATCH 15/80] clean up format issue --- applications/vas-sample-app/cmd/main.go | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index 05b1a4fc..da0c3d75 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -170,29 +170,29 @@ func main() { return } - sum := 0 - var vasInfo VasConfig - for _, s := range servList.Services { - sum ++ - if sum > 100 { - log.Fatal("abnormal services num") - return - } - log.Println("Discoverd serive:") - log.Println(" URN.ID: ", s.URN.ID) - log.Println(" URN.Namespace:", s.URN.Namespace) - log.Println(" Description: ", s.Description) - log.Println(" EndpointURI: ", s.EndpointURI) + sum := 0 + var vasInfo VasConfig + for _, s := range servList.Services { + sum ++ + if sum > 100 { + log.Fatal("abnormal services num") + return + } + log.Println("Discoverd serive:") + log.Println(" URN.ID: ", s.URN.ID) + log.Println(" URN.Namespace:", s.URN.Namespace) + log.Println(" Description: ", s.Description) + log.Println(" EndpointURI: ", s.EndpointURI) // Subscribe to all services related to my Namespace if myURN.Namespace == s.URN.Namespace { // Service Request to VA-Serving - err := json.Unmarshal(s.Info, &vasInfo) - if err != nil { - log.Println(err) - } - postVAServingRequest(s.EndpointURI, vasInfo.Pipelines[0]) - } else { - log.Println("Namespace mismatch, myURN namespace:",myURN.Namespace) - } - } + err := json.Unmarshal(s.Info, &vasInfo) + if err != nil { + log.Println(err) + } + postVAServingRequest(s.EndpointURI, vasInfo.Pipelines[0]) + } else { + log.Println("Namespace mismatch, myURN namespace:",myURN.Namespace) + } + } } From 938754a745df183408970a3f012f9d8afb75ab2d Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Mon, 27 Jul 2020 13:23:31 +0800 Subject: [PATCH 16/80] Merge blocked by Checkmarx Race Condition In Cross Functionality for openvino consumer go file - so merge from branch fix-editorials-openvino-build --- applications/openvino/consumer/cmd/openvino.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/applications/openvino/consumer/cmd/openvino.go b/applications/openvino/consumer/cmd/openvino.go index 8c71296d..21ef5383 100644 --- a/applications/openvino/consumer/cmd/openvino.go +++ b/applications/openvino/consumer/cmd/openvino.go @@ -20,7 +20,6 @@ const ( var cmd *exec.Cmd func callOpenVINO(model string, accl string) { - var err error // validate accelerator type switch accl { @@ -34,7 +33,7 @@ func callOpenVINO(model string, accl string) { // kill already running process if not the first time if cmd != nil { - err = cmd.Process.Kill() + err := cmd.Process.Kill() if err != nil { log.Fatal("Failed to kill OpenVINO process:", err) } @@ -62,18 +61,18 @@ func callOpenVINO(model string, accl string) { go func() { stdout, _ := cmd.StdoutPipe() - if _, err = io.Copy(os.Stdout, stdout); err != nil { + if _, err := io.Copy(os.Stdout, stdout); err != nil { log.Println(err) } }() go func() { stderr, _ := cmd.StderrPipe() - if _, err = io.Copy(os.Stderr, stderr); err != nil { + if _, err := io.Copy(os.Stderr, stderr); err != nil { log.Println(err) } }() - err = cmd.Start() + err := cmd.Start() if err != nil { log.Fatal("Failed to run OpenVINO process:", err) } From cefe1a5ddc564d25989ab466c59bb5f695c848ce Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Wed, 29 Jul 2020 14:02:29 +0800 Subject: [PATCH 17/80] fix copyright issue --- applications/vas-sample-app/build/build-image.sh | 2 +- applications/vas-sample-app/cmd/application.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/applications/vas-sample-app/build/build-image.sh b/applications/vas-sample-app/build/build-image.sh index b9de9955..c7d97c25 100755 --- a/applications/vas-sample-app/build/build-image.sh +++ b/applications/vas-sample-app/build/build-image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# PDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation sudo docker build -f ./build/Dockerfile \ diff --git a/applications/vas-sample-app/cmd/application.go b/applications/vas-sample-app/cmd/application.go index 792ea697..4ec16d15 100644 --- a/applications/vas-sample-app/cmd/application.go +++ b/applications/vas-sample-app/cmd/application.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2020 Intel Corporation + + package main import ( From 63754884f27f9b507803661a75db5d9ee4489f3d Mon Sep 17 00:00:00 2001 From: "xiaopeng,tong" Date: Wed, 29 Jul 2020 14:53:40 +0800 Subject: [PATCH 18/80] fix gofmt issue --- .../vas-sample-app/cmd/application.go | 108 +++++++++--------- .../vas-sample-app/cmd/eaa_interface.go | 2 +- applications/vas-sample-app/cmd/main.go | 16 ++- 3 files changed, 61 insertions(+), 65 deletions(-) diff --git a/applications/vas-sample-app/cmd/application.go b/applications/vas-sample-app/cmd/application.go index 4ec16d15..21e5e93f 100644 --- a/applications/vas-sample-app/cmd/application.go +++ b/applications/vas-sample-app/cmd/application.go @@ -1,32 +1,30 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (c) 2020 Intel Corporation - package main import ( - "github.com/levigross/grequests" - "encoding/json" - "log" - "strings" - "time" + "encoding/json" + "github.com/levigross/grequests" + "log" + "strings" + "time" ) type Source struct { - URI string `json:"uri"` - Type string `json:"type"` + URI string `json:"uri"` + Type string `json:"type"` } type Destination struct { - Path string `json:"path"` - Type string `json:"type"` - Format string `json:"format"` + Path string `json:"path"` + Type string `json:"type"` + Format string `json:"format"` } - type VASPostBody struct { - SRCE Source `json:"source"` - DEST Destination `json:"destination"` + SRCE Source `json:"source"` + DEST Destination `json:"destination"` } type VASInstanceStatus struct { @@ -39,48 +37,48 @@ type VASInstanceStatus struct { func postVAServingRequest(vaEndPoint string, vaPipeline string) error { - // Interface: POST /pipelines/{name}/{version} for Start new pipeline instance - var endPointStr string - endPointStr = vaEndPoint+"/pipelines/"+vaPipeline - log.Println("Starting POST Request:"+endPointStr) - var src Source - src.URI = "https://github.com/intel-iot-devkit/sample-videos/blob/master/bottle-detection.mp4?raw=true" - src.Type = "uri" - var dst Destination - dst.Path = "/tmp/results.txt" - dst.Type = "file" - dst.Format = "json-lines" - var vasPost = VASPostBody{SRCE: src, DEST: dst} - var vasPostJson, _ = json.Marshal(vasPost) - var vasRequestOption = &grequests.RequestOptions{} - vasRequestOption.JSON = string(vasPostJson) - //resp, err = grequests.Post("http://localhost:8080/pipelines/object_detection/1",vasRequestOption) - resp, err := grequests.Post(endPointStr,vasRequestOption) - if err != nil { - log.Fatalln ("Post Failed with: ", err) - return err - } - log.Println("Pipeline Instance Created:", resp.String()) - var instance = strings.TrimSpace(resp.String()) + // Interface: POST /pipelines/{name}/{version} for Start new pipeline instance + var endPointStr string + endPointStr = vaEndPoint + "/pipelines/" + vaPipeline + log.Println("Starting POST Request:" + endPointStr) + var src Source + src.URI = "https://github.com/intel-iot-devkit/sample-videos/blob/master/bottle-detection.mp4?raw=true" + src.Type = "uri" + var dst Destination + dst.Path = "/tmp/results.txt" + dst.Type = "file" + dst.Format = "json-lines" + var vasPost = VASPostBody{SRCE: src, DEST: dst} + var vasPostJson, _ = json.Marshal(vasPost) + var vasRequestOption = &grequests.RequestOptions{} + vasRequestOption.JSON = string(vasPostJson) + //resp, err = grequests.Post("http://localhost:8080/pipelines/object_detection/1",vasRequestOption) + resp, err := grequests.Post(endPointStr, vasRequestOption) + if err != nil { + log.Fatalln("Post Failed with: ", err) + return err + } + log.Println("Pipeline Instance Created:", resp.String()) + var instance = strings.TrimSpace(resp.String()) - // Interface: GET /pipelines/{name}/{version}/{instance_id}/status for Return pipeline instance status. - // Loop status query until instantance completed. - var statusResp VASInstanceStatus - for { - //resp, err := grequests.Get("http://localhost:8080/pipelines/object_detection/1/"+instance+"/status", nil) - resp, err := grequests.Get(endPointStr+"/"+instance+"/status", nil) - if err != nil { - log.Fatalln ("Get Failed with: ", err) - return err - } - log.Println(resp.String()) - resp.JSON(&statusResp) - if statusResp.State == "COMPLETED" { - break - } - time.Sleep(time.Duration(10)*time.Second) + // Interface: GET /pipelines/{name}/{version}/{instance_id}/status for Return pipeline instance status. + // Loop status query until instantance completed. + var statusResp VASInstanceStatus + for { + //resp, err := grequests.Get("http://localhost:8080/pipelines/object_detection/1/"+instance+"/status", nil) + resp, err := grequests.Get(endPointStr+"/"+instance+"/status", nil) + if err != nil { + log.Fatalln("Get Failed with: ", err) + return err + } + log.Println(resp.String()) + resp.JSON(&statusResp) + if statusResp.State == "COMPLETED" { + break + } + time.Sleep(time.Duration(10) * time.Second) - } - return nil + } + return nil } diff --git a/applications/vas-sample-app/cmd/eaa_interface.go b/applications/vas-sample-app/cmd/eaa_interface.go index 9a424e66..db5df287 100644 --- a/applications/vas-sample-app/cmd/eaa_interface.go +++ b/applications/vas-sample-app/cmd/eaa_interface.go @@ -40,7 +40,7 @@ type Service struct { EndpointURI string `json:"endpoint_uri,omitempty"` Status string `json:"status,omitempty"` Notifications []NotificationDescriptor `json:"notifications,omitempty"` - Info json.RawMessage `json:"info,omitempty"` + Info json.RawMessage `json:"info,omitempty"` } // SubscriptionList JSON struct diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index da0c3d75..a77c9dff 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -15,7 +15,6 @@ import ( "encoding/pem" "log" "net/http" - ) // Connectivity constants @@ -28,7 +27,6 @@ const ( var myURN URN - // VasConfig describes VAS JSON config file type VasConfig struct { Acceleration string `json:"Acceleration"` @@ -171,9 +169,9 @@ func main() { } sum := 0 - var vasInfo VasConfig + var vasInfo VasConfig for _, s := range servList.Services { - sum ++ + sum++ if sum > 100 { log.Fatal("abnormal services num") return @@ -185,14 +183,14 @@ func main() { log.Println(" EndpointURI: ", s.EndpointURI) // Subscribe to all services related to my Namespace if myURN.Namespace == s.URN.Namespace { - // Service Request to VA-Serving + // Service Request to VA-Serving err := json.Unmarshal(s.Info, &vasInfo) if err != nil { log.Println(err) - } - postVAServingRequest(s.EndpointURI, vasInfo.Pipelines[0]) - } else { - log.Println("Namespace mismatch, myURN namespace:",myURN.Namespace) + } + postVAServingRequest(s.EndpointURI, vasInfo.Pipelines[0]) + } else { + log.Println("Namespace mismatch, myURN namespace:", myURN.Namespace) } } } From 97de698b7ba03016eb29995ee6fdb7b5084ebe54 Mon Sep 17 00:00:00 2001 From: relhwigi Date: Wed, 29 Jul 2020 15:17:15 +0100 Subject: [PATCH 19/80] OP-4034 and OP-4062 pylint fixes --- network-functions/xran/test_verification.py | 434 +++++++++++--------- 1 file changed, 231 insertions(+), 203 deletions(-) diff --git a/network-functions/xran/test_verification.py b/network-functions/xran/test_verification.py index d4cb8764..68d72209 100755 --- a/network-functions/xran/test_verification.py +++ b/network-functions/xran/test_verification.py @@ -2,52 +2,48 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -"""This script runs test cases with O-DU and O-RU -""" +"""This script runs test cases with O-DU and O-RU""" import logging import sys import argparse -import re -import subprocess import os -import shutil from itertools import dropwhile from datetime import datetime -from time import gmtime, strftime import json -from threading import Timer import socket -nLteNumRbsPerSymF1 = [ +N_LTE_NUM_RBS_PER_SYM_F1 = [ # 5MHz 10MHz 15MHz 20 MHz - [25, 50, 75, 100] # LTE Numerology 0 (15KHz) + [25, 50, 75, 100] # LTE Numerology 0 (15KHz) ] -nNumRbsPerSymF1 = [ - # 5MHz 10MHz 15MHz 20 MHz 25 MHz 30 MHz 40 MHz 50MHz 60 MHz 70 MHz 80 MHz 90 MHz 100 MHz - [25, 52, 79, 106, 133, 160, 216, 270, 0, 0, 0, 0, 0], # Numerology 0 (15KHz) - [11, 24, 38, 51, 65, 78, 106, 133, 162, 0, 217, 245, 273], # Numerology 1 (30KHz) - [0, 11, 18, 24, 31, 38, 51, 65, 79, 0, 107, 121, 135] # Numerology 2 (60KHz) +N_NUM_RBS_PER_SYM_F1 = [ + # 5MHz 10MHz 15MHz 20 MHz 25 MHz 30 MHz 40 MHz 50MHz 60 MHz 70 MHz 80 MHz + # 90 MHz 100 MHz + [25, 52, 79, 106, 133, 160, 216, 270, 0, 0, 0, 0, 0], # Numerology 0 (15KHz) + [11, 24, 38, 51, 65, 78, 106, 133, 162, 0, 217, 245, 273], # Numerology 1 (30KHz) + [0, 11, 18, 24, 31, 38, 51, 65, 79, 0, 107, 121, 135] # Numerology 2 (60KHz) ] -nNumRbsPerSymF2 = [ +N_NUM_RBS_PER_SYM_F2 = [ # 50Mhz 100MHz 200MHz 400MHz - [66, 132, 264, 0], # Numerology 2 (60KHz) - [32, 66, 132, 264] # Numerology 3 (120KHz) + [66, 132, 264, 0], # Numerology 2 (60KHz) + [32, 66, 132, 264] # Numerology 3 (120KHz) ] -nRChBwOptions_keys = ['5','10','15','20', '25', '30', '40', '50', '60','70', '80', '90', '100', '200', '400'] -nRChBwOptions_values = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14] -nRChBwOptions = dict(zip(nRChBwOptions_keys, nRChBwOptions_values)) +N_RCH_BW_OPTIONS_KEYS = ['5', '10', '15', '20', '25', '30', '40', '50', '60', '70', '80', '90', + '100', '200', '400'] +N_RCH_BW_OPTIONS_VALUES = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] +N_RCH_BW_OPTIONS = dict(zip(N_RCH_BW_OPTIONS_KEYS, N_RCH_BW_OPTIONS_VALUES)) -nRChBwOptions_keys_mu2and3 = ['50', '100', '200', '400'] -nRChBwOptions_values_mu2and3 = [0,1,2,3] -nRChBwOptions_mu2and3 = dict(zip(nRChBwOptions_keys_mu2and3, nRChBwOptions_values_mu2and3)) +N_RCH_BW_OPTIONS_KEYS_MU2AND3 = ['50', '100', '200', '400'] +N_RCH_BW_OPTIONS_VALUES_MU2AND3 = [0, 1, 2, 3] +N_RCH_BW_OPTIONS_MU2AND3 = dict(zip(N_RCH_BW_OPTIONS_KEYS_MU2AND3, N_RCH_BW_OPTIONS_VALUES_MU2AND3)) -dic_dir = dict({0:'DL', 1:'UL'}) -dic_xu = dict({0:'o-du', 1:'o-ru'}) -dic_ran_tech = dict({0:'5g_nr', 1:'lte'}) +DIC_DIR = dict({0:'DL', 1:'UL'}) +DIC_XU = dict({0:'o-du', 1:'o-ru'}) +DIC_RAN_TECH = dict({0:'5g_nr', 1:'lte'}) def init_logger(console_level, logfile_level): """Initializes console and logfile logger with given logging levels""" @@ -67,108 +63,121 @@ def init_logger(console_level, logfile_level): def parse_args(args): """Configures parser and parses command line configuration""" # Parser configuration - parser = argparse.ArgumentParser(description="Run test cases: category numerology bandwidth test_num") - - parser.add_argument("--ran", type=int, default=0, help="Radio Access Tehcnology 0 (5G NR) or 1 (LTE)", metavar="ran", dest="rantech") - parser.add_argument("--cat", type=int, default=0, help="Category: 0 (A) or 1 (B)", metavar="cat", dest="category") - parser.add_argument("--mu", type=int, default=0, help="numerology [0,1,3]", metavar="num", dest="numerology") - parser.add_argument("--bw", type=int, default=20, help="bandwidth [5,10,20,100]", metavar="bw", dest="bandwidth") - parser.add_argument("--testcase", type=int, default=0, help="test case number", metavar="testcase", dest="testcase") - parser.add_argument("--verbose", type=int, default=0, help="enable verbose output", metavar="verbose", dest="verbose") + parser = argparse.ArgumentParser(description= + "Run test cases: category numerology bandwidth test_num") + + parser.add_argument("--ran", type=int, default=0, help= + "Radio Access Tehcnology 0 (5G NR) or 1 (LTE)", + metavar="ran", dest="rantech") + parser.add_argument("--cat", type=int, default=0, help="Category: 0 (A) or 1 (B)", + metavar="cat", dest="category") + parser.add_argument("--m_u", type=int, default=0, help="numerology [0,1,3]", + metavar="num", dest="numerology") + parser.add_argument("--b_w", type=int, default=20, help="bandwidth [5,10,20,100]", + metavar="b_w", dest="bandwidth") + parser.add_argument("--testcase", type=int, default=0, help="test case number", + metavar="testcase", dest="testcase") + parser.add_argument("--verbose", type=int, default=0, help="enable verbose output", + metavar="verbose", dest="verbose") # Parse arguments options = parser.parse_args(args) #parser.print_help() logging.debug("Options: ran=%d category=%d num=%d bw=%d testcase=%d", - options.rantech, options.category, options.numerology, options.bandwidth, options.testcase) + options.rantech, options.category, options.numerology, options.bandwidth, + options.testcase) return options -def is_comment(s): +def is_comment(line): """ function to check if a line starts with some character. Here # for comment """ # return true if a line starts with # - return s.startswith('#') + return line.startswith('#') -class GetOutOfLoops( Exception ): - pass +class GetOutOfLoops(Exception): + """ get out of loops exception """ -def get_re_map(nRB, direction): - prb_map = [] - PrbElemContent = [] +def get_re_map(nrb, direction): + """method to get re map""" + prb_map = [] + prb_elem_content = [] if direction == 0: #DL if 'nPrbElemDl' in globals(): - nPrbElm = nPrbElemDl - for i in range(0, nPrbElm): - elm = str('PrbElemDl'+str(i)) + n_prb_elm = 'nPrbElemDl' + for i in range(0, n_prb_elm): + elm = str('prb_elem_dl'+str(i)) #print(elm) if elm in globals(): - PrbElemContent.insert(i,list(globals()[elm])) - xRBStart = PrbElemContent[i][0] - xRBSize = PrbElemContent[i][1] - #print(PrbElemContent,"RBStart: ", xRBStart, "RBSize: ",xRBSize, list(range(xRBStart, xRBStart + xRBSize))) - prb_map = prb_map + list(range(xRBStart*12, xRBStart*12 + xRBSize*12)) + prb_elem_content.insert(i, list(globals()[elm])) + xrbstart = prb_elem_content[i][0] + xrbsize = prb_elem_content[i][1] + #print(PrbElemContent,"RBStart: ", xRBStart, "RBSize: ",xRBSize, + #list(range(xRBStart, xRBStart + xRBSize))) + prb_map = prb_map + list(range(xrbstart*12, xrbstart*12 + xrbsize*12)) else: - nPrbElm = 0; + n_prb_elm = 0 elif direction == 1: #UL if 'nPrbElemUl' in globals(): - nPrbElm = nPrbElemUl - for i in range(0, nPrbElm): - elm = str('PrbElemUl'+str(i)) + n_prb_elm = 'nPrbElemUl' + for i in range(0, n_prb_elm): + elm = str('prb_elem_ul'+str(i)) #print(elm) - if (elm in globals()): - PrbElemContent.insert(i,list(globals()[elm])) - xRBStart = PrbElemContent[i][0] - xRBSize = PrbElemContent[i][1] - #print(PrbElemContent,"RBStart: ", xRBStart, "RBSize: ",xRBSize, list(range(xRBStart, xRBStart + xRBSize))) - prb_map = prb_map + list(range(xRBStart*12, xRBStart*12 + xRBSize*12)) + if elm in globals(): + prb_elem_content.insert(i, list(globals()[elm])) + xrbstart = prb_elem_content[i][0] + xrbsize = prb_elem_content[i][1] + #print(PrbElemContent,"RBStart: ", xRBStart, "RBSize: ",xRBSize, + #list(range(xRBStart, xRBStart + xRBSize))) + prb_map = prb_map + list(range(xrbstart*12, xrbstart*12 + xrbsize*12)) else: - nPrbElm = 0; + n_prb_elm = 0 - if nPrbElm == 0 : - prb_map = list(range(0, nRB*12)) + if n_prb_elm == 0: + prb_map = list(range(0, nrb*12)) return prb_map -def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): +def compare_resuts(rantech, cat, m_u, xran_path, direction): + """method to compare results""" res = 0 re_map = [] - if rantech==1: - if mu == 0: - nDlRB = nLteNumRbsPerSymF1[mu][nRChBwOptions.get(str(nDLBandwidth))] - nUlRB = nLteNumRbsPerSymF1[mu][nRChBwOptions.get(str(nULBandwidth))] + if rantech == 1: + if m_u == 0: + n_dirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nDLBandwidth))] + n_uirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nULBandwidth))] else: print("Incorrect arguments\n") res = -1 return res - elif rantech==0: - if mu < 3: - nDlRB = nNumRbsPerSymF1[mu][nRChBwOptions.get(str(nDLBandwidth))] - nUlRB = nNumRbsPerSymF1[mu][nRChBwOptions.get(str(nULBandwidth))] - elif (mu >=2) & (mu <= 3): - nDlRB = nNumRbsPerSymF2[mu - 2][nRChBwOptions_mu2and3.get(str(nDLBandwidth))] - nUlRB = nNumRbsPerSymF2[mu - 2][nRChBwOptions_mu2and3.get(str(nULBandwidth))] - print(nDlRB, nUlRB) + elif rantech == 0: + if m_u < 3: + n_dirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nDLBandwidth))] + n_uirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nULBandwidth))] + elif (m_u >= 2) & (m_u <= 3): + n_dirb = N_NUM_RBS_PER_SYM_F2[m_u - 2][N_RCH_BW_OPTIONS_MU2AND3.get(str(nDLBandwidth))] + n_uirb = N_NUM_RBS_PER_SYM_F2[m_u - 2][N_RCH_BW_OPTIONS_MU2AND3.get(str(nULBandwidth))] + print(n_dirb, n_uirb) else: print("Incorrect arguments\n") res = -1 return res if 'compression' in globals(): - comp = compression + comp = 'compression' else: comp = 0 if 'srsEanble' in globals(): - srs_enb = srsEanble + srs_enb = 'srsEanble' else: srs_enb = 0 - print("compare results: {} [compression {}]\n".format(dic_dir.get(direction), comp)) + print("compare results: {} [compression {}]\n".format(DIC_DIR.get(direction), comp)) #if cat == 1: # print("WARNING: Skip checking IQs and BF Weights for CAT B for now\n"); @@ -176,52 +185,53 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): #get slot config if nFrameDuplexType == 1: - SlotConfig = [] + slot_config = [] for i in range(nTddPeriod): if i == 0: - SlotConfig.insert(i, sSlotConfig0) + slot_config.insert(i, sslot_config0) elif i == 1: - SlotConfig.insert(i, sSlotConfig1) + slot_config.insert(i, sslot_config1) elif i == 2: - SlotConfig.insert(i, sSlotConfig2) + slot_config.insert(i, sslot_config2) elif i == 3: - SlotConfig.insert(i, sSlotConfig3) + slot_config.insert(i, sslot_config3) elif i == 4: - SlotConfig.insert(i, sSlotConfig4) + slot_config.insert(i, sslot_config4) elif i == 5: - SlotConfig.insert(i, sSlotConfig5) + slot_config.insert(i, sslot_config5) elif i == 6: - SlotConfig.insert(i, sSlotConfig6) + slot_config.insert(i, sslot_config6) elif i == 7: - SlotConfig.insert(i, sSlotConfig7) + slot_config.insert(i, sslot_config7) elif i == 8: - SlotConfig.insert(i, sSlotConfig8) + slot_config.insert(i, sslot_config8) elif i == 9: - SlotConfig.insert(i, sSlotConfig9) - else : - raise Exception('i should not exceed nTddPeriod %d. The value of i was: {}'.format(nTddPeriod, i)) + slot_config.insert(i, sslot_config9) + else: + raise Exception('i should not exceed nTddPeriod %d. The value of i was: {}' + .format(nTddPeriod, i)) #print(SlotConfig, type(sSlotConfig0)) try: if (direction == 1) & (cat == 1): #UL - flowId = ccNum*antNumUL + flow_id = ccNum*antNumUL else: - flowId = ccNum*antNum + flow_id = ccNum*antNum if direction == 0: - re_map = get_re_map(nDlRB, direction) + re_map = get_re_map(n_dirb, direction) elif direction == 1: - re_map = get_re_map(nUlRB, direction) + re_map = get_re_map(n_uirb, direction) else: raise Exception('Direction is not supported %d'.format(direction)) - for i in range(0, flowId): + for i in range(0, flow_id): #read ref and test files tst = [] ref = [] if direction == 0: # DL - nRB = nDlRB + nrb = n_dirb file_tst = xran_path+"/results/"+"o-ru-rx_log_ant"+str(i)+".txt" file_ref = xran_path+"/results/"+"o-du-play_ant"+str(i)+".txt" # file_tst = xran_path+"/app/logs/"+"o-ru-rx_log_ant"+str(i)+".txt" @@ -229,7 +239,7 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): elif direction == 1: # UL - nRB = nUlRB + nrb = n_uirb file_tst = xran_path+"/results/"+"o-du-rx_log_ant"+str(i)+".txt" file_ref = xran_path+"/results/"+"o-ru-play_ant"+str(i)+".txt" # file_tst = xran_path+"/app/logs/"+"o-du-rx_log_ant"+str(i)+".txt" @@ -243,7 +253,7 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): try: file_tst = open(file_tst, 'r') except OSError: - print ("Could not open/read file:", file_tst) + print("Could not open/read file:", file_tst) sys.exit() else: print(file_tst, "doesn't exist") @@ -253,7 +263,7 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): try: file_ref = open(file_ref, 'r') except OSError: - print ("Could not open/read file:", file_ref) + print("Could not open/read file:", file_ref) sys.exit() else: print(file_tst, "doesn't exist") @@ -266,40 +276,42 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): print(len(tst)) print(len(ref)) - file_tst.close(); - file_ref.close(); + file_tst.close() + file_ref.close() print(numSlots) for slot_idx in range(0, numSlots): for sym_idx in range(0, 14): - if nFrameDuplexType==1: + if nFrameDuplexType == 1: #skip sym if TDD if direction == 0: #DL - sym_dir = SlotConfig[slot_idx%nTddPeriod][sym_idx] - if(sym_dir != 0): + sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + if sym_dir != 0: continue elif direction == 1: #UL - sym_dir = SlotConfig[slot_idx%nTddPeriod][sym_idx] - if(sym_dir != 1): + sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + if sym_dir != 1: continue #print("Check:","[",i,"]", slot_idx, sym_idx) for line_idx in re_map: - offset = (slot_idx*nRB*12*14) + sym_idx*nRB*12 + line_idx + offset = (slot_idx*nrb*12*14) + sym_idx*nrb*12 + line_idx try: line_tst = tst[offset].rstrip() except IndexError: res = -1 - print("FAIL:","IndexError on tst: ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx, len(tst)) + print("FAIL:", "IndexError on tst: ant:[", i, "]:", + offset, slot_idx, sym_idx, line_idx, len(tst)) raise GetOutOfLoops try: - line_ref = ref[offset].rstrip() + line_ref = ref[offset].rstrip() except IndexError: res = -1 - print("FAIL:","IndexError on ref: ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx, len(ref)) + print("FAIL:", "IndexError on ref: ant:[", i, "]:", + offset, slot_idx, sym_idx, line_idx, len(ref)) raise GetOutOfLoops if comp == 1: @@ -309,16 +321,22 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): ref_i_value = int(line_ref.split(" ")[0]) & 0xFF80 ref_q_value = int(line_ref.split(" ")[1]) & 0xFF80 - #print("check:","ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx,":","tst: ", tst_i_value, " ", tst_q_value, " " , "ref: ", ref_i_value, " ", ref_q_value, " ") - if (tst_i_value != ref_i_value) or (tst_q_value != ref_q_value) : - print("FAIL:","ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx,":","tst: ", tst_i_value, " ", tst_q_value, " " , "ref: ", ref_i_value, " ", ref_q_value, " ") + #print("check:","ant:[",i,"]:",offset, slot_idx, sym_idx, + #line_idx,":","tst: ",tst_i_value, " ", tst_q_value, " " , + # "ref: ", ref_i_value, " ", ref_q_value, " ") + if (tst_i_value != ref_i_value) or (tst_q_value != ref_q_value): + print("FAIL:", "ant:[", i, "]:", offset, slot_idx, sym_idx, + line_idx, ":", "tst: ", tst_i_value, " ", tst_q_value, " ", + "ref: ", ref_i_value, " ", ref_q_value, " ") res = -1 raise GetOutOfLoops else: #if line_idx == 0: - #print("Check:", offset,"[",i,"]", slot_idx, sym_idx,":",line_tst, line_ref) + #print("Check:", offset,"[",i,"]", slot_idx, sym_idx,":", + #line_tst, line_ref) if line_ref != line_tst: - print("FAIL:","ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx,":","tst:", line_tst, "ref:", line_ref) + print("FAIL:", "ant:[", i, "]:", offset, slot_idx, sym_idx, + line_idx, ":", "tst:", line_tst, "ref:", line_ref) res = -1 raise GetOutOfLoops except GetOutOfLoops: @@ -326,22 +344,22 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): #if (direction == 0) | (cat == 0) | (srs_enb == 0): #DL or Cat A #done - return res + #return res print("compare results: {} [compression {}]\n".format('SRS', comp)) #srs - symbMask = srsSym + symb_mask = srsSym try: - flowId = ccNum*antElmTRx - for i in range(0, flowId): + flow_id = ccNum*antElmTRx + for i in range(0, flow_id): #read ref and test files tst = [] ref = [] if direction == 1: # UL - nRB = nUlRB + nrb = n_uirb file_tst = xran_path+"/results/"+"o-du-srs_log_ant"+str(i)+".txt" file_ref = xran_path+"/results/logs/"+"o-ru-play_srs_ant"+str(i)+".txt" @@ -356,7 +374,7 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): try: file_tst = open(file_tst, 'r') except OSError: - print ("Could not open/read file:", file_tst) + print("Could not open/read file:", file_tst) sys.exit() else: print(file_tst, "doesn't exist") @@ -366,7 +384,7 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): try: file_ref = open(file_ref, 'r') except OSError: - print ("Could not open/read file:", file_ref) + print("Could not open/read file:", file_ref) sys.exit() else: print(file_tst, "doesn't exist") @@ -379,44 +397,46 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): print(len(tst)) print(len(ref)) - file_tst.close(); - file_ref.close(); + file_tst.close() + file_ref.close() print(numSlots) for slot_idx in range(0, numSlots): for sym_idx in range(0, 14): - if symbMask & (1 << sym_idx): + if symb_mask & (1 << sym_idx): print("SRS check sym ", sym_idx) - if nFrameDuplexType==1: + if nFrameDuplexType == 1: #skip sym if TDD if direction == 0: #DL - sym_dir = SlotConfig[slot_idx%nTddPeriod][sym_idx] - if(sym_dir != 0): + sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + if sym_dir != 0: continue elif direction == 1: #UL - sym_dir = SlotConfig[slot_idx%nTddPeriod][sym_idx] - if(sym_dir != 1): + sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + if sym_dir != 1: continue #print("Check:","[",i,"]", slot_idx, sym_idx) - for line_idx in range(0, nRB*12): - offset = (slot_idx*nRB*12*14) + sym_idx*nRB*12 + line_idx + for line_idx in range(0, nrb*12): + offset = (slot_idx*nrb*12*14) + sym_idx*nrb*12 + line_idx try: line_tst = tst[offset].rstrip() except IndexError: res = -1 - print("FAIL:","IndexError on tst: ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx, len(tst)) + print("FAIL:", "IndexError on tst: ant:[", i, "]:", + offset, slot_idx, sym_idx, line_idx, len(tst)) raise GetOutOfLoops try: line_ref = ref[offset].rstrip() except IndexError: res = -1 - print("FAIL:","IndexError on ref: ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx, len(ref)) + print("FAIL:", "IndexError on ref: ant:[", i, "]:", + offset, slot_idx, sym_idx, line_idx, len(ref)) raise GetOutOfLoops - if False : #SRS sent as not compressed + if False: #SRS sent as not compressed #comp == 1: # discard LSB bits as BFP compression is not Bit Exact tst_i_value = int(line_tst.split(" ")[0]) & 0xFF80 @@ -424,16 +444,22 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): ref_i_value = int(line_ref.split(" ")[0]) & 0xFF80 ref_q_value = int(line_ref.split(" ")[1]) & 0xFF80 - print("check:","ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx,":","tst: ", tst_i_value, " ", tst_q_value, " " , "ref: ", ref_i_value, " ", ref_q_value, " ") - if (tst_i_value != ref_i_value) or (tst_q_value != ref_q_value) : - print("FAIL:","ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx,":","tst: ", tst_i_value, " ", tst_q_value, " " , "ref: ", ref_i_value, " ", ref_q_value, " ") + print("check:", "ant:[", i, "]:", offset, slot_idx, sym_idx, + line_idx, ":", "tst: ", tst_i_value, " ", tst_q_value, " ", + "ref: ", ref_i_value, " ", ref_q_value, " ") + if (tst_i_value != ref_i_value) or (tst_q_value != ref_q_value): + print("FAIL:", "ant:[", i, "]:", offset, slot_idx, sym_idx, + line_idx, ":", "tst: ", tst_i_value, " ", tst_q_value, + " ", "ref: ", ref_i_value, " ", ref_q_value, " ") res = -1 raise GetOutOfLoops else: #if line_idx == 0: - #print("Check:", offset,"[",i,"]", slot_idx, sym_idx,":",line_tst, line_ref) + #print("Check:", offset,"[",i,"]", slot_idx, sym_idx,":" + #, line_tst, line_ref) if line_ref != line_tst: - print("FAIL:","ant:[",i,"]:",offset, slot_idx, sym_idx, line_idx,":","tst:", line_tst, "ref:", line_ref) + print("FAIL:", "ant:[", i, "]:", offset, slot_idx, sym_idx, + line_idx, ":", "tst:", line_tst, "ref:", line_ref) res = -1 raise GetOutOfLoops except GetOutOfLoops: @@ -442,57 +468,57 @@ def compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, direction): return res -def parse_dat_file(rantech, cat, mu, bw, tcase, xran_path, test_cfg): - #parse config files +def parse_dat_file(test_cfg): + """parse config files""" logging.info("parse config files %s\n", test_cfg[0]) - lineList = list() + line_list = list() sep = '#' - with open(test_cfg[0],'r') as fh: - for curline in dropwhile(is_comment, fh): + with open(test_cfg[0], 'r') as f_h: + for curline in dropwhile(is_comment, f_h): my_line = curline.rstrip().split(sep, 1)[0].strip() if my_line: - lineList.append(my_line) + line_list.append(my_line) global_env = {} local_env = {} - for line in lineList: + for line in line_list: exe_line = line.replace(":", ",") - if exe_line.find("/") > 0 : + if exe_line.find("/") > 0: exe_line = exe_line.replace('./', "'") exe_line = exe_line+"'" code = compile(str(exe_line), '', 'exec') - exec (code, global_env, local_env) + exec(code, global_env, local_env) - for k, v in local_env.items(): - globals()[k] = v - print(k, v) + for k, var in local_env.items(): + globals()[k] = var + print(k, var) return local_env def del_dat_file_vars(local_env): - - for k, v in local_env.items(): + """method for deleting variables in file""" + for k, in local_env.items(): del globals()[k] return 0 -def run_tcase(rantech, cat, mu, bw, tcase, verbose, xran_path): - +def run_tcase(rantech, cat, m_u, b_w, tcase, xran_path): + """ method for runing test cases""" if rantech == 1: #LTE if cat == 1: - test_config =xran_path+"/app/usecase/lte_b/mu{0:d}_{1:d}mhz".format(mu, bw) - elif cat == 0 : - test_config =xran_path+"/app/usecase/lte_a/mu{0:d}_{1:d}mhz".format(mu, bw) + test_config = xran_path+"/app/usecase/lte_b/m_u{0:d}_{1:d}mhz".format(m_u, b_w) + elif cat == 0: + test_config = xran_path+"/app/usecase/lte_a/m_u{0:d}_{1:d}mhz".format(m_u, b_w) else: print("Incorrect cat arguments\n") return -1 elif rantech == 0: #5G NR if cat == 1: - test_config =xran_path+"/app/usecase/cat_b/mu{0:d}_{1:d}mhz".format(mu, bw) - elif cat == 0 : - test_config =xran_path+"/app/usecase/mu{0:d}_{1:d}mhz".format(mu, bw) + test_config = xran_path+"/app/usecase/cat_b/m_u{0:d}_{1:d}mhz".format(m_u, b_w) + elif cat == 0: + test_config = xran_path+"/app/usecase/m_u{0:d}_{1:d}mhz".format(m_u, b_w) else: print("Incorrect cat argument\n") return -1 @@ -500,7 +526,7 @@ def run_tcase(rantech, cat, mu, bw, tcase, verbose, xran_path): print("Incorrect rantech argument\n") return -1 - if(tcase > 0) : + if tcase > 0: test_config = test_config+"/"+str(tcase) test_cfg = [] @@ -508,26 +534,26 @@ def run_tcase(rantech, cat, mu, bw, tcase, verbose, xran_path): test_cfg.append(test_config+"/config_file_o_du.dat") test_cfg.append(test_config+"/config_file_o_ru.dat") - wd = os.getcwd() + w_d = os.getcwd() os.chdir(xran_path+"/app/") # make_copy_mlog(rantech, cat, mu, bw, tcase, xran_path) - usecase_cfg = parse_dat_file(rantech, cat, mu, bw, tcase, xran_path, test_cfg) + usecase_cfg = parse_dat_file(test_cfg) - res = compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, 0) + res = compare_resuts(rantech, cat, m_u, xran_path, 0) if res != 0: - os.chdir(wd) + os.chdir(w_d) print("FAIL") return res - res = compare_resuts(rantech, cat, mu, bw, tcase, xran_path, test_cfg, 1) + res = compare_resuts(rantech, cat, m_u, xran_path, 1) if res != 0: - os.chdir(wd) + os.chdir(w_d) print("FAIL") return res - os.chdir(wd) + os.chdir(w_d) print("PASS") del_dat_file_vars(usecase_cfg) @@ -535,59 +561,61 @@ def run_tcase(rantech, cat, mu, bw, tcase, verbose, xran_path): return res def main(): + """Processes input files to produce IACA files""" test_results = [] - test_executed_total = 0 + all_test_cases = [] run_total = 0 - cat = 0 - mu = 0 - bw = 0 + cat = 0 + m_u = 0 + b_w = 0 tcase = 0 - """Processes input files to produce IACA files""" # Find path to XRAN if os.getenv("XRAN_DIR") is not None: xran_path = os.getenv("XRAN_DIR") else: print("please set 'export XRAN_DIR' in the OS") return -1 - + # Set up logging with given level (DEBUG, INFO, ERROR) for console end logfile init_logger(logging.INFO, logging.DEBUG) - host_name = socket.gethostname() - logging.info("host: %s Started testverification script: master.py from XRAN path %s",host_name, xran_path) + host_name = socket.gethostname() + logging.info("host: %s Started testverification script: master.py from XRAN path %s" + , host_name, xran_path) options = parse_args(sys.argv[1:]) rantech = options.rantech - cat = options.category - mu = options.numerology - bw = options.bandwidth - tcase = options.testcase - verbose = options.verbose + cat = options.category + m_u = options.numerology + b_w = options.bandwidth + tcase = options.testcase + #verbose = options.verbose - if (run_total): + if run_total: for test_run_ix in range(0, run_total): rantech = all_test_cases[test_run_ix][0] - cat = all_test_cases[test_run_ix][1] - mu = all_test_cases[test_run_ix][2] - bw = all_test_cases[test_run_ix][3] - tcase = all_test_cases[test_run_ix][4] - verbose = 0 - - logging.info("Test# %d out of %d: ran %d cat %d mu %d bw %d test case %d\n",test_run_ix, run_total, rantech, cat, mu, bw, tcase) - res = run_tcase(rantech, cat, mu, bw, tcase, verbose, xran_path) - if (res != 0): - test_results.append((rantech, cat, mu, bw, tcase,'FAIL')) + cat = all_test_cases[test_run_ix][1] + m_u = all_test_cases[test_run_ix][2] + b_w = all_test_cases[test_run_ix][3] + tcase = all_test_cases[test_run_ix][4] + #verbose = 0 + + logging.info("Test# %d out of %d: ran %d cat %d m_u %d b_w %d test case %d\n", + test_run_ix, run_total, rantech, cat, m_u, b_w, tcase) + res = run_tcase(rantech, cat, m_u, b_w, tcase, xran_path) + if res != 0: + test_results.append((rantech, cat, m_u, b_w, tcase, 'FAIL')) continue - test_results.append((rantech, cat, mu, bw, tcase,'PASS')) + test_results.append((rantech, cat, m_u, b_w, tcase, 'PASS')) with open('testresult.txt', 'w') as reshandle: json.dump(test_results, reshandle) else: - res = run_tcase(rantech, cat, mu, bw, tcase, verbose, xran_path) - if (res != 0): - test_results.append((rantech, cat, mu, bw, tcase,'FAIL')) - test_results.append((rantech, cat, mu, bw, tcase,'PASS')) + res = run_tcase(rantech, cat, m_u, b_w, tcase, xran_path) + if res != 0: + test_results.append((rantech, cat, m_u, b_w, tcase, 'FAIL')) + test_results.append((rantech, cat, m_u, b_w, tcase, 'PASS')) with open('testresult.txt', 'w') as reshandle: json.dump(test_results, reshandle) @@ -596,9 +624,9 @@ def main(): if __name__ == '__main__': START_TIME = datetime.now() - res = main() + RES = main() END_TIME = datetime.now() logging.debug("Start time: %s, end time: %s", START_TIME, END_TIME) logging.info("Execution time: %s", END_TIME - START_TIME) logging.shutdown() - sys.exit(res) + sys.exit(RES) From 0f911c1bc346a65df18897059fc8b55f278c65a2 Mon Sep 17 00:00:00 2001 From: relhwigi Date: Wed, 29 Jul 2020 15:39:37 +0100 Subject: [PATCH 20/80] OP-4034 pylint fixes --- .../etcd/deploy/files/eis_integ/eis_integ.py | 91 ++++++++++--------- .../etcd/deploy/files/eis_integ/etcd_users.py | 3 +- .../etcd/deploy/files/eis_integ/etcd_write.py | 3 +- .../deploy/files/eis_integ/etcd_zmq_key.py | 7 +- 4 files changed, 54 insertions(+), 50 deletions(-) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py index e9100cbe..29fbc4da 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py @@ -4,11 +4,8 @@ """ EIS integration module """ import enum import os -import sys import json import logging -import argparse -import shlex import subprocess import zmq import zmq.auth @@ -47,8 +44,9 @@ def load_json(json_path): try: with open(json_path, 'r') as json_file: return json.load(json_file) - except IOError as e: - logging.error("I/O error occurred during loading of {} json file: {}".format(json_path, e)) + except IOError as err: + logging.error("I/O error occurred during loading of %s json file: %s", + json_path, err) raise EisIntegError(CODES.FILE_SYSTEM_ERROR) @@ -62,24 +60,27 @@ def put_zmqkeys(appname): try: etcd_put("/Publickeys/" + appname, pub_key) - except Exception: - logging.error("Error putting Etcd public key for {}".format(appname)) + except RuntimeError: + logging.error("Error putting Etcd public key for %s", appname) try: etcd_put("/" + appname + "/private_key", secret_key) - except Exception: - logging.error("Error putting Etcd private key for {}".format(appname)) + except RuntimeError: + logging.error("Error putting Etcd private key for %s", appname) def enable_etcd_auth(password): """ Enable Auth for etcd and Create root user with root role """ try: - subprocess.check_output(["etcdctl", "user", "add", "root:"+password], stderr=subprocess.STDOUT) - subprocess.check_output(["etcdctl", "role", "add", "root"], stderr=subprocess.STDOUT) - subprocess.check_output(["etcdctl", "user", "grant-role", "root", "root"], stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "user", "add", "root:"+password], + stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "role", "add", "root"], + stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "user", "grant-role", "root", "root"], + stderr=subprocess.STDOUT) subprocess.check_output(["etcdctl", "auth", "enable"], stderr=subprocess.STDOUT) logging.info("Authentication has been enabled.") - except subprocess.CalledProcessError as e: - logging.error("Error while enabling authentication: {}".format(e)) + except subprocess.CalledProcessError as err: + logging.error("Error while enabling authentication: %s", err) raise EisIntegError(CODES.EXT_CMD_ERROR) def etcd_put(key, value): @@ -90,8 +91,8 @@ def etcd_put(key, value): """ try: subprocess.check_output(["etcdctl", "put", "--", key, value], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - logging.error("Error returned while adding the {} key to the etcd: {}".format(key, e)) + except subprocess.CalledProcessError as err: + logging.error("Error returned while adding the %s key to the etcd: %s", key, err) raise EisIntegError(CODES.EXT_CMD_ERROR) @@ -99,7 +100,7 @@ def etcd_put_json(json_data): """ Adds the contents of the json file to the etcd database """ for key, value in json_data.items(): etcd_put(key, bytes(json.dumps(value, indent=4).encode())) - logging.info("Value for the {} key has been added to the etcd.".format(key)) + logging.info("Value for the %s key has been added to the etcd.", key) def create_etcd_users(appname): @@ -110,22 +111,28 @@ def create_etcd_users(appname): :type appname: String """ try: - subprocess.check_output(["etcdctl", "user", "add", appname, "--no-password"], stderr=subprocess.STDOUT) - logging.info("User {} has been created.".format(appname)) - except subprocess.CalledProcessError as e: - logging.error("Error while creating {} user: {}".format(appname, e)) + subprocess.check_output(["etcdctl", "user", "add", appname, "--no-password"], + stderr=subprocess.STDOUT) + logging.info("User %s has been created.", appname) + except subprocess.CalledProcessError as err: + logging.error("Error while creating %s user: %s", appname, err) raise EisIntegError(CODES.EXT_CMD_ERROR) try: subprocess.check_output(["etcdctl", "role", "add", appname], stderr=subprocess.STDOUT) - subprocess.check_output(["etcdctl", "user", "grant-role", appname, appname], stderr=subprocess.STDOUT) - subprocess.check_output(["etcdctl", "role", "grant-permission", appname, "read", "/"+appname+"/", "--prefix"], stderr=subprocess.STDOUT) - subprocess.check_output(["etcdctl", "role", "grant-permission", appname, "readwrite", "/"+appname+"/datastore", "--prefix"], stderr=subprocess.STDOUT) - subprocess.check_output(["etcdctl", "role", "grant-permission", appname, "read", "/Publickeys/", "--prefix"], stderr=subprocess.STDOUT) - subprocess.check_output(["etcdctl", "role", "grant-permission", appname, "read", "/GlobalEnv/", "--prefix"], stderr=subprocess.STDOUT) - logging.info("Role {} has been created.".format(appname)) - except subprocess.CalledProcessError as e: - logging.error("Error while creating {} role: {}".format(appname, e)) + subprocess.check_output(["etcdctl", "user", "grant-role", appname, appname], + stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "role", "grant-permission", appname, + "read", "/"+appname+"/", "--prefix"], stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "role", "grant-permission", appname, "readwrite", + "/"+appname+"/datastore", "--prefix"], stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "role", "grant-permission", appname, + "read", "/Publickeys/", "--prefix"], stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "role", "grant-permission", appname, + "read", "/GlobalEnv/", "--prefix"], stderr=subprocess.STDOUT) + logging.info("Role %s has been created.", appname) + except subprocess.CalledProcessError as err: + logging.error("Error while creating %s role: %s", appname, err) raise EisIntegError(CODES.EXT_CMD_ERROR) @@ -136,7 +143,7 @@ def read_config(client): :type client: String """ logging.info("Read the configuration from etcd") - subprocess.run(["etcdctl", "get", client, "--prefix"]) + subprocess.run(["etcdctl", "get", client, "--prefix"], check=True) def get_etcd_users(): @@ -153,9 +160,9 @@ def remove_user_privilege(name): """ try: subprocess.check_output(["etcdctl", "role", "remove", name], stderr=subprocess.STDOUT) - logging.info("Role {} has been removed.".format(name)) - except subprocess.CalledProcessError as e: - logging.error("Error returned while removing the {} role: {}".format(name, e)) + logging.info("Role %s has been removed.", name) + except subprocess.CalledProcessError as err: + logging.error("Error returned while removing the %s role: %s", name, err) raise EisIntegError(CODES.EXT_CMD_ERROR) def remove_user(name): @@ -165,9 +172,9 @@ def remove_user(name): """ try: subprocess.check_output(["etcdctl", "user", "delete", name], stderr=subprocess.STDOUT) - logging.info("User {} has been removed.".format(name)) - except subprocess.CalledProcessError as e: - logging.error("Error returned while removing the {} user: {}".format(name, e)) + logging.info("User %s has been removed.", name) + except subprocess.CalledProcessError as err: + logging.error("Error returned while removing the %s user: %s", name, err) raise EisIntegError(CODES.EXT_CMD_ERROR) def add_user_role(): @@ -183,7 +190,7 @@ def remove_eis_application(): def remove_eis_key(key): """ Remove existing eis key. """ - subprocess.run(["etcdctl", "del", key, "--prefix"]) + subprocess.run(["etcdctl", "del", key, "--prefix"], check=True) def init_logger(): @@ -204,11 +211,11 @@ def extract_etcd_endpoint(): try: etcd_endpoint_ip = subprocess.check_output(["kubectl", "get", "svc", "-n", "eis", "-o", - jsonpath_ip_string]).decode('utf-8') + jsonpath_ip_string]).decode('utf-8') etcd_endpoint_port = subprocess.check_output(["kubectl", "get", "svc", "-n", "eis", "-o", - jsonpath_port_string]).decode('utf-8') - except subprocess.CalledProcessError as e: - logging.error("Failed to call kubectl command to extract ETCD endpoint: '{}'".format(e)) + jsonpath_port_string]).decode('utf-8') + except subprocess.CalledProcessError as err: + logging.error("Failed to call kubectl command to extract ETCD endpoint: '%s'", err) raise EisIntegError(CODES.EXT_CMD_ERROR) - + return etcd_endpoint_ip + ':' + etcd_endpoint_port diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_users.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_users.py index d3d1ab8d..f7073d66 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_users.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_users.py @@ -7,7 +7,6 @@ import logging import os import sys -import traceback import eis_integ def parse_arguments(_cli_args): @@ -51,5 +50,5 @@ def main(args): try: sys.exit(main(parse_arguments(sys.argv[1:])).value) except eis_integ.EisIntegError as exception: - logging.error("Error while doing operations on ETCD users: {}".format(exception)) + logging.error("Error while doing operations on ETCD users: %s", exception) sys.exit(exception.code.value) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_write.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_write.py index 207d0d63..e0f5fda4 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_write.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_write.py @@ -7,7 +7,6 @@ import logging import os import sys -import traceback import eis_integ def parse_arguments(_cli_args): @@ -37,5 +36,5 @@ def main(args): try: sys.exit(main(parse_arguments(sys.argv[1:])).value) except eis_integ.EisIntegError as exception: - logging.error("Error while adding entries to ETCD database: {}".format(exception)) + logging.error("Error while adding entries to ETCD database: %s", exception) sys.exit(exception.code.value) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py index 3117b462..8233eb73 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py @@ -7,14 +7,13 @@ import logging import os import sys -import traceback import eis_integ def parse_arguments(_cli_args): """ Parse argument passed to function """ parser = argparse.ArgumentParser(description="Specify Application name") - parser.add_argument("app", help= "Name of the client application") + parser.add_argument("app", help="Name of the client application") return parser.parse_args() def main(args): @@ -29,7 +28,7 @@ def main(args): eis_integ.check_path_variable("ETCDCTL_CERT", os.environ.get("ETCDCTL_CERT")) eis_integ.check_path_variable("ETCDCTL_KEY", os.environ.get("ETCDCTL_KEY")) - logging.info("Generate ZMQ pair keys for {} and put them to the etcd database".format(args.app)) + logging.info("Generate ZMQ pair keys for %s and put them to the etcd database", args.app) eis_integ.put_zmqkeys(args.app) return eis_integ.CODES.NO_ERROR @@ -39,5 +38,5 @@ def main(args): try: sys.exit(main(parse_arguments(sys.argv[1:])).value) except eis_integ.EisIntegError as exception: - logging.error("Error while generating ZMQ keys: {}".format(exception)) + logging.error("Error while generating ZMQ keys: %s", exception) sys.exit(exception.code.value) From 15e972dc0f0292bf0135c884ea5b726b296b6d33 Mon Sep 17 00:00:00 2001 From: konradja <48330825+konradja@users.noreply.github.com> Date: Sat, 1 Aug 2020 02:58:03 +0100 Subject: [PATCH 21/80] [OP-3963] Fix shellcheck issues in edgeapps repo (#103) * Fix shellcheck issues in cdn-transcode/self-sign.sh. * Fix quoting in applications/cdn-transcode/clean.sh. * Rewrite the for loop in clean.sh to fix SC2044. * Fix shellcheck issues in ./applications/cdn-transcode/ovc-clone-compile.sh. * Fix shellcheck issues in ./applications/cdn-transcode/pre-install.sh. * Fix shellcheck issues in ansible-precheck.sh * Fix shellcheck issues in eis_patch_rel_branch.sh. * Fix shellcheck issues in eis_repo_config.sh. * Fix shellcheck problems in fpga-sample-app/build-image.sh. * Fix shebang in build_image.sh. * Fix shebang in run_container.sh. * Fix shellcheck issues in build-image.sh. * Fix shellcheck issues in ./applications/openvino/consumer/build-image.sh. * Fix applications/openvino/producer/build-image.sh. * Fix shellcheck issues in applications/smart-city-app/certs/self-sign.sh. * Remove usused var. * Fix shellcheck issues in ./applications/smart-city-app/pre-install.sh. * Fix shebang in ./applications/telemetry-sample-app/create-secret.sh * Fix shellcheck issues in ./applications/telemetry-sample-app/image/build.sh. * Fix shellcheck issues in ./iot-gateway/baidu-openedge/scripts/ansible/examples/setup_baidu_openedge/composefile/precheck.sh. * Fix shellcheck issues in ./network-functions/core-network/5G/UPF/build_image.sh. * All shellcheck issues in ./network-functions/core-network/5G/UPF/run_upf.sh fixed. * Fixed quotes in the usage print functions in ./network-functions/xran/run_test.sh. * run_test.sh fixes - first bunch. * Fix the complex quoting issues in ./network-functions/xran/run_test.sh. * Fix reminder of the issues in ./network-functions/xran/run_test.sh. --- applications/cdn-transcode/certs/self-sign.sh | 10 +- applications/cdn-transcode/clean.sh | 5 +- .../cdn-transcode/ovc-clone-compile.sh | 8 +- applications/cdn-transcode/pre-install.sh | 8 +- .../scripts/ansible-precheck.sh | 2 +- .../scripts/eis_patch_rel_branch.sh | 9 +- .../scripts/eis_repo_config.sh | 6 +- applications/fpga-sample-app/build-image.sh | 11 +- .../openvino/benchmark/build_image.sh | 4 +- .../openvino/benchmark/run_container.sh | 4 +- .../openvino/clientsim/build-image.sh | 7 +- applications/openvino/consumer/build-image.sh | 7 +- applications/openvino/producer/build-image.sh | 7 +- .../smart-city-app/certs/self-sign.sh | 10 +- applications/smart-city-app/clean.sh | 2 - applications/smart-city-app/pre-install.sh | 3 +- .../telemetry-sample-app/create-secret.sh | 5 +- .../telemetry-sample-app/image/build.sh | 11 +- .../composefile/precheck.sh | 15 ++- .../core-network/5G/UPF/build_image.sh | 9 +- .../core-network/5G/UPF/run_upf.sh | 4 +- network-functions/xran/run.sh | 9 +- network-functions/xran/run_test.sh | 116 +++++++++--------- 23 files changed, 135 insertions(+), 137 deletions(-) diff --git a/applications/cdn-transcode/certs/self-sign.sh b/applications/cdn-transcode/certs/self-sign.sh index 0fe5720c..468ffae9 100755 --- a/applications/cdn-transcode/certs/self-sign.sh +++ b/applications/cdn-transcode/certs/self-sign.sh @@ -3,10 +3,11 @@ # Copyright (c) 2020 Intel Corporation IMAGE="ovc_self_certificate" -DIR=$(dirname $(readlink -f "$0")) +LINK=$(readlink -f "$0") +DIR=$(dirname "$LINK") USER="docker" -case "$(cat /proc/1/sched | head -n 1)" in +case "$(head -n 1 /proc/1/sched)" in *self-sign*) openssl req -x509 -nodes -days 30 -newkey rsa:4096 -keyout /home/$USER/self.key -out /home/$USER/self.crt << EOL US @@ -26,11 +27,10 @@ EOL pid="$(docker ps -f ancestor=$IMAGE --format='{{.ID}}' | head -n 1)" if [ -n "$pid" ] && [ "$#" -le "1" ]; then echo "bash into running container...$IMAGE" - docker exec -it $pid ${*-/bin/bash} + docker exec -it "$pid" "${*-/bin/bash}" else echo "bash into new container...$IMAGE" - args=("$@") - docker run --rm ${OPTIONS[@]} $(env | cut -f1 -d= | grep -E '_(proxy|REPO|VER)$' | sed 's/^/-e /') --entrypoint /home/$USER/self-sign.sh -it "${IMAGE}" "$(hostname -f)" + docker run --rm "${OPTIONS[@]}" "$(env | cut -f1 -d= | grep -E '_(proxy|REPO|VER)$' | sed 's/^/-e /')" --entrypoint /home/$USER/self-sign.sh -it "${IMAGE}" "$(hostname -f)" fi ;; esac diff --git a/applications/cdn-transcode/clean.sh b/applications/cdn-transcode/clean.sh index 50969ed8..768690c1 100755 --- a/applications/cdn-transcode/clean.sh +++ b/applications/cdn-transcode/clean.sh @@ -2,10 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -DIR=$(dirname $(readlink -f "$0")) +LINK=$(readlink -f "$0") +DIR=$(dirname "$LINK") # delete pvs and scs -for yaml in $(find "${DIR}/CDN-Transcode-Sample/deployment/kubernetes" -maxdepth 1 -name "*-pv.yaml" -print); do +for yaml in ${DIR}/CDN-Transcode-Sample/deployment/kubernetes/*-pv.yaml; do kubectl delete --wait=false -f "$yaml" --ignore-not-found=true 2>/dev/null done diff --git a/applications/cdn-transcode/ovc-clone-compile.sh b/applications/cdn-transcode/ovc-clone-compile.sh index 0d0a46b7..72034f57 100755 --- a/applications/cdn-transcode/ovc-clone-compile.sh +++ b/applications/cdn-transcode/ovc-clone-compile.sh @@ -13,13 +13,13 @@ cd build if kubectl get pods -A | grep docker-registry ; then mName=$(kubectl get nodes | grep master | cut -f 1 -d ' ') - if [ ! -z $mName ]; then - mIP=$(kubectl get node -o 'custom-columns=NAME:.status.addresses[?(@.type=="Hostname")].address,IP:.status.addresses[?(@.type=="InternalIP")].address' | awk '!/NAME/{print $1":"$2}' | grep $mName | cut -f 2 -d':') + if [ ! -z "$mName" ]; then + mIP=$(kubectl get node -o 'custom-columns=NAME:.status.addresses[?(@.type=="Hostname")].address,IP:.status.addresses[?(@.type=="InternalIP")].address' | awk '!/NAME/{print $1":"$2}' | grep "$mName" | cut -f 2 -d':') fi echo "mName=$mName mIP=$mIP" fi -if [ ! -z $mIP ]; then +if [ ! -z "$mIP" ]; then cmake -DREGISTRY="${mIP}:5000/" .. else cmake .. @@ -27,5 +27,3 @@ fi make || { echo "failed to compile CDN transcode Sample"; exit 2;} echo "Success: Compiled the CDN Transcode Sample" - - diff --git a/applications/cdn-transcode/pre-install.sh b/applications/cdn-transcode/pre-install.sh index 54d8a334..ed410a2c 100755 --- a/applications/cdn-transcode/pre-install.sh +++ b/applications/cdn-transcode/pre-install.sh @@ -3,7 +3,9 @@ # Copyright (c) 2020 Intel Corporation echo "Pre-Installation setup for CDN Transcode Sample" -DIR=$(dirname $(readlink -f "$0")) + +LNK=$(readlink -f "$0") +DIR=$(dirname "$LNK") function create_secret { kubectl create secret generic self-signed-certificate "--from-file=${DIR}/certs/self.crt" "--from-file=${DIR}/certs/self.key" } @@ -12,9 +14,9 @@ function create_secret { "$DIR/certs/self-sign.sh" create_secret 2>/dev/null || (kubectl delete secret self-signed-certificate; create_secret) -for yaml in $(find "$DIR/CDN-Transcode-Sample/deployment/kubernetes/" -maxdepth 1 -name "*-pv.yaml" -print); do +for yaml in "$DIR"/CDN-Transcode-Sample/deployment/kubernetes/*-pv.yaml; do kubectl apply -f "$yaml" done # create volume -$DIR/CDN-Transcode-Sample/deployment/kubernetes/mkvolume.sh +"$DIR/CDN-Transcode-Sample/deployment/kubernetes/mkvolume.sh" diff --git a/applications/eis-experience-kit/scripts/ansible-precheck.sh b/applications/eis-experience-kit/scripts/ansible-precheck.sh index 666be4a7..c4802a92 100755 --- a/applications/eis-experience-kit/scripts/ansible-precheck.sh +++ b/applications/eis-experience-kit/scripts/ansible-precheck.sh @@ -8,7 +8,7 @@ if [[ $? -ne 0 ]]; then exit 1 fi -if [ ${0##*/} == ${BASH_SOURCE[0]##*/} ]; then +if [ "${0##*/}" == "${BASH_SOURCE[0]##*/}" ]; then echo "ERROR: This script cannot be executed directly" exit 1 fi diff --git a/applications/eis-experience-kit/scripts/eis_patch_rel_branch.sh b/applications/eis-experience-kit/scripts/eis_patch_rel_branch.sh index 83a672b9..82062335 100755 --- a/applications/eis-experience-kit/scripts/eis_patch_rel_branch.sh +++ b/applications/eis-experience-kit/scripts/eis_patch_rel_branch.sh @@ -35,20 +35,21 @@ then fi # Extract release package to apply patch -tar xvzf $release_source_path -C $tempdir +tar xvzf "$release_source_path" -C "$tempdir" # Save current working directory to get patch file path and move to release source path cwd=$(pwd) package_dir=($tempdir/*) -sources_dir=$tempdir/$(basename $package_dir)/IEdgeInsights -cd $sources_dir +bdir=$(basename "${package_dir[0]}") +sources_dir="$tempdir/$bdir/IEdgeInsights" +cd "$sources_dir" # Apply patch patch -ruN -p1 < "$cwd"/eis_diff_patch.patch # Zip the folder after applying patch cd $tempdir -tar zcvf $release_destination_path $(basename $package_dir) +tar zcvf "$release_destination_path" "$bdir" # Remove temporary diretory rm -rf $tempdir diff --git a/applications/eis-experience-kit/scripts/eis_repo_config.sh b/applications/eis-experience-kit/scripts/eis_repo_config.sh index 6552668d..477f5be0 100755 --- a/applications/eis-experience-kit/scripts/eis_repo_config.sh +++ b/applications/eis-experience-kit/scripts/eis_repo_config.sh @@ -8,10 +8,10 @@ if [ "$#" -ne 1 ]; then fi # get eis_sources_dir location from ./group_vars/all.yml -sources=$(cat $1/group_vars/all.yml | sed -n -e 's/^eis_sources_dir: //p' | sed 's@"@@g') +sources=$(sed -n -e 's/^eis_sources_dir: //p' < "$1/group_vars/all.yml" | sed 's@"@@g') # combine two elements and get .env file location -env=$(cat $1/host_vars/localhost.yml | sed -n -e 's/^eis_env_file: //p' | sed -e "s@{{ eis_sources_dir }}@$sources@" | sed 's@"@@g') +env=$(sed -n -e 's/^eis_env_file: //p' < "$1/host_vars/localhost.yml" | sed -e "s@{{ eis_sources_dir }}@$sources@" | sed 's@"@@g') if [ ! -f "$env" ]; then echo "$env doesn't exist. Terminating." @@ -19,5 +19,5 @@ if [ ! -f "$env" ]; then fi set -a -source $env +source "$env" set +a diff --git a/applications/fpga-sample-app/build-image.sh b/applications/fpga-sample-app/build-image.sh index cbd13d5c..6db5c37a 100755 --- a/applications/fpga-sample-app/build-image.sh +++ b/applications/fpga-sample-app/build-image.sh @@ -9,16 +9,17 @@ BBDEV_PATCH=./dpdk_19.11_new.patch if test -f "$BBDEV_PATCH"; then echo "BBDEV patch found" else - echo "$BBDEV_PATCH does not exist." \ + printf "%s does not exist." \ "Please place the BBDEV patch for DPDK 19.11 into root directory of this script.\n" \ "The patch is provided as part of v19.12 Release of FlexRAN package.\n" \ - "The script will now exit, the image will not be created.\n" + "The script will now exit, the image will not be created.\n" $BBDEV_PATCH exit fi +declare http_proxy https_proxy no_proxy sudo docker build \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg no_proxy=$no_proxy \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg no_proxy="$no_proxy" \ -t bbdev-sample-app:1.0 . diff --git a/applications/openvino/benchmark/build_image.sh b/applications/openvino/benchmark/build_image.sh index 1f8971a2..dd91e5a0 100755 --- a/applications/openvino/benchmark/build_image.sh +++ b/applications/openvino/benchmark/build_image.sh @@ -1,6 +1,6 @@ +#!/bin/bash -xe + # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -#!/bin/bash -xe - docker build -t openvino-benchmark:1.0 . diff --git a/applications/openvino/benchmark/run_container.sh b/applications/openvino/benchmark/run_container.sh index 3eca5c59..9d5a6418 100755 --- a/applications/openvino/benchmark/run_container.sh +++ b/applications/openvino/benchmark/run_container.sh @@ -1,8 +1,8 @@ +#!/bin/bash -xe + # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -#!/bin/bash -xe - docker run --rm --device-cgroup-rule='c 10:* rmw' --device-cgroup-rule='c 89:* rmw' --device-cgroup-rule='c 189:* rmw' --device-cgroup-rule='c 180:* rmw' -v /dev:/dev -v /var/tmp:/var/tmp --name openvino-benchmark --network host -it openvino-benchmark:1.0 /bin/bash diff --git a/applications/openvino/clientsim/build-image.sh b/applications/openvino/clientsim/build-image.sh index 0f1789ef..57f043f6 100755 --- a/applications/openvino/clientsim/build-image.sh +++ b/applications/openvino/clientsim/build-image.sh @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2019 Intel Corporation +declare http_proxy https_proxy no_proxy sudo docker build \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg no_proxy=$no_proxy \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg no_proxy="$no_proxy" \ -t client-sim:1.0 . diff --git a/applications/openvino/consumer/build-image.sh b/applications/openvino/consumer/build-image.sh index c956c18e..80fb51e1 100755 --- a/applications/openvino/consumer/build-image.sh +++ b/applications/openvino/consumer/build-image.sh @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2019 Intel Corporation +declare http_proxy https_proxy no_proxy sudo docker build \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg no_proxy=$no_proxy \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg no_proxy="$no_proxy" \ -t openvino-cons-app:1.0 . diff --git a/applications/openvino/producer/build-image.sh b/applications/openvino/producer/build-image.sh index 4e69f41c..28c364c4 100755 --- a/applications/openvino/producer/build-image.sh +++ b/applications/openvino/producer/build-image.sh @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2019 Intel Corporation +declare http_proxy https_proxy no_proxy sudo docker build \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg no_proxy=$no_proxy \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg no_proxy="$no_proxy" \ -t openvino-prod-app:1.0 . diff --git a/applications/smart-city-app/certs/self-sign.sh b/applications/smart-city-app/certs/self-sign.sh index 84b8dd87..4abff6e3 100755 --- a/applications/smart-city-app/certs/self-sign.sh +++ b/applications/smart-city-app/certs/self-sign.sh @@ -3,10 +3,11 @@ # Copyright (c) 2020 Intel Corporation IMAGE="smtc_certificate" -DIR=$(dirname $(readlink -f "$0")) +LNK=$(readlink -f "$0") +DIR=$(dirname "$LNK") USER="docker" -case "$(cat /proc/1/sched | head -n 1)" in +case "$(head -n 1 /proc/1/sched)" in *self-sign*) openssl req -x509 -nodes -days 30 -newkey rsa:4096 -keyout /home/$USER/self.key -out /home/$USER/self.crt << EOL US @@ -26,11 +27,10 @@ EOL pid="$(docker ps -f ancestor=$IMAGE --format='{{.ID}}' | head -n 1)" if [ -n "$pid" ] && [ "$#" -le "1" ]; then echo "bash into running container...$IMAGE" - docker exec -it $pid ${*-/bin/bash} + docker exec -it "$pid" "${*-/bin/bash}" else echo "bash into new container...$IMAGE" - args=("$@") - docker run --rm ${OPTIONS[@]} $(env | cut -f1 -d= | grep -E '_(proxy|REPO|VER)$' | sed 's/^/-e /') --entrypoint /home/$USER/self-sign.sh -it "${IMAGE}" "$(hostname -f)" + docker run --rm "${OPTIONS[@]}" "$(env | cut -f1 -d= | grep -E '_(proxy|REPO|VER)$' | sed 's/^/-e /')" --entrypoint /home/$USER/self-sign.sh -it "${IMAGE}" "$(hostname -f)" fi ;; esac diff --git a/applications/smart-city-app/clean.sh b/applications/smart-city-app/clean.sh index 754096fb..38344939 100755 --- a/applications/smart-city-app/clean.sh +++ b/applications/smart-city-app/clean.sh @@ -2,7 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -DIR=$(dirname $(readlink -f "$0")) - kubectl delete secret self-signed-certificate 2> /dev/null || echo -n "" kubectl delete configmap sensor-info 2> /dev/null || echo -n "" diff --git a/applications/smart-city-app/pre-install.sh b/applications/smart-city-app/pre-install.sh index 0aa51f40..4b650cfe 100755 --- a/applications/smart-city-app/pre-install.sh +++ b/applications/smart-city-app/pre-install.sh @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -DIR=$(dirname $(readlink -f "$0")) +LNK=$(readlink -f "$0") +DIR=$(dirname "$LNK") function create_secret { kubectl create secret generic self-signed-certificate "--from-file=${DIR}/certs/self.crt" "--from-file=${DIR}/certs/self.key" diff --git a/applications/telemetry-sample-app/create-secret.sh b/applications/telemetry-sample-app/create-secret.sh index 742fef94..65cff415 100755 --- a/applications/telemetry-sample-app/create-secret.sh +++ b/applications/telemetry-sample-app/create-secret.sh @@ -1,10 +1,9 @@ +#!/bin/bash + # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -#!/bin/bash - #Default path to Telemetry root CA on EdgeC Controller CERT_PATH="/etc/openness/certs/telemetry/CA/cert.pem" kubectl create secret generic root-cert --from-file=${CERT_PATH} --namespace=default - diff --git a/applications/telemetry-sample-app/image/build.sh b/applications/telemetry-sample-app/image/build.sh index ca964cbc..88cd2542 100755 --- a/applications/telemetry-sample-app/image/build.sh +++ b/applications/telemetry-sample-app/image/build.sh @@ -1,8 +1,8 @@ +#!/bin/bash + # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -#!/bin/bash - #Build docker image echo "Building docker image" @@ -10,11 +10,10 @@ docker build -t metricapp:0.1.0 . args=("$@") -if [ ${args[0]} == "push" ] +if [ "${args[0]}" == "push" ] then #tag docker image - docker image tag metricapp:0.1.0 ${args[1]}:${args[2]}/intel/metricapp:0.1.0 + docker image tag metricapp:0.1.0 "${args[1]}:${args[2]}/intel/metricapp:0.1.0" #push to docker registry - docker push ${args[1]}:${args[2]}/intel/metricapp:0.1.0 + docker push "${args[1]}:${args[2]}/intel/metricapp:0.1.0" fi - diff --git a/iot-gateway/baidu-openedge/scripts/ansible/examples/setup_baidu_openedge/composefile/precheck.sh b/iot-gateway/baidu-openedge/scripts/ansible/examples/setup_baidu_openedge/composefile/precheck.sh index f67bd24a..89094848 100644 --- a/iot-gateway/baidu-openedge/scripts/ansible/examples/setup_baidu_openedge/composefile/precheck.sh +++ b/iot-gateway/baidu-openedge/scripts/ansible/examples/setup_baidu_openedge/composefile/precheck.sh @@ -5,24 +5,23 @@ # check democfg directory -currdir=${PWD##*/} BASE_PATH="$( cd "$(dirname "$0")" ; pwd -P )" echo "Starting check whether certification and configuraiton files exsit in the democfg" if [ -d "${BASE_PATH}/democfg/agent-cert" ]; then - if [ ! -f ${BASE_PATH}/democfg/agent-cert/client.key ]; then + if [ ! -f "${BASE_PATH}/democfg/agent-cert/client.key" ]; then echo "Error: no client.key in the agent-cert directory." exit 1 fi - if [ ! -f ${BASE_PATH}/democfg/agent-cert/client.pem ]; then + if [ ! -f "${BASE_PATH}/democfg/agent-cert/client.pem" ]; then echo "Error: no client.pem in the agent-cert directory." exit 1 fi - if [ ! -f ${BASE_PATH}/democfg/agent-cert/openapi.pem ]; then + if [ ! -f "${BASE_PATH}/democfg/agent-cert/openapi.pem" ]; then echo "Error: no openapi.pem in the agent-cert directory." exit 1 fi - if [ ! -f ${BASE_PATH}/democfg/agent-cert/root.pem ]; then + if [ ! -f "${BASE_PATH}/democfg/agent-cert/root.pem" ]; then echo "Error: no root.pem in the agent-cert directory." exit 1 fi @@ -32,7 +31,7 @@ else fi if [ -d "${BASE_PATH}/democfg/agent-conf" ]; then - if [ ! -f ${BASE_PATH}/democfg/agent-conf/service.yml ]; then + if [ ! -f "${BASE_PATH}/democfg/agent-conf/service.yml" ]; then echo "Error: no service.yml in the agent-conf directory." exit 1 fi @@ -42,7 +41,7 @@ else fi if [ -d "${BASE_PATH}/democfg/remote-iothub-conf" ]; then - if [ ! -f ${BASE_PATH}/democfg/remote-iothub-conf/service.yml ]; then + if [ ! -f "${BASE_PATH}/democfg/remote-iothub-conf/service.yml" ]; then echo "Error: no service.yml in the remote-iothub-conf directory." exit 1 fi @@ -51,7 +50,7 @@ else exit 1 fi -if [ ! -f ${BASE_PATH}/democfg/application.yml ]; then +if [ ! -f "${BASE_PATH}/democfg/application.yml" ]; then echo "Error: no application.yml in the democfg directory." exit 1 fi diff --git a/network-functions/core-network/5G/UPF/build_image.sh b/network-functions/core-network/5G/UPF/build_image.sh index c46b0ef5..6e8adee1 100755 --- a/network-functions/core-network/5G/UPF/build_image.sh +++ b/network-functions/core-network/5G/UPF/build_image.sh @@ -3,10 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation +declare http_proxy https_proxy no_proxy sudo docker build \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg no_proxy=$no_proxy \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg no_proxy="$no_proxy" \ -t upf-cnf:1.0 . - - diff --git a/network-functions/core-network/5G/UPF/run_upf.sh b/network-functions/core-network/5G/UPF/run_upf.sh index 107865b8..c5fadc9c 100755 --- a/network-functions/core-network/5G/UPF/run_upf.sh +++ b/network-functions/core-network/5G/UPF/run_upf.sh @@ -20,7 +20,7 @@ n6_ip=$CONFIG_N6_IP_ADDR # remove the subnet from the n3_ip address -n3_ip_plain=`echo $n3_ip | awk -F'/' '{ print $1 }'` +n3_ip_plain=$(echo "$n3_ip" | awk -F'/' '{ print $1 }') # n4 ip address without subnet n4_ip=$CONFIG_N4_IP_ADDR # Gateway IP address @@ -334,7 +334,7 @@ echo 2 > /proc/sys/fs/suid_dumpable export LD_LIBRARY_PATH=$upf_path/install-vpp-native/vpp/lib:$LD_LIBRARY_PATH echo "=========== Start =========" > /root/upf/run.log -echo $LD_LIBRARY_PATH >> $upf_path/run.log +echo "$LD_LIBRARY_PATH" >> $upf_path/run.log echo "at step-1" >> $upf_path/run.log cd $upf_path echo "at step-2" >> $upf_path/run.log diff --git a/network-functions/xran/run.sh b/network-functions/xran/run.sh index c742c02d..f5cb3498 100755 --- a/network-functions/xran/run.sh +++ b/network-functions/xran/run.sh @@ -2,9 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation +declare http_proxy https_proxy ftp_proxy no_proxy sudo docker build \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg ftp_proxy=$ftp_proxy \ - --build-arg no_proxy=$no_proxy \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg ftp_proxy="$ftp_proxy" \ + --build-arg no_proxy="$no_proxy" \ -t xran-sample-app:1.0 . diff --git a/network-functions/xran/run_test.sh b/network-functions/xran/run_test.sh index ed92db2b..29ae89df 100755 --- a/network-functions/xran/run_test.sh +++ b/network-functions/xran/run_test.sh @@ -19,13 +19,12 @@ APP_BIN="./build/sample-app" declare -a ethUpDev declare -a ethCpDev -declare -a script printEnv() { - echo -e "\tXRAN_DIR="$XRAN_DIR - echo -e "\tRTE_SDK="$RTE_SDK - echo -e "\tRTE_TARGET"=$RTE_TARGET + echo -e "\tXRAN_DIR=$XRAN_DIR" + echo -e "\tRTE_SDK=$RTE_SDK" + echo -e "\tRTE_TARGET=$RTE_TARGET" } @@ -47,17 +46,17 @@ printHelp() printArgs() { - echo -e "\tappMode="$mode - echo -e "\ttech="$tech" ( 0 for 5G, 1 for LTE )" - echo -e "\tcat="$cat - echo -e "\ttc="$tc - echo -e "\tmu="$mu - echo -e "\tbw="$bw - echo -e "\tportsNum="$portNum - echo -e "\tport_up_du="${ethUpDev[0]} - echo -e "\tport_cp_du="${ethCpDev[0]} - echo -e "\tport_up_ru="${ethUpDev[1]} - echo -e "\tport_cp_ru="${ethCpDev[1]} + echo -e "\tappMode=$mode" + echo -e "\ttech=$tech ( 0 for 5G, 1 for LTE )" + echo -e "\tcat=$cat" + echo -e "\ttc=$tc" + echo -e "\tmu=$mu" + echo -e "\tbw=$bw" + echo -e "\tportsNum=$portNum" + echo -e "\tport_up_du=${ethUpDev[0]}" + echo -e "\tport_cp_du=${ethCpDev[0]}" + echo -e "\tport_up_ru=${ethUpDev[1]}" + echo -e "\tport_cp_ru=${ethCpDev[1]}" } @@ -72,10 +71,10 @@ checkMode() "du" ) reqPortNum=2 ;; "none" ) - echo -e "\tExiting application started with mode: "$mode + echo -e "\tExiting application started with mode: $mode" exit 0 ;; * ) - echo -e "\tWrong application mode: "$mode + echo -e "\tWrong application mode: $mode" printHelp exit 1 ;; esac @@ -83,7 +82,7 @@ checkMode() if [[ -z "$portNum" ]]; then portNum=$reqPortNum elif [[ $portNum -ne $reqPortNum ]]; then - echo "Number of ports is set to "$portNum ". Application requires "$reqPortNum" ports." + echo "Number of ports is set to "$portNum ". Application requires $reqPortNum ports." exit 1 fi @@ -93,7 +92,7 @@ setEnvPorts() { IFS=$'\n' read -r -d '' -a vfs < <( lspci | grep "Virtual Function" | awk '{ print $1}' && printf '\0' ) if [[ ${#vfs[@]} -lt $portNum ]]; then - echo -e "\tFound "${#vfs[@]}" virtual ports available, required "$portNum + echo -e "\tFound ${#vfs[@]} virtual ports available, required $portNum" exit 1 fi @@ -123,7 +122,7 @@ procModeReq() "ru-du" ) if [ -z "$p1" ]; then p1=$DEF_DU_P1 - echo "SETTING P1: "$p1 + echo "SETTING P1: $p1" fi if [ -z "$p2" ]; then p2=$DEF_DU_P2 @@ -170,15 +169,15 @@ runTestScripts() scripts="$1" len=${#scripts[@]} - for ((i=1; i<${len}+1; i++)) + for ((i=1; i "test_output"$i".txt" 2>&1 & + "./${scripts[$i-1]}" > "test_output$i.txt" 2>&1 & pids+="$! " done p=0 for pid in $pids; do - wait $pid + wait "$pid" if [ $? -eq 0 ]; then echo -e "\tProcess $pid running ${scripts[$p]} exited with a status $?" else @@ -195,49 +194,49 @@ run() len=${#scripts[@]} cmd="./usecase/" - if [ $tech -eq 1 ]; then - if [ $cat -eq 1 ]; then - cmd=$cmd"lte_b/mu"$mu"_"$bw"mhz/" - elif [ $cat -eq 0 ]; then - cmd=$cmd"lte_a/mu"$mu"_"$bw"mhz/" + if [ "$tech" -eq 1 ]; then + if [ "$cat" -eq 1 ]; then + cmd="${cmd}lte_b/mu${mu}_${bw}mhz/" + elif [ "$cat" -eq 0 ]; then + cmd="${cmd}lte_a/mu${mu}_${bw}mhz/" else echo "Wrong cat argument" exit 1 fi - elif [ $tech -eq 0 ]; then - if [ $cat -eq 1 ]; then - echo "cat ="$cat - cmd=$cmd"cat_b/mu"$mu"_"$bw"mhz/" - elif [ $cat -eq 0 ]; then - cmd=$cmd"mu"$mu"_"$bw"mhz/" + elif [ "$tech" -eq 0 ]; then + if [ "$cat" -eq 1 ]; then + echo "cat = $cat" + cmd="${cmd}cat_b/mu${mu}_${bw}mhz/" + elif [ "$cat" -eq 0 ]; then + cmd="${cmd}mu${mu}_${bw}mhz/" else echo "Wrong cat argument" exit 1 fi fi - if [ $tc -gt 0 ]; then - cmd=$cmd$tcNum"/" + if [ "$tc" -gt 0 ]; then + cmd=$cmd$tc"/" fi - for ((i=1; i<${len}+1; i++)) + for ((i=1; i $outFile 2>&1 & + echo -e "\tWriting application output to $outFile" + $testCmd > "$outFile" 2>&1 & pids+="$! " done p=0 for pid in $pids; do - wait $pid + wait "$pid" if [ $? -eq 0 ]; then echo -e "\tProcess $pid running ${scripts[$p]} exited with a status $?" else @@ -250,20 +249,20 @@ run() testDuRu() { scripts=( $CONFIG_FILE_DU $CONFIG_FILE_RU ) - cd $XRAN_DIR/app - run $scripts + cd "$XRAN_DIR/app" + run "${scripts[0]}" } testDu() { scripts=( $CONFIG_FILE_DU ) - run $scripts + run "${scripts[0]}" } testRu() { scripts=( $CONFIG_FILE_RU ) - run $scripts + run "${scripts[0]}" } verifyTests() @@ -274,7 +273,7 @@ verifyTests() echo "Error while removing RU files" exit 1 fi - cp $XRAN_DIR/app/logs/o-ru-* $RESULTS_DIR + cp "$XRAN_DIR"/app/logs/o-ru-* $RESULTS_DIR if [ $? -ne 0 ]; then echo "Error while coping RU files" exit 1 @@ -286,25 +285,23 @@ verifyTests() echo "Error while removing DU files" exit 1 fi - cp $XRAN_DIR/app/logs/o-du-* $RESULTS_DIR + cp "$XRAN_DIR"/app/logs/o-du-* $RESULTS_DIR if [ $? -ne 0 ]; then echo "Error while coping DU files" exit 1 fi fi if [[ "$mode" = "ru" ]]; then - echo -e "\t"$(date)" RU test completed" + echo -e "\t$(date) RU test completed" return fi - du_files=$(cd $RESULTS_DIR && ls -dq *o-du-*.txt | wc -l) - ANT_NUM=$du_files - cd $XRAN_DIR/app + cd "$XRAN_DIR/app" echo "STEP 2: VERIFY TESTS RESULTS" - python $XRAN_DIR/app/test_verification.py --ran $tech --cat $cat --testcase $tc --mu $mu --b $bw --verbose $TEST_VERBOSE - echo -e "\t"$(date)" xRAN sample app test and test verififaction completed" + python "$XRAN_DIR/app/test_verification.py" --ran "$tech" --cat "$cat" --testcase "$tc" --mu "$mu" --b "$bw" --verbose "$TEST_VERBOSE" + echo -e "\t$(date) xRAN sample app test and test verififaction completed" } @@ -337,7 +334,7 @@ do i ) read -a ports <<< "$OPTARG" if [[ ${#ports[*]} -ne $portNum ]]; then - echo -e "\t Received "${#ports[*]} " port(s), while awaiting "$portNum + echo -e "\t Received ${#ports[*]} port(s), while awaiting $portNum" fi p1=${ports[0]} p2=${ports[1]} @@ -370,19 +367,18 @@ printArgs echo "STEP 1: RUN TESTS" -cd $XRAN_DIR/app +cd "$XRAN_DIR/app" case $mode in "ru-du" ) - echo -e "\t"$(date) "Starting RU and DU" + echo -e "\t$(date) Starting RU and DU" testDuRu ;; "ru" ) - echo -e "\t"$(date) "Starting RU" + echo -e "\t$(date) Starting RU" testRu ;; "du" ) - echo -e "\t"$(date) "Starting DU" + echo -e "\t$(date) Starting DU" testDu ;; esac #VERIFY TESTS RESULTS" verifyTests - From 8682b7f8f47049980d2ed1a46322a6b7ebd01311 Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:25:39 +0530 Subject: [PATCH 22/80] Enabled Safety Hat demo and rtsp stream from remote host (#105) * enabled Safety Hat demo and rtsp stream from remote host --- .../eis-experience-kit/deploy_eis_pcb_demo.sh | 1 + .../eis-experience-kit/eis_pcb_demo.yml | 1 + .../eis-experience-kit/group_vars/all.yml | 10 ++++++++ .../camera_stream/build/defaults/main.yml | 3 ++- .../camera_stream/build/files/Dockerfile | 2 +- .../camera_stream/build/files/streamstart.sh | 24 +++++++++++++++++-- .../roles/camera_stream/build/tasks/main.yml | 9 +++++-- .../camera_stream/deploy/defaults/main.yml | 1 - .../templates/camera-stream-pod.yaml | 1 + .../templates/camera-stream-values.yaml.j2 | 3 ++- .../video_analytics/deploy/tasks/main.yml | 7 +++--- .../video_analytics_pcb.json.j2} | 0 .../templates/video_analytics_safety.json.j2 | 18 ++++++++++++++ .../templates/video-ingestion-pod.yaml | 2 ++ .../video_ingestion/deploy/tasks/main.yml | 7 +++--- .../video_ingestion_pcb.json.j2} | 2 +- .../templates/video_ingestion_safety.json.j2 | 21 ++++++++++++++++ .../templates/video_ingestion_values.yaml.j2 | 1 + .../roles/visualizer/deploy/defaults/main.yml | 2 ++ .../visualizer/templates/visualizer-pod.yaml | 4 ++++ .../roles/visualizer/deploy/tasks/main.yml | 7 +++--- .../templates/visualizer-values.yaml.j2 | 2 ++ .../deploy/templates/visualizer_pcb.json.j2 | 12 ++++++++++ .../visualizer_safety.json.j2} | 10 +++----- .../scripts/task_log_file.sh | 11 +++++++++ 25 files changed, 136 insertions(+), 25 deletions(-) mode change 100644 => 100755 applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh rename applications/eis-experience-kit/roles/video_analytics/deploy/{files/video_analytics.json => templates/video_analytics_pcb.json.j2} (100%) create mode 100644 applications/eis-experience-kit/roles/video_analytics/deploy/templates/video_analytics_safety.json.j2 rename applications/eis-experience-kit/roles/video_ingestion/deploy/{files/video_ingestion.json => templates/video_ingestion_pcb.json.j2} (73%) create mode 100644 applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 create mode 100644 applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer_pcb.json.j2 rename applications/eis-experience-kit/roles/visualizer/deploy/{files/visualizer.json => templates/visualizer_safety.json.j2} (50%) create mode 100644 applications/eis-experience-kit/scripts/task_log_file.sh diff --git a/applications/eis-experience-kit/deploy_eis_pcb_demo.sh b/applications/eis-experience-kit/deploy_eis_pcb_demo.sh index 08db37fb..b126323e 100755 --- a/applications/eis-experience-kit/deploy_eis_pcb_demo.sh +++ b/applications/eis-experience-kit/deploy_eis_pcb_demo.sh @@ -4,6 +4,7 @@ source scripts/ansible-precheck.sh # check if ansibles are already installed # and ready to use +source scripts/task_log_file.sh ansible-playbook -vv ./eis_sources.yml --inventory inventory.ini diff --git a/applications/eis-experience-kit/eis_pcb_demo.yml b/applications/eis-experience-kit/eis_pcb_demo.yml index aa068cc8..1c01348d 100644 --- a/applications/eis-experience-kit/eis_pcb_demo.yml +++ b/applications/eis-experience-kit/eis_pcb_demo.yml @@ -20,6 +20,7 @@ - role: machine_setup/k8s_master - role: common/deploy - role: camera_stream/deploy + when: camera_stream_pod | default(True) - role: etcd/deploy - role: video_analytics/deploy - role: visualizer/deploy diff --git a/applications/eis-experience-kit/group_vars/all.yml b/applications/eis-experience-kit/group_vars/all.yml index 6a0319d4..7f27b194 100644 --- a/applications/eis-experience-kit/group_vars/all.yml +++ b/applications/eis-experience-kit/group_vars/all.yml @@ -15,3 +15,13 @@ etcd_version: "{{ lookup('env', 'ETCD_VERSION') }}" dev_mode: "{{ lookup('env', 'DEV_MODE') }}" eis_sources_dir: "/opt/eis_repo" + +demo_type: "safety" # set "pcb" for pcp demo or "safety" for Safety Hat demo. +camera_stream_pod: true # enabled if rtsp stream receive from camera stream pod + +rtsp_camera_stream_ip: "ia-camera-stream-service" # set remote rtsp camera IP +rtsp_camera_stream_port: 8554 # set remote rtsp stream port no + +display_host_ip: "192.168.0.1" # update ip for visualizer HOST GUI. +display_no: "1" # update DISPLAY number for visualizer GUI output + diff --git a/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml b/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml index eb66f80a..a05e5cc0 100644 --- a/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml +++ b/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml @@ -6,5 +6,6 @@ docker_image_name: "ia_camera_stream" docker_image_tag: "{{ docker_image_name }}:{{ eis_version }}" -cam_stream_source: "{{ eis_sources_dir }}/VideoIngestion/test_videos/pcb_d2000.avi" +pcb_cam_stream_source: "{{ eis_sources_dir }}/VideoIngestion/test_videos/pcb_d2000.avi" +safety_cam_stream_source: "{{ eis_sources_dir }}/VideoIngestion/test_videos/Safety_Full_Hat_and_Vest.mp4" docker_src_folder: "roles/camera_stream/build/files" diff --git a/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile b/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile index f78c6d0f..0f2b1da8 100644 --- a/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile +++ b/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile @@ -9,4 +9,4 @@ RUN useradd -m vlcuser COPY streamstart.sh /tmp/ RUN chmod +x /tmp/streamstart.sh COPY pcb_d2000.avi /tmp/ -ENTRYPOINT ["/tmp/streamstart.sh"] +COPY Safety_Full_Hat_and_Vest.mp4 /tmp/ diff --git a/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh b/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh old mode 100644 new mode 100755 index 46fbe137..e50d45e8 --- a/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh +++ b/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh @@ -1,6 +1,26 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation -name="/tmp/pcb_d2000.avi"; -sout="#gather:rtp{sdp=rtsp://0.0.0.0:8554/}" + +if [[ $# -ne 2 ]] ; then + echo "No of argument are not correct" + exit 1 +fi + +if [ $1 == "pcb" ] +then + echo "Set pcb demo rtsp stream file" + name="/tmp/pcb_d2000.avi" + +elif [ $1 == "safety" ] +then + echo "Set Safety Hat demo rtsp stream file" + name="/tmp/Safety_Full_Hat_and_Vest.mp4" + +else + echo "Wrong argument pass for rtsp stream demo" + exit 1 + +fi +sout="#gather:rtp{sdp=rtsp://0.0.0.0:$2/}" su vlcuser -c "cvlc -vvv $name --sout '$sout' --loop --sout-keep" diff --git a/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml b/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml index 137522d1..a224c353 100644 --- a/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml @@ -3,9 +3,14 @@ --- -- name: Copy streaming file from EIS repo to folder having Dockerfile +- name: Copy pcb streaming file from EIS repo to folder having Dockerfile copy: - src: "{{ cam_stream_source }}" + src: "{{ pcb_cam_stream_source }}" + dest: "{{ docker_src_folder }}" + +- name: Copy Safety Hat streaming file from EIS repo to folder having Dockerfile + copy: + src: "{{ safety_cam_stream_source }}" dest: "{{ docker_src_folder }}" - name: Build {{ docker_image_name }} docker image diff --git a/applications/eis-experience-kit/roles/camera_stream/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/camera_stream/deploy/defaults/main.yml index 10a2e47e..82785fd8 100644 --- a/applications/eis-experience-kit/roles/camera_stream/deploy/defaults/main.yml +++ b/applications/eis-experience-kit/roles/camera_stream/deploy/defaults/main.yml @@ -6,7 +6,6 @@ app_name: "CameraStream" docker_name: "ia_camera_stream" docker_image: "{{ docker_registry_address }}/{{ docker_name }}:{{ eis_version }}" -camera_stream_port: 8554 helm_release_name: "camera-stream-release" helm_chart_camstream: "{{ helm_charts_location }}/camera-stream" diff --git a/applications/eis-experience-kit/roles/camera_stream/deploy/files/camera-stream/templates/camera-stream-pod.yaml b/applications/eis-experience-kit/roles/camera_stream/deploy/files/camera-stream/templates/camera-stream-pod.yaml index 40a9cf65..199cbd76 100644 --- a/applications/eis-experience-kit/roles/camera_stream/deploy/files/camera-stream/templates/camera-stream-pod.yaml +++ b/applications/eis-experience-kit/roles/camera_stream/deploy/files/camera-stream/templates/camera-stream-pod.yaml @@ -22,6 +22,7 @@ spec: containers: - name: {{ .Chart.Name }} image: {{ .Values.image }} + command: ["/bin/bash", "-c", "trap : TERM INT;/tmp/streamstart.sh {{ .Values.demoType }} {{.Values.cameraStreamPort }}"] ports: - name: camstream-port containerPort: {{ .Values.cameraStreamPort }} diff --git a/applications/eis-experience-kit/roles/camera_stream/deploy/templates/camera-stream-values.yaml.j2 b/applications/eis-experience-kit/roles/camera_stream/deploy/templates/camera-stream-values.yaml.j2 index 27696960..757f9c72 100644 --- a/applications/eis-experience-kit/roles/camera_stream/deploy/templates/camera-stream-values.yaml.j2 +++ b/applications/eis-experience-kit/roles/camera_stream/deploy/templates/camera-stream-values.yaml.j2 @@ -4,4 +4,5 @@ --- image: "{{ docker_registry_address }}/{{ docker_name }}:{{ eis_version }}" -cameraStreamPort: {{ camera_stream_port }} +cameraStreamPort: {{ rtsp_camera_stream_port }} +demoType: "{{ demo_type }}" diff --git a/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml index 803ce60b..5721b24a 100644 --- a/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml @@ -14,9 +14,10 @@ changed_when: true - name: Copy {{ app_name }} config file - copy: - src: "video_analytics.json" - dest: "{{ eis_integ_path }}/configs/" + template: + src: "video_analytics_{{ demo_type }}.json.j2" + dest: "{{ eis_integ_path }}/configs/video_analytics.json" + mode: 0744 - name: Initialize {{ app_name }} config in ETCD command: ./etcd_write.py ./configs/video_analytics.json diff --git a/applications/eis-experience-kit/roles/video_analytics/deploy/files/video_analytics.json b/applications/eis-experience-kit/roles/video_analytics/deploy/templates/video_analytics_pcb.json.j2 similarity index 100% rename from applications/eis-experience-kit/roles/video_analytics/deploy/files/video_analytics.json rename to applications/eis-experience-kit/roles/video_analytics/deploy/templates/video_analytics_pcb.json.j2 diff --git a/applications/eis-experience-kit/roles/video_analytics/deploy/templates/video_analytics_safety.json.j2 b/applications/eis-experience-kit/roles/video_analytics/deploy/templates/video_analytics_safety.json.j2 new file mode 100644 index 00000000..4f9dfd08 --- /dev/null +++ b/applications/eis-experience-kit/roles/video_analytics/deploy/templates/video_analytics_safety.json.j2 @@ -0,0 +1,18 @@ +{ + "/VideoAnalytics/config": { + "encoding": { + "type": "jpeg", + "level": 95 + }, + "queue_size": 10, + "max_jobs": 20, + "max_workers":4, + "udfs": [{ + "name": "safety_gear_demo", + "type": "native", + "model_xml": "common/udfs/native/safety_gear_demo/ref/frozen_inference_graph.xml", + "model_bin": "common/udfs/native/safety_gear_demo/ref/frozen_inference_graph.bin", + "device": "CPU" + }] + } +} diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml b/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml index a7e884f5..74e5412c 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml @@ -54,6 +54,8 @@ spec: value: "{{ .Values.keyPath }}" - name: CONFIGMGR_CACERT value: "{{ .Values.rootCACertPath }}" + - name: no_proxy + value: "{{ .Values.eis_no_proxy }}" volumes: - name: etcd-certs diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml index 6090275d..452dd216 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml @@ -14,9 +14,10 @@ changed_when: true - name: Copy {{ app_name }} config file - copy: - src: "video_ingestion.json" - dest: "{{ eis_integ_path }}/configs/" + template: + src: "video_ingestion_{{ demo_type }}.json.j2" + dest: "{{ eis_integ_path }}/configs/video_ingestion.json" + mode: 0744 - name: Initialize {{ app_name }} config in ETCD command: ./etcd_write.py ./configs/video_ingestion.json diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video_ingestion.json b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 similarity index 73% rename from applications/eis-experience-kit/roles/video_ingestion/deploy/files/video_ingestion.json rename to applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 index 432b73eb..98d39285 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video_ingestion.json +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 @@ -6,7 +6,7 @@ }, "ingestor": { "type": "opencv", - "pipeline": "rtspsrc location=\"rtsp://ia-camera-stream-service:8554/\" latency=100 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! appsink max_buffers=100 drop=true", + "pipeline": "rtspsrc location=\"rtsp://{{ rtsp_camera_stream_ip }}:{{ rtsp_camera_stream_port }}/\" latency=100 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! appsink max_buffers=100 drop=true", "loop_video": "true", "queue_size": 10, "poll_interval": 0.2 diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 new file mode 100644 index 00000000..b002c05a --- /dev/null +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 @@ -0,0 +1,21 @@ +{ + "/VideoIngestion/config": { + "encoding": { + "type": "jpeg", + "level": 95 + }, + "ingestor": { + "type": "opencv", + "pipeline": "rtspsrc location=\"rtsp://{{ rtsp_camera_stream_ip }}:{{ rtsp_camera_stream_port }}/\" latency=100 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! appsink max_buffers=100 drop=true", + "loop_video": "true", + "queue_size": 10, + "poll_interval": 0.2 + }, + "max_jobs": 20, + "max_workers":4, + "udfs": [{ + "name": "dummy", + "type": "native" + }] + } +} diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_values.yaml.j2 b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_values.yaml.j2 index 0b569416..a8041476 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_values.yaml.j2 +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_values.yaml.j2 @@ -13,4 +13,5 @@ eisSecretsPath: "{{ eis_secrets_path }}" certPath: "{{ eis_secrets_path }}/{{ cert_name }}" keyPath: "{{ eis_secrets_path }}/{{ key_name }}" rootCACertPath: "{{ eis_secrets_path }}/{{ rootca_cert_name }}" +eis_no_proxy: "{{ rtsp_camera_stream_ip }}" diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml index 49965ae4..fa05c822 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml +++ b/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml @@ -8,6 +8,8 @@ docker_image: "{{ docker_registry_address }}/{{ docker_name }}:{{ eis_version }} camera_stream_results_port: 65013 app_name: "Visualizer" peer: "client" +display_gui: "true" +save_image: "false" cert_name: "etcd_{{ app_name }}_cert" key_name: "etcd_{{ app_name }}_key" publisher_pod_service: "ia-video-analytics-service" diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/files/visualizer/templates/visualizer-pod.yaml b/applications/eis-experience-kit/roles/visualizer/deploy/files/visualizer/templates/visualizer-pod.yaml index 3c321332..0c230128 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/files/visualizer/templates/visualizer-pod.yaml +++ b/applications/eis-experience-kit/roles/visualizer/deploy/files/visualizer/templates/visualizer-pod.yaml @@ -53,6 +53,10 @@ spec: value: "{{ .Values.keyPath }}" - name: CONFIGMGR_CACERT value: "{{ .Values.rootCACertPath }}" + - name: DISPLAY + value: "{{ .Values.visualizer_display }}" + - name: no_proxy + value: "{{ .Values.eis_no_proxy }}" volumes: - name: etcd-certs projected: diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml index 436dd6c2..a210a7e8 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml @@ -14,9 +14,10 @@ changed_when: true - name: Copy {{ app_name }} config file - copy: - src: "visualizer.json" - dest: "{{ eis_integ_path }}/configs/" + template: + src: "visualizer_{{ demo_type }}.json.j2" + dest: "{{ eis_integ_path }}/configs/visualizer.json" + mode: 0744 - name: Initialize {{ app_name }} config in ETCD command: ./etcd_write.py ./configs/visualizer.json diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer-values.yaml.j2 b/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer-values.yaml.j2 index 7520a2bb..9c56ac5d 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer-values.yaml.j2 +++ b/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer-values.yaml.j2 @@ -14,3 +14,5 @@ eisSecretsPath: "{{ eis_secrets_path }}" certPath: "{{ eis_secrets_path }}/{{ cert_name }}" keyPath: "{{ eis_secrets_path }}/{{ key_name }}" rootCACertPath: "{{ eis_secrets_path }}/{{ rootca_cert_name }}" +visualizer_display: "{{ display_host_ip }}:{{ display_no }}" +eis_no_proxy: "{{ display_host_ip }}" diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer_pcb.json.j2 b/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer_pcb.json.j2 new file mode 100644 index 00000000..f61ff379 --- /dev/null +++ b/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer_pcb.json.j2 @@ -0,0 +1,12 @@ +{ + "/Visualizer/config": { + "display": "{{ display_gui }}", + "save_image": "{{ save_image }}", + "labels" : { + "camera1_stream_results": { + "0": "MISSING", + "1": "SHORT" + } + } + } +} diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/files/visualizer.json b/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer_safety.json.j2 similarity index 50% rename from applications/eis-experience-kit/roles/visualizer/deploy/files/visualizer.json rename to applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer_safety.json.j2 index 85afdc12..cb2d398f 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/files/visualizer.json +++ b/applications/eis-experience-kit/roles/visualizer/deploy/templates/visualizer_safety.json.j2 @@ -1,13 +1,9 @@ { "/Visualizer/config": { - "display": "false", - "save_image": "true", + "display": "{{ display_gui }}", + "save_image": "{{ save_image }}", "labels" : { - "camera1_stream_results": { - "0": "MISSING", - "1": "SHORT" - }, - "camera2_stream_results":{ + "camera1_stream_results":{ "1": "safety_helmet", "2": "safety_jacket", "3": "Safe", diff --git a/applications/eis-experience-kit/scripts/task_log_file.sh b/applications/eis-experience-kit/scripts/task_log_file.sh new file mode 100644 index 00000000..0cc49263 --- /dev/null +++ b/applications/eis-experience-kit/scripts/task_log_file.sh @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2019-2020 Intel Corporation + +currdir=${PWD##*/} +BASE_PATH="$( cd "$(dirname "$0")" ; pwd -P )" + +[ -d ${BASE_PATH}/logs ] || mkdir ${BASE_PATH}/logs +FILENAME="`date +%Y-%m-%d_%H-%M-%S_ansible.log`" + +# Comment out below export to disable console logs saving to files. +export ANSIBLE_LOG_PATH="${BASE_PATH}/logs/${FILENAME}" From d41b1b9b9f002b951413cf0c195deba582f72ac6 Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Mon, 17 Aug 2020 15:41:52 +0530 Subject: [PATCH 23/80] added changes from EIS2.2 to EIS2.3 (#106) --- .../eis-experience-kit/group_vars/all.yml | 4 ++- .../host_vars/localhost.yml | 2 +- .../host_vars/openness_controller.yaml | 2 +- .../camera_stream/build/defaults/main.yml | 2 +- .../camera_stream/build/files/Dockerfile | 3 +- .../camera_stream/build/files/streamstart.sh | 2 +- .../roles/common/build/defaults/main.yml | 2 +- .../roles/common/build/tasks/main.yml | 18 +----------- .../roles/eis_sources/defaults/main.yml | 6 ++-- .../roles/eis_sources/tasks/main.yml | 29 ++++++++++++++----- .../roles/etcd/build/defaults/main.yml | 2 +- .../video_analytics/build/tasks/main.yml | 2 +- .../video_analytics/deploy/defaults/main.yml | 2 +- .../video_ingestion/build/tasks/main.yml | 2 +- .../video_ingestion/deploy/defaults/main.yml | 2 +- .../templates/video-ingestion-pod.yaml | 4 ++- .../templates/video_ingestion_pcb.json.j2 | 3 ++ .../templates/video_ingestion_safety.json.j2 | 3 ++ .../roles/visualizer/build/tasks/main.yml | 2 +- 19 files changed, 51 insertions(+), 41 deletions(-) diff --git a/applications/eis-experience-kit/group_vars/all.yml b/applications/eis-experience-kit/group_vars/all.yml index 7f27b194..f8cdbfe1 100644 --- a/applications/eis-experience-kit/group_vars/all.yml +++ b/applications/eis-experience-kit/group_vars/all.yml @@ -14,7 +14,9 @@ eis_version: "{{ lookup('env', 'EIS_VERSION') }}" etcd_version: "{{ lookup('env', 'ETCD_VERSION') }}" dev_mode: "{{ lookup('env', 'DEV_MODE') }}" -eis_sources_dir: "/opt/eis_repo" +eis_repo_dir: "/opt/eis_repo" +eis_sources_dir: "/opt/eis_repo/IEdgeInsights" + demo_type: "safety" # set "pcb" for pcp demo or "safety" for Safety Hat demo. camera_stream_pod: true # enabled if rtsp stream receive from camera stream pod diff --git a/applications/eis-experience-kit/host_vars/localhost.yml b/applications/eis-experience-kit/host_vars/localhost.yml index 7939c347..faf92534 100644 --- a/applications/eis-experience-kit/host_vars/localhost.yml +++ b/applications/eis-experience-kit/host_vars/localhost.yml @@ -23,6 +23,6 @@ proxy_yum_url: "http://proxy.example.org:3128" eis_source: "gitclone" # set string as "gitclone" or "release" release_package_path: "" # provide release package archive path when eis_source set to "release" -eis_env_file: "{{ eis_sources_dir }}/docker_setup/.env" +eis_env_file: "{{ eis_sources_dir }}/build/.env" os_yum_base_packages: "jq,vim-common,curl,yum-utils,python2-pip,wget,git,python-devel,moreutils" diff --git a/applications/eis-experience-kit/host_vars/openness_controller.yaml b/applications/eis-experience-kit/host_vars/openness_controller.yaml index 7286f822..1df76fdb 100644 --- a/applications/eis-experience-kit/host_vars/openness_controller.yaml +++ b/applications/eis-experience-kit/host_vars/openness_controller.yaml @@ -21,7 +21,7 @@ k8s_eis_namespace: "eis" san_ca: "DNS:{{ pod_name }}-service" cert_days: 3650 -eis_openssl_cnf: "{{ eis_sources_dir }}/docker_setup/provision/config/openssl.cnf" +eis_openssl_cnf: "{{ eis_sources_dir }}/build/provision/config/openssl.cnf" rootca_cert_name: "ca_certificate.pem" rootca_key_name: "ca_key.pem" diff --git a/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml b/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml index a05e5cc0..fa56166c 100644 --- a/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml +++ b/applications/eis-experience-kit/roles/camera_stream/build/defaults/main.yml @@ -7,5 +7,5 @@ docker_image_name: "ia_camera_stream" docker_image_tag: "{{ docker_image_name }}:{{ eis_version }}" pcb_cam_stream_source: "{{ eis_sources_dir }}/VideoIngestion/test_videos/pcb_d2000.avi" -safety_cam_stream_source: "{{ eis_sources_dir }}/VideoIngestion/test_videos/Safety_Full_Hat_and_Vest.mp4" +safety_cam_stream_source: "{{ eis_sources_dir }}/VideoIngestion/test_videos/Safety_Full_Hat_and_Vest.avi" docker_src_folder: "roles/camera_stream/build/files" diff --git a/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile b/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile index 0f2b1da8..c87fc6ac 100644 --- a/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile +++ b/applications/eis-experience-kit/roles/camera_stream/build/files/Dockerfile @@ -9,4 +9,5 @@ RUN useradd -m vlcuser COPY streamstart.sh /tmp/ RUN chmod +x /tmp/streamstart.sh COPY pcb_d2000.avi /tmp/ -COPY Safety_Full_Hat_and_Vest.mp4 /tmp/ +COPY Safety_Full_Hat_and_Vest.avi /tmp/ + diff --git a/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh b/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh index e50d45e8..0a320f52 100755 --- a/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh +++ b/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh @@ -15,7 +15,7 @@ then elif [ $1 == "safety" ] then echo "Set Safety Hat demo rtsp stream file" - name="/tmp/Safety_Full_Hat_and_Vest.mp4" + name="/tmp/Safety_Full_Hat_and_Vest.avi" else echo "Wrong argument pass for rtsp stream demo" diff --git a/applications/eis-experience-kit/roles/common/build/defaults/main.yml b/applications/eis-experience-kit/roles/common/build/defaults/main.yml index 8cbb02d8..96cc67f7 100644 --- a/applications/eis-experience-kit/roles/common/build/defaults/main.yml +++ b/applications/eis-experience-kit/roles/common/build/defaults/main.yml @@ -5,5 +5,5 @@ openvino_download_url: "http://registrationcenter-download.intel.com/akdlm/irc_nas/16612/l_openvino_toolkit_p_2020.2.120.tgz" -eis_base_images_names: "ia_eisbase ia_common ia_openvino_base" +eis_base_images_names: "ia_eisbase ia_common ia_video_common ia_openvino_base" diff --git a/applications/eis-experience-kit/roles/common/build/tasks/main.yml b/applications/eis-experience-kit/roles/common/build/tasks/main.yml index 6a80b7c5..08e0a451 100644 --- a/applications/eis-experience-kit/roles/common/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/common/build/tasks/main.yml @@ -3,25 +3,9 @@ --- -- name: Get Openvino archive name - shell: var={{ openvino_download_url }}; var=${var##*/}; echo ${var%.tgz} - register: output - changed_when: false - -- name: Check if openvino is already downloaded - find: - paths: "{{ eis_sources_dir }}/common/openvino/{{ output.stdout }}" - register: openvino_detected - -- name: Download and unarchive OpenVino - unarchive: - src: "{{ openvino_download_url }}" - dest: "{{ eis_sources_dir }}/common/openvino" - remote_src: yes - when: openvino_detected.matched == 0 - name: Build EIS common docker images shell: "docker-compose build {{ eis_base_images_names }}" args: - chdir: "{{ eis_sources_dir }}/docker_setup" + chdir: "{{ eis_sources_dir }}/build" changed_when: true diff --git a/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml b/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml index 3ab6766b..2b7ef59d 100644 --- a/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml +++ b/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml @@ -2,7 +2,7 @@ # Copyright (c) 2020 Intel Corporation --- - -eis_repo_url: "ssh://git@gitlab.devtools.intel.com:29418/Indu/IEdgeInsights/IEdgeInsights.git" -eis_repo_branch: EIS_OpenNESS_Integration +repo_tool: "https://storage.googleapis.com/git-repo-downloads/repo" +eis_repo_url: "ssh://git@gitlab.devtools.intel.com:29418/Indu/IEdgeInsights/eis-manifests" +eis_repo_branch: "refs/tags/v2.3-Alpha-RC1" diff --git a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml index 3906a48d..9ea5ec14 100644 --- a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml +++ b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml @@ -5,16 +5,31 @@ - name: create directory if not exist file: - path: "{{ eis_sources_dir }}" + path: "{{ eis_repo_dir }}" state: directory mode: '0755' -- name: EIS code repository using git clone - git: - repo: '{{ eis_repo_url }}' - dest: '{{ eis_sources_dir }}' - version: "{{ eis_repo_branch }}" - force: yes +- name: download repo tool for multi-repo codebase checkout. + shell: "{{ item }}" + with_items: + - curl {{ repo_tool }} > repo + - mv repo -f /bin/ + - chmod 755 /bin/repo + changed_when: true + args: + chdir: "{{ eis_repo_dir }}" + changed_when: true + when: eis_source == "gitclone" + +- name: EIS codebase checkout using repo tool + shell : "{{ item }}" + with_items: + - repo init -u "{{ eis_repo_url }}" -b {{ eis_repo_branch }} + - cd .repo/manifests; repo init -m video.xml; repo sync + - cd IEdgeInsights/build; pip3 install -r requirements.txt; python3 eis_builder.py + args: + chdir: "{{ eis_repo_dir }}" + changed_when: true when: eis_source == "gitclone" - name: EIS code repository from baseline release diff --git a/applications/eis-experience-kit/roles/etcd/build/defaults/main.yml b/applications/eis-experience-kit/roles/etcd/build/defaults/main.yml index 742b789e..d4586dae 100644 --- a/applications/eis-experience-kit/roles/etcd/build/defaults/main.yml +++ b/applications/eis-experience-kit/roles/etcd/build/defaults/main.yml @@ -6,5 +6,5 @@ docker_image_name: ia_etcd docker_image_tag: "{{ docker_image_name }}:{{ eis_version }}" -docker_compose_file: "{{ eis_sources_dir }}/docker_setup/provision/dep/docker-compose-provision.yml" +docker_compose_file: "{{ eis_sources_dir }}/build/provision/dep/docker-compose-provision.yml" diff --git a/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml b/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml index 18d568e5..2dcf941a 100644 --- a/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml @@ -6,7 +6,7 @@ - name: Build {{ docker_image_name }} docker image shell: docker-compose build {{ docker_image_name }} args: - chdir: "{{ eis_sources_dir }}/docker_setup" + chdir: "{{ eis_sources_dir }}/build" changed_when: true - name: Add tag to the {{ docker_image_name }} image diff --git a/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml index 933f1c90..04c2cd0f 100644 --- a/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml +++ b/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml @@ -7,7 +7,7 @@ docker_name: "ia_video_analytics" docker_image: "{{ docker_registry_address }}/{{ docker_name }}:{{ eis_version }}" camera_stream_port: 9000 camera_stream_results_port: 65013 -clients_list: "Visualizer" +clients_list: "Visualizer,WebVisualizer" app_name: "VideoAnalytics" peer: "client" cert_name: "etcd_{{ app_name }}_cert" diff --git a/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml b/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml index 18d568e5..2dcf941a 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml @@ -6,7 +6,7 @@ - name: Build {{ docker_image_name }} docker image shell: docker-compose build {{ docker_image_name }} args: - chdir: "{{ eis_sources_dir }}/docker_setup" + chdir: "{{ eis_sources_dir }}/build" changed_when: true - name: Add tag to the {{ docker_image_name }} image diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml index 495d07a1..a8c1ec8e 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml @@ -6,7 +6,7 @@ docker_name: "ia_video_ingestion" docker_image: "{{ docker_registry_address }}/{{ docker_name }}:{{ eis_version }}" camera_stream_port: 9000 -clients_list: "VideoAnalytics,Visualizer" +clients_list: "VideoAnalytics,Visualizer,WebVisualizer" app_name: "VideoIngestion" peer: "client" cert_name: "etcd_{{ app_name }}_cert" diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml b/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml index 74e5412c..5fea59dc 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/files/video-ingestion/templates/video-ingestion-pod.yaml @@ -43,7 +43,7 @@ spec: - name: CertType value: "zmq" - name: ZMQ_RECV_HWM - value: "1000" + value: "50" - name: PubTopics value: "camera1_stream" - name: camera1_stream_cfg @@ -56,6 +56,8 @@ spec: value: "{{ .Values.rootCACertPath }}" - name: no_proxy value: "{{ .Values.eis_no_proxy }}" + - name: Server + value: "zmq_tcp,127.0.0.1:66013" volumes: - name: etcd-certs diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 index 98d39285..d1188529 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 @@ -13,6 +13,9 @@ }, "max_jobs": 20, "max_workers":4, + "sw_trigger": { + "init_state": "running" + }, "udfs": [{ "name": "pcb.pcb_filter", "type": "python", diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 index b002c05a..feaf8879 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 @@ -13,6 +13,9 @@ }, "max_jobs": 20, "max_workers":4, + "sw_trigger": { + "init_state": "running" + }, "udfs": [{ "name": "dummy", "type": "native" diff --git a/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml b/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml index 18d568e5..2dcf941a 100644 --- a/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml @@ -6,7 +6,7 @@ - name: Build {{ docker_image_name }} docker image shell: docker-compose build {{ docker_image_name }} args: - chdir: "{{ eis_sources_dir }}/docker_setup" + chdir: "{{ eis_sources_dir }}/build" changed_when: true - name: Add tag to the {{ docker_image_name }} image From be04b162204ac86ff36aa4c24463a98160d91dad Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Thu, 20 Aug 2020 09:57:25 +0200 Subject: [PATCH 24/80] CERA-1581: EIS Pods Redeployment (#107) * CERA-1581: EIS Pods Redeployment * Fixing issues after testing cleanup and deployment * Remove empty functions from eis_integ.py and move etcd cleanup to the end of cleanup playbook. --- .../eis_pcb_demo_cleanup.yml | 12 ++-- .../host_vars/openness_controller.yaml | 3 +- .../common/deploy/tasks/gen_zmq_keys.yaml | 12 ++++ .../etcd/deploy/files/eis_integ/eis_integ.py | 64 +++++++++++++++---- .../deploy/files/eis_integ/etcd_remove.py | 40 ++++++++++++ .../deploy/files/eis_integ/etcd_zmq_key.py | 16 +++-- .../video_analytics/deploy/defaults/main.yml | 2 + .../video_analytics/deploy/tasks/cleanup.yaml | 22 +++++++ .../video_analytics/deploy/tasks/main.yml | 1 + .../video_ingestion/deploy/defaults/main.yml | 2 + .../video_ingestion/deploy/tasks/cleanup.yaml | 22 +++++++ .../video_ingestion/deploy/tasks/main.yml | 1 + .../roles/visualizer/deploy/defaults/main.yml | 2 + .../visualizer/deploy/tasks/cleanup.yaml | 22 +++++++ .../roles/visualizer/deploy/tasks/main.yml | 1 + 15 files changed, 197 insertions(+), 25 deletions(-) create mode 100755 applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_remove.py diff --git a/applications/eis-experience-kit/eis_pcb_demo_cleanup.yml b/applications/eis-experience-kit/eis_pcb_demo_cleanup.yml index 0a804cba..6b4b2c78 100644 --- a/applications/eis-experience-kit/eis_pcb_demo_cleanup.yml +++ b/applications/eis-experience-kit/eis_pcb_demo_cleanup.yml @@ -49,12 +49,6 @@ - hosts: controller_group tasks: - - name: Load ETCD vars - include_vars: ./roles/etcd/deploy/defaults/main.yml - - - name: Cleanup ETCD - include_tasks: ./roles/etcd/deploy/tasks/cleanup.yaml - - name: Load CameraStream vars include_vars: ./roles/camera_stream/deploy/defaults/main.yml @@ -79,5 +73,11 @@ - name: Cleanup Visualizer include_tasks: ./roles/visualizer/deploy/tasks/cleanup.yaml + - name: Load ETCD vars + include_vars: ./roles/etcd/deploy/defaults/main.yml + + - name: Cleanup ETCD + include_tasks: ./roles/etcd/deploy/tasks/cleanup.yaml + - name: Cleanup Common include_tasks: ./roles/common/deploy/tasks/cleanup.yaml diff --git a/applications/eis-experience-kit/host_vars/openness_controller.yaml b/applications/eis-experience-kit/host_vars/openness_controller.yaml index 1df76fdb..80013cb2 100644 --- a/applications/eis-experience-kit/host_vars/openness_controller.yaml +++ b/applications/eis-experience-kit/host_vars/openness_controller.yaml @@ -10,7 +10,8 @@ eis_install_path: "/opt/eis" helm_charts_location: "{{ eis_install_path }}/charts" eis_integ_path: "{{ eis_install_path }}/etcd/eis_integ" etcd_certs_location: "{{ eis_install_path }}/etcd/Certificates" - +zmq_keys_gen_force: false +del_zmq_keys: false os_yum_base_packages: "python3,python3-pip" ca_default_bits: "{{ lookup('env', 'SSL_KEY_LENGTH') }}" diff --git a/applications/eis-experience-kit/roles/common/deploy/tasks/gen_zmq_keys.yaml b/applications/eis-experience-kit/roles/common/deploy/tasks/gen_zmq_keys.yaml index 563cbe19..a3c2fcb4 100644 --- a/applications/eis-experience-kit/roles/common/deploy/tasks/gen_zmq_keys.yaml +++ b/applications/eis-experience-kit/roles/common/deploy/tasks/gen_zmq_keys.yaml @@ -12,3 +12,15 @@ args: chdir: "{{ eis_integ_path }}" changed_when: true + when: zmq_keys_gen_force == false + +- name: Force generate ZMQ keys for {{ app_name }} + command: /usr/bin/python3 etcd_zmq_key.py {{ app_name }} --force + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + args: + chdir: "{{ eis_integ_path }}" + changed_when: true + when: zmq_keys_gen_force == true diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py index e9100cbe..4488a9be 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py @@ -70,6 +70,30 @@ def put_zmqkeys(appname): except Exception: logging.error("Error putting Etcd private key for {}".format(appname)) +def check_zmqkeys(appname): + """ Check if public/private key already exist in etcd for given app. Returns + true if keys exist, otherwise false + + :param appname: App Name + :type file: String + """ + try: + app_pub_key = subprocess.check_output(["etcdctl", "get", "--", "/Publickeys/" + appname], stderr=subprocess.STDOUT).decode('utf-8') + except subprocess.CalledProcessError as e: + logging.error("Error returned while getting the public key for {} app from etcd: {}".format(appname, e)) + raise EisIntegError(CODES.EXT_CMD_ERROR) + + try: + app_priv_key = subprocess.check_output(["etcdctl", "get", "--", "/" + appname + "/private_key"], stderr=subprocess.STDOUT).decode('utf-8') + except subprocess.CalledProcessError as e: + logging.error("Error returned while getting the private key for {} app from etcd: {}".format(appname, e)) + raise EisIntegError(CODES.EXT_CMD_ERROR) + + if not app_priv_key or not app_pub_key: + return False + else: + return True + def enable_etcd_auth(password): """ Enable Auth for etcd and Create root user with root role """ try: @@ -101,7 +125,6 @@ def etcd_put_json(json_data): etcd_put(key, bytes(json.dumps(value, indent=4).encode())) logging.info("Value for the {} key has been added to the etcd.".format(key)) - def create_etcd_users(appname): """ Create etcd user and role for given app. Allow Read only access only to appname, global and publickeys directory @@ -138,21 +161,24 @@ def read_config(client): logging.info("Read the configuration from etcd") subprocess.run(["etcdctl", "get", client, "--prefix"]) +def etcd_remove_key(key): + """ Remove key from etcd -def get_etcd_users(): - """ Read all the member info from etcd. """ - - -def grant_user_privilege(): - """ Give access right to the specific user. """ + :param key: key will be removed from etcd + """ + try: + subprocess.check_output(["etcdctl", "del", "--", key], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + logging.error("Error returned while removing the {} key from etcd: {}".format(key, e)) + raise EisIntegError(CODES.EXT_CMD_ERROR) def remove_user_privilege(name): """Removes role - :param name: Rolre name + :param name: Role name :type file: String """ try: - subprocess.check_output(["etcdctl", "role", "remove", name], stderr=subprocess.STDOUT) + subprocess.check_output(["etcdctl", "role", "delete", name], stderr=subprocess.STDOUT) logging.info("Role {} has been removed.".format(name)) except subprocess.CalledProcessError as e: logging.error("Error returned while removing the {} role: {}".format(name, e)) @@ -170,16 +196,26 @@ def remove_user(name): logging.error("Error returned while removing the {} user: {}".format(name, e)) raise EisIntegError(CODES.EXT_CMD_ERROR) -def add_user_role(): - """ Add role to the specific user. """ - - def deploy_eis_app(): """ Deploy another eis application, e.g. second video stream. """ +def remove_zmq_keys(appname): + """ Remove ZMQ private/public keys pair for app """ + etcd_remove_key("/" + appname + "/private_key") + etcd_remove_key("/Publickeys/" + appname) + +def remove_app_config(appname, del_keys): + """ Remove EIS application config including ZMQ keys """ + if del_keys: + remove_zmq_keys(appname) + + etcd_remove_key("/" + appname + "/config") -def remove_eis_application(): +def remove_eis_app(appname, del_keys=False): """ Remove existing eis application. """ + remove_app_config(appname, del_keys) + remove_user_privilege(appname) + remove_user(appname) def remove_eis_key(key): """ Remove existing eis key. """ diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_remove.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_remove.py new file mode 100755 index 00000000..5fbb9d06 --- /dev/null +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_remove.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +""" ETCD Data Write Tool """ +import argparse +import logging +import os +import sys +import traceback +import eis_integ + +def parse_arguments(_cli_args): + """ Parse argument passed to function """ + parser = argparse.ArgumentParser(description= + "Removing app config from etcd.") + parser.add_argument("app", help= + "Name of the application which config should be removed from etcd.") + parser.add_argument("--delete_keys", action='store_true', help="Remove all app entries in etcd including ZMQ keys") + return parser.parse_args() + +def main(args): + eis_integ.init_logger() + + os.environ["ETCDCTL_ENDPOINTS"] = "https://" + eis_integ.extract_etcd_endpoint() + + eis_integ.check_path_variable("ETCDCTL_CACERT", os.environ.get("ETCDCTL_CACERT")) + eis_integ.check_path_variable("ETCDCTL_CERT", os.environ.get("ETCDCTL_CERT")) + eis_integ.check_path_variable("ETCDCTL_KEY", os.environ.get("ETCDCTL_KEY")) + + print("Remove all {} app related content from etcd database".format(args.arg)) + eis_integ.remove_eis_app(args.app, args.delete_keys) + return eis_integ.CODES.NO_ERROR + +if __name__ == '__main__': + try: + sys.exit(main(parse_arguments(sys.argv[1:])).value) + except eis_integ.EisIntegError as exception: + logging.error("Error while deleting entries from ETCD database: {}".format(exception)) + sys.exit(exception.code.value) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py index 3117b462..d645193e 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py @@ -15,11 +15,11 @@ def parse_arguments(_cli_args): """ Parse argument passed to function """ parser = argparse.ArgumentParser(description="Specify Application name") parser.add_argument("app", help= "Name of the client application") + parser.add_argument("--force", action='store_true', help="Force putting new generated ZMQ keys without checking" + + "if they already exist") return parser.parse_args() def main(args): - """ Calls the eis_integ.etcd_put_json function to add the contents of the json file - to the etcd database """ eis_integ.init_logger() os.environ["ETCDCTL_ENDPOINTS"] = "https://" + \ @@ -29,9 +29,17 @@ def main(args): eis_integ.check_path_variable("ETCDCTL_CERT", os.environ.get("ETCDCTL_CERT")) eis_integ.check_path_variable("ETCDCTL_KEY", os.environ.get("ETCDCTL_KEY")) - logging.info("Generate ZMQ pair keys for {} and put them to the etcd database".format(args.app)) + skip_gen = False + if not args.force: + logging.info("Check if ZMQ key pair for {} app already exists".format(args.app)) + skip_gen = eis_integ.check_zmqkeys(args.app) + + if not skip_gen: + logging.info("Generate ZMQ pair keys for {} and put them to the etcd database".format(args.app)) + eis_integ.put_zmqkeys(args.app) + else: + logging.info("ZMQ pair keys generation skipped for {} app".format(args.app)) - eis_integ.put_zmqkeys(args.app) return eis_integ.CODES.NO_ERROR diff --git a/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml index 04c2cd0f..aa656fd6 100644 --- a/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml +++ b/applications/eis-experience-kit/roles/video_analytics/deploy/defaults/main.yml @@ -15,3 +15,5 @@ key_name: "etcd_{{ app_name }}_key" publisher_pod_service: "ia-video-ingestion-service" helm_release_name: "video-analytics-release" helm_chart_analytics: "{{ helm_charts_location }}/video-analytics" +zmq_keys_gen_force: false +del_zmq_keys: false diff --git a/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/cleanup.yaml b/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/cleanup.yaml index 0e3d7750..ea4f4bac 100644 --- a/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/cleanup.yaml +++ b/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/cleanup.yaml @@ -13,6 +13,28 @@ path: "{{ helm_chart_analytics }}" state: absent +- name: Remove {{ app_name }} ETCD config, user and role + command: "./etcd_remove.py {{ app_name }}" + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + when: del_zmq_keys == false + +- name: Remove {{ app_name }} ETCD config, user and role including ZMQ keys + command: "./etcd_remove.py {{ app_name }} --delete-keys" + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + when: del_zmq_keys == true + - name: Remove {{ app_name }} ETCD certificates file: path: "{{ etcd_certs_location }}/{{ app_name }}" diff --git a/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml index 5721b24a..f4ac8001 100644 --- a/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_analytics/deploy/tasks/main.yml @@ -12,6 +12,7 @@ ETCDCTL_CERT: "{{ root_client_cert }}" ETCDCTL_KEY: "{{ root_client_key }}" changed_when: true + ignore_errors: true - name: Copy {{ app_name }} config file template: diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml index a8c1ec8e..d4780b63 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/defaults/main.yml @@ -13,3 +13,5 @@ cert_name: "etcd_{{ app_name }}_cert" key_name: "etcd_{{ app_name }}_key" helm_release_name: "video-ingestion-release" helm_chart_ingestion: "{{ helm_charts_location }}/video-ingestion" +zmq_keys_gen_force: false +del_zmq_keys: false diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/cleanup.yaml b/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/cleanup.yaml index 3da76bf9..7fde65e8 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/cleanup.yaml +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/cleanup.yaml @@ -13,6 +13,28 @@ path: "{{ helm_chart_ingestion }}" state: absent +- name: Remove {{ app_name }} ETCD config, user and role + command: "./etcd_remove.py {{ app_name }}" + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + when: del_zmq_keys == false + +- name: Remove {{ app_name }} ETCD config, user and role including ZMQ keys + command: "./etcd_remove.py {{ app_name }} --delete-keys" + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + when: del_zmq_keys == true + - name: Remove {{ app_name }} ETCD certificates file: path: "{{ etcd_certs_location }}/{{ app_name }}" diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml index 452dd216..36876a3a 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/tasks/main.yml @@ -12,6 +12,7 @@ ETCDCTL_CERT: "{{ root_client_cert }}" ETCDCTL_KEY: "{{ root_client_key }}" changed_when: true + ignore_errors: true - name: Copy {{ app_name }} config file template: diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml index fa05c822..98a38ebe 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml +++ b/applications/eis-experience-kit/roles/visualizer/deploy/defaults/main.yml @@ -15,3 +15,5 @@ key_name: "etcd_{{ app_name }}_key" publisher_pod_service: "ia-video-analytics-service" helm_release_name: "visualizer-release" helm_chart_visualizer: "{{ helm_charts_location }}/visualizer" +zmq_keys_gen_force: false +del_zmq_keys: false diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/tasks/cleanup.yaml b/applications/eis-experience-kit/roles/visualizer/deploy/tasks/cleanup.yaml index 89d86a85..fa6983da 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/tasks/cleanup.yaml +++ b/applications/eis-experience-kit/roles/visualizer/deploy/tasks/cleanup.yaml @@ -14,6 +14,28 @@ state: absent ignore_errors: yes +- name: Remove {{ app_name }} ETCD config, user and role + command: "./etcd_remove.py {{ app_name }}" + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + when: del_zmq_keys == false + +- name: Remove {{ app_name }} ETCD config, user and role including ZMQ keys + command: "./etcd_remove.py {{ app_name }} --delete-keys" + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + when: del_zmq_keys == true + - name: Remove {{ app_name }} ETCD certificates file: path: "{{ etcd_certs_location }}/{{ app_name }}" diff --git a/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml index a210a7e8..9ed7a8fc 100644 --- a/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/visualizer/deploy/tasks/main.yml @@ -12,6 +12,7 @@ ETCDCTL_CERT: "{{ root_client_cert }}" ETCDCTL_KEY: "{{ root_client_key }}" changed_when: true + ignore_errors: true - name: Copy {{ app_name }} config file template: From 6603f9ea6364033e1b4da7f661213c0e4d49929c Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Fri, 21 Aug 2020 15:57:25 +0800 Subject: [PATCH 25/80] Optimize the OpenNESS with K8s Benchmark APP for CPU and HDDL-R mode and Test Automation for openvino benchmark --- .../openvino/benchmark/benchmark_job.yaml | 19 ++++++++++++++++++- .../openvino/benchmark/do_benchmark.sh | 4 ++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/applications/openvino/benchmark/benchmark_job.yaml b/applications/openvino/benchmark/benchmark_job.yaml index 03d410aa..fd778a70 100644 --- a/applications/openvino/benchmark/benchmark_job.yaml +++ b/applications/openvino/benchmark/benchmark_job.yaml @@ -1,6 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation + +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm-benchmark +data: + env.txt: | + NIREQ=4 + NITER=10 + # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable + TARGET_DEVICE=CPU +--- apiVersion: batch/v1 kind: Job metadata: @@ -20,11 +32,13 @@ spec: image: docker imagePullPolicy: IfNotPresent command: [ "/bin/sh" ] - args: ["-c", "/usr/local/bin/docker run --rm --device-cgroup-rule='c 10:* rmw' --device-cgroup-rule='c 89:* rmw' --device-cgroup-rule='c 189:* rmw' --device-cgroup-rule='c 180:* rmw' -v /dev:/dev -v /var/tmp:/var/tmp openvino-benchmark:1.0 /do_benchmark.sh"] + args: ["-c", "/usr/local/bin/docker run --rm --device-cgroup-rule='c 10:* rmw' --device-cgroup-rule='c 89:* rmw' --device-cgroup-rule='c 189:* rmw' --device-cgroup-rule='c 180:* rmw' -v /dev:/dev -v /var/tmp:/var/tmp --env-file /dockerenv/env.txt openvino-benchmark:1.0 /do_benchmark.sh"] securityContext: readOnlyRootFilesystem: false privileged: true volumeMounts: + - name: dockerenv + mountPath: /dockerenv - name: dockersock mountPath: /var/run/docker.sock - name: usrsrc @@ -37,6 +51,9 @@ spec: - name: vartmp mountPath: /var/tmp volumes: + - name: dockerenv + configMap: + name: cm-benchmark - name: dockersock hostPath: path: /var/run/docker.sock diff --git a/applications/openvino/benchmark/do_benchmark.sh b/applications/openvino/benchmark/do_benchmark.sh index ad3588e5..cf27b117 100755 --- a/applications/openvino/benchmark/do_benchmark.sh +++ b/applications/openvino/benchmark/do_benchmark.sh @@ -5,7 +5,7 @@ source /opt/intel/openvino/bin/setupvars.sh cd /opt/intel/openvino/deployment_tools/demo -./demo_squeezenet_download_convert_run.sh -d HDDL +./demo_squeezenet_download_convert_run.sh -d ${TARGET_DEVICE} -/root/inference_engine_samples_build/intel64/Release/benchmark_app -i /opt/intel/openvino/deployment_tools/demo/car.png -m /root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml -d HDDL -nireq 32 -niter 9999999 -api async +/root/inference_engine_samples_build/intel64/Release/benchmark_app -i /opt/intel/openvino/deployment_tools/demo/car.png -m /root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api async From f2d1623d207f65e4d6c3baca037a7895356c6b9d Mon Sep 17 00:00:00 2001 From: relhwigi Date: Fri, 21 Aug 2020 13:52:49 +0100 Subject: [PATCH 26/80] initial verification and step 1 of refactoring test_verification.py --- network-functions/xran/test_verification.py | 92 +++++++++------------ 1 file changed, 40 insertions(+), 52 deletions(-) diff --git a/network-functions/xran/test_verification.py b/network-functions/xran/test_verification.py index 68d72209..bc3f25a7 100755 --- a/network-functions/xran/test_verification.py +++ b/network-functions/xran/test_verification.py @@ -142,37 +142,39 @@ def get_re_map(nrb, direction): return prb_map -def compare_resuts(rantech, cat, m_u, xran_path, direction): +def compare_results(rantech, cat, m_u, xran_path, direction, context): """method to compare results""" res = 0 re_map = [] if rantech == 1: if m_u == 0: - n_dirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nDLBandwidth))] - n_uirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nULBandwidth))] + n_dirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(context["nDLBandwidth"]))] + n_uirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(context["nULBandwidth"]))] else: print("Incorrect arguments\n") res = -1 return res elif rantech == 0: if m_u < 3: - n_dirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nDLBandwidth))] - n_uirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(nULBandwidth))] + n_dirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(context["nDLBandwidth"]))] + n_uirb = N_NUM_RBS_PER_SYM_F1[m_u][N_RCH_BW_OPTIONS.get(str(context["nULBandwidth"]))] elif (m_u >= 2) & (m_u <= 3): - n_dirb = N_NUM_RBS_PER_SYM_F2[m_u - 2][N_RCH_BW_OPTIONS_MU2AND3.get(str(nDLBandwidth))] - n_uirb = N_NUM_RBS_PER_SYM_F2[m_u - 2][N_RCH_BW_OPTIONS_MU2AND3.get(str(nULBandwidth))] + n_dirb = N_NUM_RBS_PER_SYM_F2[m_u - 2][N_RCH_BW_OPTIONS_MU2AND3.get( + str(context["nDLBandwidth"]))] + n_uirb = N_NUM_RBS_PER_SYM_F2[m_u - 2][N_RCH_BW_OPTIONS_MU2AND3.get( + str(context["nULBandwidth"]))] print(n_dirb, n_uirb) else: print("Incorrect arguments\n") res = -1 return res - if 'compression' in globals(): + if "compression" in context: comp = 'compression' else: comp = 0 - if 'srsEanble' in globals(): + if "srsEanble" in context: srs_enb = 'srsEanble' else: srs_enb = 0 @@ -184,39 +186,39 @@ def compare_resuts(rantech, cat, m_u, xran_path, direction): # return res #get slot config - if nFrameDuplexType == 1: + if context["nFrameDuplexType"] == 1: slot_config = [] - for i in range(nTddPeriod): + for i in range(context["nTddPeriod"]): if i == 0: - slot_config.insert(i, sslot_config0) + slot_config.insert(i, context["sslot_config0"]) elif i == 1: - slot_config.insert(i, sslot_config1) + slot_config.insert(i, context["sslot_config1"]) elif i == 2: - slot_config.insert(i, sslot_config2) + slot_config.insert(i, context["sslot_config2"]) elif i == 3: - slot_config.insert(i, sslot_config3) + slot_config.insert(i, context["sslot_config3"]) elif i == 4: - slot_config.insert(i, sslot_config4) + slot_config.insert(i, context["sslot_config4"]) elif i == 5: - slot_config.insert(i, sslot_config5) + slot_config.insert(i, context["sslot_config5"]) elif i == 6: - slot_config.insert(i, sslot_config6) + slot_config.insert(i, context["sslot_config6"]) elif i == 7: - slot_config.insert(i, sslot_config7) + slot_config.insert(i, context["sslot_config7"]) elif i == 8: - slot_config.insert(i, sslot_config8) + slot_config.insert(i, context["sslot_config8"]) elif i == 9: - slot_config.insert(i, sslot_config9) + slot_config.insert(i, context["sslot_config9"]) else: raise Exception('i should not exceed nTddPeriod %d. The value of i was: {}' - .format(nTddPeriod, i)) + .format(context["nTddPeriod"], i)) #print(SlotConfig, type(sSlotConfig0)) try: if (direction == 1) & (cat == 1): #UL - flow_id = ccNum*antNumUL + flow_id = context["ccNum"]*context["antNumUL"] else: - flow_id = ccNum*antNum + flow_id = context["ccNum"]*context["antNum"] if direction == 0: re_map = get_re_map(n_dirb, direction) @@ -279,20 +281,20 @@ def compare_resuts(rantech, cat, m_u, xran_path, direction): file_tst.close() file_ref.close() - print(numSlots) + print(context["numSlots"]) - for slot_idx in range(0, numSlots): + for slot_idx in range(0, context["numSlots"]): for sym_idx in range(0, 14): - if nFrameDuplexType == 1: + if context["nFrameDuplexType"] == 1: #skip sym if TDD if direction == 0: #DL - sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + sym_dir = slot_config[slot_idx%context["nTddPeriod"]][sym_idx] if sym_dir != 0: continue elif direction == 1: #UL - sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + sym_dir = slot_config[slot_idx%context["nTddPeriod"]][sym_idx] if sym_dir != 1: continue @@ -349,9 +351,9 @@ def compare_resuts(rantech, cat, m_u, xran_path, direction): print("compare results: {} [compression {}]\n".format('SRS', comp)) #srs - symb_mask = srsSym + symb_mask = context["srsSym"] try: - flow_id = ccNum*antElmTRx + flow_id = context["ccNum"]*context["antElmTRx"] for i in range(0, flow_id): #read ref and test files tst = [] @@ -400,22 +402,22 @@ def compare_resuts(rantech, cat, m_u, xran_path, direction): file_tst.close() file_ref.close() - print(numSlots) + print(context["numSlots"]) - for slot_idx in range(0, numSlots): + for slot_idx in range(0, context["numSlots"]): for sym_idx in range(0, 14): if symb_mask & (1 << sym_idx): print("SRS check sym ", sym_idx) - if nFrameDuplexType == 1: + if context["nFrameDuplexType"] == 1: #skip sym if TDD if direction == 0: #DL - sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + sym_dir = slot_config[slot_idx%context["nTddPeriod"]][sym_idx] if sym_dir != 0: continue elif direction == 1: #UL - sym_dir = slot_config[slot_idx%nTddPeriod][sym_idx] + sym_dir = slot_config[slot_idx%context["nTddPeriod"]][sym_idx] if sym_dir != 1: continue @@ -490,20 +492,8 @@ def parse_dat_file(test_cfg): code = compile(str(exe_line), '', 'exec') exec(code, global_env, local_env) - for k, var in local_env.items(): - globals()[k] = var - print(k, var) - return local_env -def del_dat_file_vars(local_env): - """method for deleting variables in file""" - for k, in local_env.items(): - del globals()[k] - - return 0 - - def run_tcase(rantech, cat, m_u, b_w, tcase, xran_path): """ method for runing test cases""" if rantech == 1: #LTE @@ -541,13 +531,13 @@ def run_tcase(rantech, cat, m_u, b_w, tcase, xran_path): usecase_cfg = parse_dat_file(test_cfg) - res = compare_resuts(rantech, cat, m_u, xran_path, 0) + res = compare_results(rantech, cat, m_u, xran_path, 0, usecase_cfg) if res != 0: os.chdir(w_d) print("FAIL") return res - res = compare_resuts(rantech, cat, m_u, xran_path, 1) + res = compare_results(rantech, cat, m_u, xran_path, 1, usecase_cfg) if res != 0: os.chdir(w_d) print("FAIL") @@ -556,8 +546,6 @@ def run_tcase(rantech, cat, m_u, b_w, tcase, xran_path): os.chdir(w_d) print("PASS") - del_dat_file_vars(usecase_cfg) - return res def main(): From 17ae6652e19e0a48a45067b1ccf31b413120535f Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Fri, 21 Aug 2020 16:38:01 +0200 Subject: [PATCH 27/80] Added returning of exit codes from ansible-playbook process for better error handling in tests automation scripts (#109) --- applications/eis-experience-kit/cleanup_eis_pcb_demo.sh | 1 + applications/eis-experience-kit/deploy_eis_pcb_demo.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/eis-experience-kit/cleanup_eis_pcb_demo.sh b/applications/eis-experience-kit/cleanup_eis_pcb_demo.sh index ea9ba446..3608483a 100755 --- a/applications/eis-experience-kit/cleanup_eis_pcb_demo.sh +++ b/applications/eis-experience-kit/cleanup_eis_pcb_demo.sh @@ -6,3 +6,4 @@ source scripts/ansible-precheck.sh # check if ansibles are already installed # and ready to use ansible-playbook -vv ./eis_pcb_demo_cleanup.yml --inventory inventory.ini +exit $? diff --git a/applications/eis-experience-kit/deploy_eis_pcb_demo.sh b/applications/eis-experience-kit/deploy_eis_pcb_demo.sh index b126323e..69902439 100755 --- a/applications/eis-experience-kit/deploy_eis_pcb_demo.sh +++ b/applications/eis-experience-kit/deploy_eis_pcb_demo.sh @@ -11,4 +11,4 @@ ansible-playbook -vv ./eis_sources.yml --inventory inventory.ini source scripts/eis_repo_config.sh . ansible-playbook -vv ./eis_pcb_demo.yml --inventory inventory.ini - +exit $? From 72a88d8a9b1f913de1717e3c9bc1737c24d74c4e Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Tue, 25 Aug 2020 16:59:38 +0800 Subject: [PATCH 28/80] add a README.md --- applications/openvino/benchmark/README.md | 52 +++++++++++++++++++ .../openvino/benchmark/benchmark_job.yaml | 4 +- 2 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 applications/openvino/benchmark/README.md diff --git a/applications/openvino/benchmark/README.md b/applications/openvino/benchmark/README.md new file mode 100644 index 00000000..e878210b --- /dev/null +++ b/applications/openvino/benchmark/README.md @@ -0,0 +1,52 @@ +# Optimize the OpenNESS with K8s Benchmark APP for CPU and HDDL-R mode + +use k8s configMap to store all the parameters passed to `benchmark_app` command which is in the `do_benchmark.sh` shell script file. + +### The details are as follows: + +1. First, create a configMap named `cm-benchmark` defining a file containing all environment variables for the `benchmark_app` command executed in `docker` container.\ + + ```yaml + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: cm-benchmark + data: + env.txt: | + NIREQ=4 + NITER=10 + # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable + TARGET_DEVICE=CPU + --- + ``` + +Insert the configMap content to the top of `benchmark_job.yaml` file. + +2. modify the `benchmark_job.yaml` file, add a item to `volumeMounts` and `volumes` fields respectively and add a `--env-file` option to docker run command at args field. + + ```yaml + ... ... + args: ["-c", "/usr/local/bin/docker run --rm --device-cgroup-rule='c 10:* rmw' --device-cgroup-rule='c 89:* rmw' --device-cgroup-rule='c 189:* rmw' --device-cgroup-rule='c 180:* rmw' -v /dev:/dev -v /var/tmp:/var/tmp --env-file /dockerenv/env.txt openvino-benchmark:1.0 /do_benchmark.sh"] + ... ... + volumeMounts: + - name: dockerenv + mountPath: /dockerenv + ... ... + volumes: + - name: dockerenv + configMap: + name: cm-benchmark + ``` + +3. Edit the `do_benchmark.sh` file and replace the parameter by environment variables defined at step 1. + + ```sh + ... ... + ./demo_squeezenet_download_convert_run.sh -d ${TARGET_DEVICE} + + /root/inference_engine_samples_build/intel64/Release/benchmark_app -i /opt/intel/openvino/deployment_tools/demo/car.png -m /root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api async + ``` + + + diff --git a/applications/openvino/benchmark/benchmark_job.yaml b/applications/openvino/benchmark/benchmark_job.yaml index fd778a70..fa79fb1b 100644 --- a/applications/openvino/benchmark/benchmark_job.yaml +++ b/applications/openvino/benchmark/benchmark_job.yaml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation - +--- apiVersion: v1 kind: ConfigMap metadata: @@ -20,7 +20,7 @@ metadata: labels: jobgroup: openvino-benchmark spec: - parallelism: 1 + parallelism: 2 template: metadata: labels: From c73ec6103d10d8d49daf0a7a045169b406a30140 Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Wed, 26 Aug 2020 18:18:31 +0800 Subject: [PATCH 29/80] add more parameters in cm --- applications/openvino/benchmark/benchmark_job.yaml | 11 ++++++++--- applications/openvino/benchmark/do_benchmark.sh | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/applications/openvino/benchmark/benchmark_job.yaml b/applications/openvino/benchmark/benchmark_job.yaml index fa79fb1b..8f2884c3 100644 --- a/applications/openvino/benchmark/benchmark_job.yaml +++ b/applications/openvino/benchmark/benchmark_job.yaml @@ -8,10 +8,15 @@ metadata: name: cm-benchmark data: env.txt: | - NIREQ=4 - NITER=10 + NIREQ=32 + NITER=50000 # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable TARGET_DEVICE=CPU + IMAGE=/opt/intel/openvino/deployment_tools/demo/car.png + MODEL=/root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml + # sync/async + API=async + BATCH_SIZE=1 --- apiVersion: batch/v1 kind: Job @@ -20,7 +25,7 @@ metadata: labels: jobgroup: openvino-benchmark spec: - parallelism: 2 + parallelism: 1 template: metadata: labels: diff --git a/applications/openvino/benchmark/do_benchmark.sh b/applications/openvino/benchmark/do_benchmark.sh index cf27b117..99231d2b 100755 --- a/applications/openvino/benchmark/do_benchmark.sh +++ b/applications/openvino/benchmark/do_benchmark.sh @@ -7,5 +7,5 @@ source /opt/intel/openvino/bin/setupvars.sh cd /opt/intel/openvino/deployment_tools/demo ./demo_squeezenet_download_convert_run.sh -d ${TARGET_DEVICE} -/root/inference_engine_samples_build/intel64/Release/benchmark_app -i /opt/intel/openvino/deployment_tools/demo/car.png -m /root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api async +/root/inference_engine_samples_build/intel64/Release/benchmark_app -i ${IMAGE} -m ${MODEL} -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api ${API} -b ${BATCH_SIZE} From e055de5e1234a0a0739a84314ff1fe9e07c5f670 Mon Sep 17 00:00:00 2001 From: relhwigi Date: Wed, 26 Aug 2020 13:39:31 +0100 Subject: [PATCH 30/80] resolved conflicts --- .../etcd/deploy/files/eis_integ/eis_integ.py | 6 +++--- .../etcd/deploy/files/eis_integ/etcd_zmq_key.py | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py index 29fbc4da..55276cf2 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/eis_integ.py @@ -159,10 +159,10 @@ def remove_user_privilege(name): :type file: String """ try: - subprocess.check_output(["etcdctl", "role", "remove", name], stderr=subprocess.STDOUT) - logging.info("Role %s has been removed.", name) + subprocess.check_output(["etcdctl", "role", "delete", name], stderr=subprocess.STDOUT) + logging.info("Role {} has been removed.".format(name)) except subprocess.CalledProcessError as err: - logging.error("Error returned while removing the %s role: %s", name, err) + logging.error("Error returned while removing the {} role: {}".format(name, err)) raise EisIntegError(CODES.EXT_CMD_ERROR) def remove_user(name): diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py index 8233eb73..2472f357 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py @@ -14,6 +14,9 @@ def parse_arguments(_cli_args): """ Parse argument passed to function """ parser = argparse.ArgumentParser(description="Specify Application name") parser.add_argument("app", help="Name of the client application") + parser.add_argument("--force", action='store_true', + help="Force putting new generated ZMQ keys without checking" + + "if they already exist") return parser.parse_args() def main(args): @@ -28,7 +31,17 @@ def main(args): eis_integ.check_path_variable("ETCDCTL_CERT", os.environ.get("ETCDCTL_CERT")) eis_integ.check_path_variable("ETCDCTL_KEY", os.environ.get("ETCDCTL_KEY")) - logging.info("Generate ZMQ pair keys for %s and put them to the etcd database", args.app) + skip_gen = False + if not args.force: + logging.info("Check if ZMQ key pair for {} app already exists".format(args.app)) + skip_gen = eis_integ.check_zmqkeys(args.app) + + if not skip_gen: + logging.info("Generate ZMQ pair keys for {} and put them to the etcd database" + .format(args.app)) + eis_integ.put_zmqkeys(args.app) + else: + logging.info("ZMQ pair keys generation skipped for {} app".format(args.app)) eis_integ.put_zmqkeys(args.app) return eis_integ.CODES.NO_ERROR @@ -38,5 +51,5 @@ def main(args): try: sys.exit(main(parse_arguments(sys.argv[1:])).value) except eis_integ.EisIntegError as exception: - logging.error("Error while generating ZMQ keys: %s", exception) + logging.error("Error while generating ZMQ keys: %s", str(exception)) sys.exit(exception.code.value) From 3035d0f9a75b1ce2c204babfadcedff4a5e1e4dd Mon Sep 17 00:00:00 2001 From: relhwigi Date: Wed, 26 Aug 2020 13:41:58 +0100 Subject: [PATCH 31/80] resolved conflicts --- .../roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py index 2472f357..b3ee3755 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py @@ -14,8 +14,7 @@ def parse_arguments(_cli_args): """ Parse argument passed to function """ parser = argparse.ArgumentParser(description="Specify Application name") parser.add_argument("app", help="Name of the client application") - parser.add_argument("--force", action='store_true', - help="Force putting new generated ZMQ keys without checking" + + parser.add_argument("--force", action='store_true', help="Force putting new generated ZMQ keys without checking" + "if they already exist") return parser.parse_args() From b41903d1c7b2b674d6078e83992c911937006db6 Mon Sep 17 00:00:00 2001 From: relhwigi Date: Wed, 26 Aug 2020 13:44:09 +0100 Subject: [PATCH 32/80] resolved conflicts --- .../roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py index b3ee3755..ea8b3714 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py @@ -15,7 +15,7 @@ def parse_arguments(_cli_args): parser = argparse.ArgumentParser(description="Specify Application name") parser.add_argument("app", help="Name of the client application") parser.add_argument("--force", action='store_true', help="Force putting new generated ZMQ keys without checking" + - "if they already exist") + "if they already exist") return parser.parse_args() def main(args): @@ -36,8 +36,7 @@ def main(args): skip_gen = eis_integ.check_zmqkeys(args.app) if not skip_gen: - logging.info("Generate ZMQ pair keys for {} and put them to the etcd database" - .format(args.app)) + logging.info("Generate ZMQ pair keys for {} and put them to the etcd database".format(args.app)) eis_integ.put_zmqkeys(args.app) else: logging.info("ZMQ pair keys generation skipped for {} app".format(args.app)) From effe5fa65ecc55ab377100f63a3d1f03221b061f Mon Sep 17 00:00:00 2001 From: relhwigi Date: Wed, 26 Aug 2020 13:45:29 +0100 Subject: [PATCH 33/80] resolved conflicts --- .../roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py index ea8b3714..80c72622 100755 --- a/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py +++ b/applications/eis-experience-kit/roles/etcd/deploy/files/eis_integ/etcd_zmq_key.py @@ -13,7 +13,7 @@ def parse_arguments(_cli_args): """ Parse argument passed to function """ parser = argparse.ArgumentParser(description="Specify Application name") - parser.add_argument("app", help="Name of the client application") + parser.add_argument("app", help= "Name of the client application") parser.add_argument("--force", action='store_true', help="Force putting new generated ZMQ keys without checking" + "if they already exist") return parser.parse_args() From 2408959f3fc33eb06e21c6abdcb560db8d433631 Mon Sep 17 00:00:00 2001 From: cjnolan <47635874+cjnolan@users.noreply.github.com> Date: Thu, 27 Aug 2020 09:17:22 +0100 Subject: [PATCH 34/80] Fix shellcheck issues (#112) * Fix shellcheck issues * Address review comments --- .../roles/camera_stream/build/files/streamstart.sh | 4 ++-- applications/eis-experience-kit/scripts/task_log_file.sh | 6 +++--- applications/vas-sample-app/build/build-image.sh | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh b/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh index 0a320f52..5f4da952 100755 --- a/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh +++ b/applications/eis-experience-kit/roles/camera_stream/build/files/streamstart.sh @@ -7,12 +7,12 @@ if [[ $# -ne 2 ]] ; then exit 1 fi -if [ $1 == "pcb" ] +if [ "$1" == "pcb" ] then echo "Set pcb demo rtsp stream file" name="/tmp/pcb_d2000.avi" -elif [ $1 == "safety" ] +elif [ "$1" == "safety" ] then echo "Set Safety Hat demo rtsp stream file" name="/tmp/Safety_Full_Hat_and_Vest.avi" diff --git a/applications/eis-experience-kit/scripts/task_log_file.sh b/applications/eis-experience-kit/scripts/task_log_file.sh index 0cc49263..517b7c0a 100644 --- a/applications/eis-experience-kit/scripts/task_log_file.sh +++ b/applications/eis-experience-kit/scripts/task_log_file.sh @@ -1,11 +1,11 @@ +#!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2019-2020 Intel Corporation -currdir=${PWD##*/} BASE_PATH="$( cd "$(dirname "$0")" ; pwd -P )" -[ -d ${BASE_PATH}/logs ] || mkdir ${BASE_PATH}/logs -FILENAME="`date +%Y-%m-%d_%H-%M-%S_ansible.log`" +[ -d "${BASE_PATH}/logs" ] || mkdir "${BASE_PATH}/logs" +FILENAME="$(date +%Y-%m-%d_%H-%M-%S_ansible.log)" # Comment out below export to disable console logs saving to files. export ANSIBLE_LOG_PATH="${BASE_PATH}/logs/${FILENAME}" diff --git a/applications/vas-sample-app/build/build-image.sh b/applications/vas-sample-app/build/build-image.sh index c7d97c25..1c2f7525 100755 --- a/applications/vas-sample-app/build/build-image.sh +++ b/applications/vas-sample-app/build/build-image.sh @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2020 Intel Corporation +declare http_proxy https_proxy no_proxy sudo docker build -f ./build/Dockerfile \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg no_proxy=$no_proxy \ + --build-arg http_proxy="$http_proxy" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg no_proxy="$no_proxy" \ -t vas-cons-app:1.0 . From de893fd9ad1b70f21eb94a4397ef3577fb8d78cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Thu, 27 Aug 2020 15:15:26 +0200 Subject: [PATCH 35/80] added support for multiple VFs in config --- .../core-network/5G/UPF/Dockerfile | 1 + .../core-network/5G/UPF/run_upf.sh | 106 +++++++++++++++++- .../charts/upf/templates/5g-upf.yaml | 20 +++- .../core-network/charts/upf/values.yaml | 88 ++++++++------- 4 files changed, 171 insertions(+), 44 deletions(-) diff --git a/network-functions/core-network/5G/UPF/Dockerfile b/network-functions/core-network/5G/UPF/Dockerfile index 8d1aa516..9877cad0 100644 --- a/network-functions/core-network/5G/UPF/Dockerfile +++ b/network-functions/core-network/5G/UPF/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update && \ apt-get -y install iputils-ping ethtool && \ apt-get -y install python-pip libcap-ng-dev gawk pciutils kmod && \ apt-get -y install libc6 pciutils python3 lsb-base libnuma-dev iproute2 findutils apt-utils libtool fakeroot libmbedtls-dev libhyperscan4 sudo +RUN groupadd vpp WORKDIR /root diff --git a/network-functions/core-network/5G/UPF/run_upf.sh b/network-functions/core-network/5G/UPF/run_upf.sh index c5fadc9c..b495efa3 100755 --- a/network-functions/core-network/5G/UPF/run_upf.sh +++ b/network-functions/core-network/5G/UPF/run_upf.sh @@ -54,7 +54,9 @@ uio_drv=$CONFIG_UIO_DRIVER upf_config=/tmp/upf.conf rm -f $upf_config - +#default edgeapp config: +if [ -z "$CONFIG_N3_IF_NAME" ] && [ -z "$CONFIG_N4_IF_NAME" ] +then cat < $upf_config comment {set N6 interface} @@ -78,6 +80,72 @@ upf nwi name sgi vrf 0 comment {set N3 gtpu endpoint} upf gtpu endpoint ip $n3_ip_plain nwi epc TEID-Range-Indication 2 +EOF +fi +#I-UPF config: +if [ -n "$CONFIG_N3_IF_NAME" ] && [ -n "$CONFIG_N4_IF_NAME" ] +then +cat < $upf_config + +comment {set N6 interface} +set int ip address $n6_iface $n6_ip +set int state $n6_iface up + +comment {set N3 interface} +set interface mac address $CONFIG_N3_IF_NAME $CONFIG_N3_IF_MAC +set int ip address $CONFIG_N3_IF_NAME $CONFIG_N3_IP_ADDR +set int state $CONFIG_N3_IF_NAME up + +comment {set N4 interface} +set int ip address $CONFIG_N4_IF_NAME ${CONFIG_N4_IP_ADDR}/24 +set int state $CONFIG_N4_IF_NAME up + +comment {set route} +ip route add 0.0.0.0/0 table 0 via $n6_gw_ip $n6_iface + +comment {set N4 pfcp endpoint} +upf pfcp endpoint ip $CONFIG_N4_IP_ADDR vrf 0 +comment {upf pfcp endpoint ip $CONFIG_N4_IP_ADDR vrf 0} + +upf nwi name epc vrf 0 +upf nwi name sgi vrf 0 + +comment {set N3 gtpu endpoint} +upf gtpu endpoint ip $n3_ip_plain nwi epc TEID-Range-Indication 2 + +EOF +fi + +#PSA-UPF config: +if [ -z "$CONFIG_N3_IF_NAME" ] && [ -n "$CONFIG_N4_IF_NAME" ] +then +cat < $upf_config + +comment {set N6 interface} +set int ip address $n6_iface $n6_ip +set int state $n6_iface up + +comment {set N3 interface} +set int ip address $CONFIG_N4_IF_NAME ${CONFIG_N4_IP_ADDR}/24 +set int state $CONFIG_N4_IF_NAME up + +comment {set route} +ip route add 0.0.0.0/0 table 0 via $n6_gw_ip $n6_iface + +comment {set N4 pfcp endpoint} +upf pfcp endpoint ip $CONFIG_N4_IP_ADDR vrf 0 +comment {upf pfcp endpoint ip $CONFIG_N4_IP_ADDR vrf 0} + +upf nwi name epc vrf 0 +upf nwi name sgi vrf 0 + +comment {set N3 gtpu endpoint} +upf gtpu endpoint ip $CONFIG_N4_IP_ADDR nwi epc TEID-Range-Indication 2 + +EOF +fi +#the and of the config is common: +cat <> $upf_config comment {for pfcp-thread} upf enable-disable @@ -218,6 +286,42 @@ dpdk { } +EOF + +if [ -n "$CONFIG_N3_PCI_BUS_ADDR" ] +then +cat <> $upf_vpp_config + + dev $CONFIG_N3_PCI_BUS_ADDR { + num-rx-queues 1 + num-rx-desc 2048 + #ddp-enabled + #flow-control high 940 low 920 pause 50 xon 1 + #workers 0 1 2 3 4 5 6 7 + #num-rx-desc 4096 + } + +EOF +fi + +if [ -n "$CONFIG_N4_PCI_BUS_ADDR" ] +then +cat <> $upf_vpp_config + + dev $CONFIG_N4_PCI_BUS_ADDR { + num-rx-queues 1 + num-rx-desc 2048 + #ddp-enabled + #flow-control high 940 low 920 pause 50 xon 1 + #workers 0 1 2 3 4 5 6 7 + #num-rx-desc 4096 + } + +EOF +fi + +cat <> $upf_vpp_config + ## Specify bonded interface and its slaves via PCI addresses ## ## Bonded interface in XOR load balance mode (mode 2) with L3 and L4 headers diff --git a/network-functions/core-network/charts/upf/templates/5g-upf.yaml b/network-functions/core-network/charts/upf/templates/5g-upf.yaml index 7d73a2e3..e9772b58 100644 --- a/network-functions/core-network/charts/upf/templates/5g-upf.yaml +++ b/network-functions/core-network/charts/upf/templates/5g-upf.yaml @@ -19,10 +19,7 @@ spec: privileged: {{ .Values.isPrivileged }} command: ["/bin/bash", "-c", "--"] args: - - | - while true ; do - sleep 90000000 - done + - "/root/run_upf.sh" imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: CONFIG_VF_IF_NAME @@ -49,6 +46,21 @@ spec: value: {{ .Values.upf.n6_addr }} - name: CONFIG_N6_GW_IP_ADDR value: {{ .Values.upf.n6_gw_addr }} + - name: CONFIG_N3_IF_NAME + value: {{ .Values.n3_if_name }} + - name: CONFIG_N3_IF_MAC + value: {{ .Values.n3_if_mac }} + - name: CONFIG_N3_PCI_BUS_ADDR + value: {{ .Values.n3_pci_bus_addr }} + - name: CONFIG_N3_UIO_DRIVER + value: {{ .Values.n3_uio_driver }} + - name: CONFIG_N4_IF_NAME + value: {{ .Values.n4_if_name }} + - name: CONFIG_N4_PCI_BUS_ADDR + value: {{ .Values.n4_pci_bus_addr }} + - name: CONFIG_N4_UIO_DRIVER + value: {{ .Values.n4_uio_driver }} + volumeMounts: - name: hugepages mountPath: /hugepages diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index 75d03688..03599f4b 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -2,43 +2,53 @@ # Copyright (c) 2020 Intel Corporation --- -image: - # Change Me! - please provide IP address and port of docker registry in - # the format :/ - # :/intel/upf - # of Docker registry where UPF docker image is uploaded - repository: upf-cnf - tag: "1.0" # The tag identifying the upf docker image, - pullPolicy: IfNotPresent # Pull policy for docker image - -restartPolicy: Never # Restart policy for the pod -allowPrivilegeEscalation: true # Allow privilege escalation level values: true false -isPrivileged: true # Pod privilege level, possible values: true false - -#huge pages information -hugePageSize: hugepages-1Gi # Possible values: hugepages-1Gi hugepages-2Mi -hugePageAmount: 4Gi # Amount of hugepages to be reserved for the pod -memory: 6Gi # Amount of memory to be reserved for the pod - -#node information -node: - name: upf-node # Node on which the upf needs to be run - path: /root/upf # Path on the node where the upf binary is stored - -# upf configuration -upf: - vf_if_name: VirtualFunctionEthernetaf/a/0 # VF Interface name - pci_bus_addr: 0000:af:0a.0 # full format of the PCI bus addr of the VF interface the UPF needs to be attached - uio_driver: igb_uio # UIO driver used. Options are: igb_uio, vfio-pci - huge_memory: 4G # huge page amount - main_core: "1" # main core - worker_cores: "2,3" # worker cores - pfcp_thread: - cores: "4" # cores for the pfcp thread - count: "1" # number of pfcp threads - n3_addr: 192.179.120.170/24 # ip address of the n3 interface along with subnet - n4_addr: 192.179.120.170 # ip address of the n4 interface along with subnet - n6_addr: 192.168.1.170/24 # ip address of the n6 interface along with subnet - n6_gw_addr: 192.168.1.170 # ip address of the n6 gateway - + image: + # Change Me! - please provide IP address and port of docker registry in + # the format :/ + # :/intel/upf + # of Docker registry where UPF docker image is uploaded + repository: upf-cnf + tag: "1.0" # The tag identifying the upf docker image, + pullPolicy: IfNotPresent # Pull policy for docker image + + restartPolicy: Never # Restart policy for the pod + allowPrivilegeEscalation: true # Allow privilege escalation level values: true false + isPrivileged: true # Pod privilege level, possible values: true false + + #huge pages information + hugePageSize: hugepages-1Gi # Possible values: hugepages-1Gi hugepages-2Mi + hugePageAmount: 4Gi # Amount of hugepages to be reserved for the pod + memory: 6Gi # Amount of memory to be reserved for the pod + + #node information + node: + name: upf-node # Node on which the upf needs to be run + path: /root/upf # Path on the node where the upf binary is stored + + # upf configuration + upf: + vf_if_name: VirtualFunctionEthernetaf/a/0 # VF Interface name + pci_bus_addr: 0000:af:0a.0 # full format of the PCI bus addr of the VF interface the UPF needs to be attached + uio_driver: igb_uio # UIO driver used. Options are: igb_uio, vfio-pci + huge_memory: 4G # huge page amount + main_core: "1" # main core + worker_cores: "2,3" # worker cores + pfcp_thread: + cores: "4" # cores for the pfcp thread + count: "1" # number of pfcp threads + n3_addr: 192.179.120.170/24 # ip address of the n3 interface along with subnet + n4_addr: 192.179.120.170 # ip address of the n4 interface along with subnet + n6_addr: 192.168.1.170/24 # ip address of the n6 interface along with subnet + n6_gw_addr: 192.168.1.170 # ip address of the n6 gateway + + # extended upf configuration + + n3_if_name: "" # VF Interface name + n3_if_mac: "" # VF Interface MAC address + n3_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached + n3_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci + n4_if_name: "" # VF Interface name + n4_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached + n4_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci + \ No newline at end of file From 78e7bd04c7a13269c7b6a800b679cadfd0188668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Thu, 27 Aug 2020 15:31:02 +0200 Subject: [PATCH 36/80] removed unnecessary indent in /network-functions/core-network/charts/upf/values.yaml --- .../core-network/charts/upf/values.yaml | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index 03599f4b..f1d7d5cd 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -2,53 +2,53 @@ # Copyright (c) 2020 Intel Corporation --- - image: - # Change Me! - please provide IP address and port of docker registry in - # the format :/ - # :/intel/upf - # of Docker registry where UPF docker image is uploaded - repository: upf-cnf - tag: "1.0" # The tag identifying the upf docker image, - pullPolicy: IfNotPresent # Pull policy for docker image - - restartPolicy: Never # Restart policy for the pod - allowPrivilegeEscalation: true # Allow privilege escalation level values: true false - isPrivileged: true # Pod privilege level, possible values: true false - - #huge pages information - hugePageSize: hugepages-1Gi # Possible values: hugepages-1Gi hugepages-2Mi - hugePageAmount: 4Gi # Amount of hugepages to be reserved for the pod - memory: 6Gi # Amount of memory to be reserved for the pod - - #node information - node: - name: upf-node # Node on which the upf needs to be run - path: /root/upf # Path on the node where the upf binary is stored - - # upf configuration - upf: - vf_if_name: VirtualFunctionEthernetaf/a/0 # VF Interface name - pci_bus_addr: 0000:af:0a.0 # full format of the PCI bus addr of the VF interface the UPF needs to be attached - uio_driver: igb_uio # UIO driver used. Options are: igb_uio, vfio-pci - huge_memory: 4G # huge page amount - main_core: "1" # main core - worker_cores: "2,3" # worker cores - pfcp_thread: - cores: "4" # cores for the pfcp thread - count: "1" # number of pfcp threads - n3_addr: 192.179.120.170/24 # ip address of the n3 interface along with subnet - n4_addr: 192.179.120.170 # ip address of the n4 interface along with subnet - n6_addr: 192.168.1.170/24 # ip address of the n6 interface along with subnet - n6_gw_addr: 192.168.1.170 # ip address of the n6 gateway - - # extended upf configuration - - n3_if_name: "" # VF Interface name - n3_if_mac: "" # VF Interface MAC address - n3_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached - n3_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci - - n4_if_name: "" # VF Interface name - n4_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached - n4_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci +image: + # Change Me! - please provide IP address and port of docker registry in + # the format :/ + # :/intel/upf + # of Docker registry where UPF docker image is uploaded + repository: upf-cnf + tag: "1.0" # The tag identifying the upf docker image, + pullPolicy: IfNotPresent # Pull policy for docker image + +restartPolicy: Never # Restart policy for the pod +allowPrivilegeEscalation: true # Allow privilege escalation level values: true false +isPrivileged: true # Pod privilege level, possible values: true false + +#huge pages information +hugePageSize: hugepages-1Gi # Possible values: hugepages-1Gi hugepages-2Mi +hugePageAmount: 4Gi # Amount of hugepages to be reserved for the pod +memory: 6Gi # Amount of memory to be reserved for the pod + +#node information +node: + name: upf-node # Node on which the upf needs to be run + path: /root/upf # Path on the node where the upf binary is stored + +# upf configuration +upf: + vf_if_name: VirtualFunctionEthernetaf/a/0 # VF Interface name + pci_bus_addr: 0000:af:0a.0 # full format of the PCI bus addr of the VF interface the UPF needs to be attached + uio_driver: igb_uio # UIO driver used. Options are: igb_uio, vfio-pci + huge_memory: 4G # huge page amount + main_core: "1" # main core + worker_cores: "2,3" # worker cores + pfcp_thread: + cores: "4" # cores for the pfcp thread + count: "1" # number of pfcp threads + n3_addr: 192.179.120.170/24 # ip address of the n3 interface along with subnet + n4_addr: 192.179.120.170 # ip address of the n4 interface along with subnet + n6_addr: 192.168.1.170/24 # ip address of the n6 interface along with subnet + n6_gw_addr: 192.168.1.170 # ip address of the n6 gateway + +# extended upf configuration + +n3_if_name: "" # VF Interface name +n3_if_mac: "" # VF Interface MAC address +n3_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached +n3_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci + +n4_if_name: "" # VF Interface name +n4_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached +n4_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci \ No newline at end of file From 179d73aeb1d73b87d1c298357223a621ccea3312 Mon Sep 17 00:00:00 2001 From: Mateusz Szelest Date: Fri, 28 Aug 2020 01:24:19 +0200 Subject: [PATCH 37/80] Script for building AMF/SMF kubevirt image --- .../core-network/5G/AMF_SMF/README.md | 46 +++++++++++++++++++ .../5G/AMF_SMF/build_amf_smf_image.sh | 34 ++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 network-functions/core-network/5G/AMF_SMF/README.md create mode 100644 network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh diff --git a/network-functions/core-network/5G/AMF_SMF/README.md b/network-functions/core-network/5G/AMF_SMF/README.md new file mode 100644 index 00000000..ad5e87e2 --- /dev/null +++ b/network-functions/core-network/5G/AMF_SMF/README.md @@ -0,0 +1,46 @@ +```text +SPDX-License-Identifier: Apache-2.0 +Copyright (c) 2020 Intel Corporation +``` + +## Core Network 5G – AMF/SMF +AMF/SMF can be deployed currently as kubevirt virtual machine. This directory contains script that will in VM image preparation. The image is based on Ubuntu 18.04. + +### Build script +`build_amf_smf_image.sh` shell script in a few steps helps to prepare VM image that can be used for AMF/SMF deployment. + +### Prerequisites +The script requires several libvirt packets to be installed: + +`yum install -y qemu-kvm libvirt libvirt-python libguestfs-tools virt-install` + +After that start libvirt as a service: + +```sh +systemctl enable libvirtd +systemctl start libvirtd +``` + +Commands used for image resizing require also e2fsck library to be in at least 1.43 version. CentOS 7 has usually 1.42 in its repository so in that case it needs to be built from sources. Below are the commands required for installation: + +```sh +wget http://prdownloads.sourceforge.net/e2fsprogs/e2fsprogs-1.43.9.tar.gz +tar zxvf e2fsprogs-1.43.9.tar.gz +cd e2fsprogs-1.43.9 +./configure +make +make install +``` + +It is require also to have FlexCORE binaries already downloaded. + +### Running the script +Script takes two parameters - one is the destination path for the image and second is the path to the FlexCORE binaries directory. Usage looks like: + +```sh +./build_amf_smf_image.sh +``` + +As a result after successful run, customized image ready to be used by kubevirt virtual machine should be created in destination path. + +Image is based on *Ubuntu 18.04 LTS Minimal* image provided by Ubuntu site. diff --git a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh new file mode 100644 index 00000000..555087cd --- /dev/null +++ b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +if [[ ${#} -ne 2 ]]; then + echo "Wrong arguments pased. Usage: ${0} " + exit 1 +fi + +img_file=${1}/ubuntu-18.04-minimal-cloudimg-amd64.qcow2 +astri_dir=${2} + +if [[ ! -f ${img_file} ]]; then + curl https://cloud-images.ubuntu.com/minimal/releases/bionic/release/ubuntu-18.04-minimal-cloudimg-amd64.img -o ${img_file} + if [[ ${?} -ne 0 ]]; then + echo "ERROR: Failed to download Ubuntu image." + exit 1 + fi +else + echo "Skipping image download - file already exists." +fi + +virt-customize -a ${img_file} \ + --root-password password:root \ + --update \ + --install qemu-guest-agent,iputils-ping,iproute2,screen,libpcap-dev,tcpdump,libsctp-dev,apache2,python-pip,sudo,ssh \ + --mkdir /root/amf-smf \ + --copy-in /root/flexcore-5g-rel/astri-1907-rc6-0114-ulcl:/root/amf-smf + +if [[ ${?} -ne 0 ]]; then + echo "ERROR: Failed to customize the image." + exit 1 +fi From 1c298cc475352cb1f2466c9c14e1976a2f7b17e5 Mon Sep 17 00:00:00 2001 From: Mateusz Szelest Date: Fri, 28 Aug 2020 01:35:27 +0200 Subject: [PATCH 38/80] Fixed shellcheck findings --- .../core-network/5G/AMF_SMF/build_amf_smf_image.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh index 555087cd..394f3155 100644 --- a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh +++ b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh @@ -12,7 +12,7 @@ img_file=${1}/ubuntu-18.04-minimal-cloudimg-amd64.qcow2 astri_dir=${2} if [[ ! -f ${img_file} ]]; then - curl https://cloud-images.ubuntu.com/minimal/releases/bionic/release/ubuntu-18.04-minimal-cloudimg-amd64.img -o ${img_file} + curl https://cloud-images.ubuntu.com/minimal/releases/bionic/release/ubuntu-18.04-minimal-cloudimg-amd64.img -o "${img_file}" if [[ ${?} -ne 0 ]]; then echo "ERROR: Failed to download Ubuntu image." exit 1 @@ -21,12 +21,12 @@ else echo "Skipping image download - file already exists." fi -virt-customize -a ${img_file} \ +virt-customize -a "${img_file}" \ --root-password password:root \ --update \ --install qemu-guest-agent,iputils-ping,iproute2,screen,libpcap-dev,tcpdump,libsctp-dev,apache2,python-pip,sudo,ssh \ --mkdir /root/amf-smf \ - --copy-in /root/flexcore-5g-rel/astri-1907-rc6-0114-ulcl:/root/amf-smf + --copy-in "${astri_dir}:/root/amf-smf" if [[ ${?} -ne 0 ]]; then echo "ERROR: Failed to customize the image." From 02ad950c2f4aa0abfdd5fa784777d980b719f475 Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Fri, 28 Aug 2020 16:34:35 +0800 Subject: [PATCH 39/80] modify README.md --- applications/openvino/benchmark/README.md | 9 +++++++-- applications/openvino/benchmark/benchmark_job.yaml | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/applications/openvino/benchmark/README.md b/applications/openvino/benchmark/README.md index e878210b..5d91cd55 100644 --- a/applications/openvino/benchmark/README.md +++ b/applications/openvino/benchmark/README.md @@ -14,10 +14,15 @@ use k8s configMap to store all the parameters passed to `benchmark_app` command name: cm-benchmark data: env.txt: | - NIREQ=4 - NITER=10 + NIREQ=32 + NITER=50000 # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable TARGET_DEVICE=CPU + IMAGE=/opt/intel/openvino/deployment_tools/demo/car.png + MODEL=/root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml + # API: sync/async + API=async + BATCH_SIZE=1 --- ``` diff --git a/applications/openvino/benchmark/benchmark_job.yaml b/applications/openvino/benchmark/benchmark_job.yaml index 8f2884c3..41501d50 100644 --- a/applications/openvino/benchmark/benchmark_job.yaml +++ b/applications/openvino/benchmark/benchmark_job.yaml @@ -14,7 +14,7 @@ data: TARGET_DEVICE=CPU IMAGE=/opt/intel/openvino/deployment_tools/demo/car.png MODEL=/root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml - # sync/async + # API: sync/async API=async BATCH_SIZE=1 --- @@ -25,7 +25,7 @@ metadata: labels: jobgroup: openvino-benchmark spec: - parallelism: 1 + parallelism: 2 template: metadata: labels: From 42d1a3893a01cd318249a6337cfd881002faab21 Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Fri, 28 Aug 2020 16:38:06 +0800 Subject: [PATCH 40/80] modify README.md --- applications/openvino/benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/openvino/benchmark/README.md b/applications/openvino/benchmark/README.md index 5d91cd55..d6353912 100644 --- a/applications/openvino/benchmark/README.md +++ b/applications/openvino/benchmark/README.md @@ -50,7 +50,7 @@ Insert the configMap content to the top of `benchmark_job.yaml` file. ... ... ./demo_squeezenet_download_convert_run.sh -d ${TARGET_DEVICE} - /root/inference_engine_samples_build/intel64/Release/benchmark_app -i /opt/intel/openvino/deployment_tools/demo/car.png -m /root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api async + /root/inference_engine_samples_build/intel64/Release/benchmark_app -i ${IMAGE} -m ${MODEL} -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api ${API} -b ${BATCH_SIZE} ``` From d65474dd67e3c947d06f23c32c7416f4ce3ce6e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Fri, 28 Aug 2020 14:59:43 +0200 Subject: [PATCH 41/80] added upf_autostart flag --- .../core-network/charts/upf/templates/5g-upf.yaml | 12 +++++++++++- .../core-network/charts/upf/values.yaml | 5 ++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/network-functions/core-network/charts/upf/templates/5g-upf.yaml b/network-functions/core-network/charts/upf/templates/5g-upf.yaml index e9772b58..997d6ffe 100644 --- a/network-functions/core-network/charts/upf/templates/5g-upf.yaml +++ b/network-functions/core-network/charts/upf/templates/5g-upf.yaml @@ -19,7 +19,17 @@ spec: privileged: {{ .Values.isPrivileged }} command: ["/bin/bash", "-c", "--"] args: - - "/root/run_upf.sh" +{{- if eq .Values.upf_autostart true }} + - | + /root/run_upf.sh; while true ; do + sleep 90000000 + done +{{ else }} + - | + while true ; do + sleep 90000000 + done +{{ end }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: CONFIG_VF_IF_NAME diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index f1d7d5cd..60b0f181 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -27,7 +27,7 @@ node: # upf configuration upf: - vf_if_name: VirtualFunctionEthernetaf/a/0 # VF Interface name + vf_if_name: VirtualFunctionEthernetaf/a/0 # VF Interface name pci_bus_addr: 0000:af:0a.0 # full format of the PCI bus addr of the VF interface the UPF needs to be attached uio_driver: igb_uio # UIO driver used. Options are: igb_uio, vfio-pci huge_memory: 4G # huge page amount @@ -41,6 +41,9 @@ upf: n6_addr: 192.168.1.170/24 # ip address of the n6 interface along with subnet n6_gw_addr: 192.168.1.170 # ip address of the n6 gateway +#auto-startup of upf +upf_autostart: false # auto-start upf on default + # extended upf configuration n3_if_name: "" # VF Interface name From fc8f815ea42cdea8e9418de13d28d12da26cc828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Fri, 28 Aug 2020 15:10:05 +0200 Subject: [PATCH 42/80] added upf_autostart flag and refined code --- .../core-network/5G/UPF/run_upf.sh | 7 ++----- .../charts/upf/templates/5g-upf.yaml | 14 +++++++------- .../core-network/charts/upf/values.yaml | 18 +++++++++--------- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/network-functions/core-network/5G/UPF/run_upf.sh b/network-functions/core-network/5G/UPF/run_upf.sh index b495efa3..bc34f24c 100755 --- a/network-functions/core-network/5G/UPF/run_upf.sh +++ b/network-functions/core-network/5G/UPF/run_upf.sh @@ -81,9 +81,8 @@ comment {set N3 gtpu endpoint} upf gtpu endpoint ip $n3_ip_plain nwi epc TEID-Range-Indication 2 EOF -fi #I-UPF config: -if [ -n "$CONFIG_N3_IF_NAME" ] && [ -n "$CONFIG_N4_IF_NAME" ] +elif [ -n "$CONFIG_N3_IF_NAME" ] && [ -n "$CONFIG_N4_IF_NAME" ] then cat < $upf_config @@ -114,10 +113,8 @@ comment {set N3 gtpu endpoint} upf gtpu endpoint ip $n3_ip_plain nwi epc TEID-Range-Indication 2 EOF -fi - #PSA-UPF config: -if [ -z "$CONFIG_N3_IF_NAME" ] && [ -n "$CONFIG_N4_IF_NAME" ] +elif [ -z "$CONFIG_N3_IF_NAME" ] && [ -n "$CONFIG_N4_IF_NAME" ] then cat < $upf_config diff --git a/network-functions/core-network/charts/upf/templates/5g-upf.yaml b/network-functions/core-network/charts/upf/templates/5g-upf.yaml index 997d6ffe..7f0c59d5 100644 --- a/network-functions/core-network/charts/upf/templates/5g-upf.yaml +++ b/network-functions/core-network/charts/upf/templates/5g-upf.yaml @@ -57,19 +57,19 @@ spec: - name: CONFIG_N6_GW_IP_ADDR value: {{ .Values.upf.n6_gw_addr }} - name: CONFIG_N3_IF_NAME - value: {{ .Values.n3_if_name }} + value: {{ .Values.upf_extended.n3_if_name }} - name: CONFIG_N3_IF_MAC - value: {{ .Values.n3_if_mac }} + value: {{ .Values.upf_extended.n3_if_mac }} - name: CONFIG_N3_PCI_BUS_ADDR - value: {{ .Values.n3_pci_bus_addr }} + value: {{ .Values.upf_extended.n3_pci_bus_addr }} - name: CONFIG_N3_UIO_DRIVER - value: {{ .Values.n3_uio_driver }} + value: {{ .Values.upf_extended.n3_uio_driver }} - name: CONFIG_N4_IF_NAME - value: {{ .Values.n4_if_name }} + value: {{ .Values.upf_extended.n4_if_name }} - name: CONFIG_N4_PCI_BUS_ADDR - value: {{ .Values.n4_pci_bus_addr }} + value: {{ .Values.upf_extended.n4_pci_bus_addr }} - name: CONFIG_N4_UIO_DRIVER - value: {{ .Values.n4_uio_driver }} + value: {{ .Values.upf_extended.n4_uio_driver }} volumeMounts: - name: hugepages diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index 60b0f181..a5c5c643 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -45,13 +45,13 @@ upf: upf_autostart: false # auto-start upf on default # extended upf configuration - -n3_if_name: "" # VF Interface name -n3_if_mac: "" # VF Interface MAC address -n3_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached -n3_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci - -n4_if_name: "" # VF Interface name -n4_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached -n4_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci +upf_extended: + n3_if_name: "" # VF Interface name + n3_if_mac: "" # VF Interface MAC address + n3_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached + n3_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci + + n4_if_name: "" # VF Interface name + n4_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached + n4_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci \ No newline at end of file From 6ea26d683defad8a070256d1934f84176be61819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Fri, 28 Aug 2020 15:42:43 +0200 Subject: [PATCH 43/80] added variable upf path --- network-functions/core-network/charts/upf/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index a5c5c643..849eda86 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -54,4 +54,3 @@ upf_extended: n4_if_name: "" # VF Interface name n4_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached n4_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci - \ No newline at end of file From ef73cdaf46ebe72ef318427bdf3eba5d192041d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Fri, 28 Aug 2020 15:56:00 +0200 Subject: [PATCH 44/80] added variable upf path --- network-functions/core-network/5G/UPF/run_upf.sh | 2 +- network-functions/core-network/charts/upf/templates/5g-upf.yaml | 2 ++ network-functions/core-network/charts/upf/values.yaml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/network-functions/core-network/5G/UPF/run_upf.sh b/network-functions/core-network/5G/UPF/run_upf.sh index bc34f24c..79256943 100755 --- a/network-functions/core-network/5G/UPF/run_upf.sh +++ b/network-functions/core-network/5G/UPF/run_upf.sh @@ -26,7 +26,7 @@ n4_ip=$CONFIG_N4_IP_ADDR # Gateway IP address n6_gw_ip=$CONFIG_N6_GW_IP_ADDR -upf_path=/root/upf +upf_path=$CONFIG_UPF_PATH uio_drv=$CONFIG_UIO_DRIVER diff --git a/network-functions/core-network/charts/upf/templates/5g-upf.yaml b/network-functions/core-network/charts/upf/templates/5g-upf.yaml index 7f0c59d5..9e345f8f 100644 --- a/network-functions/core-network/charts/upf/templates/5g-upf.yaml +++ b/network-functions/core-network/charts/upf/templates/5g-upf.yaml @@ -56,6 +56,8 @@ spec: value: {{ .Values.upf.n6_addr }} - name: CONFIG_N6_GW_IP_ADDR value: {{ .Values.upf.n6_gw_addr }} + - name: CONFIG_UPF_PATH + value: {{ .Values.upf.path }} - name: CONFIG_N3_IF_NAME value: {{ .Values.upf_extended.n3_if_name }} - name: CONFIG_N3_IF_MAC diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index 849eda86..3b7941a2 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -40,6 +40,7 @@ upf: n4_addr: 192.179.120.170 # ip address of the n4 interface along with subnet n6_addr: 192.168.1.170/24 # ip address of the n6 interface along with subnet n6_gw_addr: 192.168.1.170 # ip address of the n6 gateway + path: /root/upf # upf path #auto-startup of upf upf_autostart: false # auto-start upf on default From 70527972d689abd1499f36ec7220b6390a66ac2f Mon Sep 17 00:00:00 2001 From: Mateusz Szelest Date: Fri, 28 Aug 2020 15:59:20 +0200 Subject: [PATCH 45/80] Fixed code review findings --- .../core-network/5G/AMF_SMF/README.md | 6 +++--- .../5G/AMF_SMF/build_amf_smf_image.sh | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) mode change 100644 => 100755 network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh diff --git a/network-functions/core-network/5G/AMF_SMF/README.md b/network-functions/core-network/5G/AMF_SMF/README.md index ad5e87e2..78f6872c 100644 --- a/network-functions/core-network/5G/AMF_SMF/README.md +++ b/network-functions/core-network/5G/AMF_SMF/README.md @@ -32,7 +32,7 @@ make make install ``` -It is require also to have FlexCORE binaries already downloaded. +It is also required to have FlexCORE binaries already downloaded. ### Running the script Script takes two parameters - one is the destination path for the image and second is the path to the FlexCORE binaries directory. Usage looks like: @@ -41,6 +41,6 @@ Script takes two parameters - one is the destination path for the image and seco ./build_amf_smf_image.sh ``` -As a result after successful run, customized image ready to be used by kubevirt virtual machine should be created in destination path. +As a result of successful run, the customized image (ready to be used by kubevirt virtual machine) should be created in destination path. -Image is based on *Ubuntu 18.04 LTS Minimal* image provided by Ubuntu site. +Image is based on *Ubuntu 18.04 LTS Minimal* provided by Ubuntu site. diff --git a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh old mode 100644 new mode 100755 index 394f3155..96da471c --- a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh +++ b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh @@ -4,11 +4,11 @@ # Copyright (c) 2020 Intel Corporation if [[ ${#} -ne 2 ]]; then - echo "Wrong arguments pased. Usage: ${0} " + echo "Wrong arguments passed. Usage: ${0} " exit 1 fi -img_file=${1}/ubuntu-18.04-minimal-cloudimg-amd64.qcow2 +img_file=${1} astri_dir=${2} if [[ ! -f ${img_file} ]]; then @@ -17,6 +17,18 @@ if [[ ! -f ${img_file} ]]; then echo "ERROR: Failed to download Ubuntu image." exit 1 fi + + # Check SHA256 if downloaded file is correct + curl https://cloud-images.ubuntu.com/minimal/releases/bionic/release/SHA256SUMS -o SHA256SUMS + if [[ ${?} -ne 0 ]]; then + echo "ERROR: Failed to download SHA256SUMS file." + exit 1 + fi + grep -q "$(sha256sum "${img_file}" | awk '{ print $1 }')" SHA256SUMS + if [[ ${?} -ne 0 ]]; then + echo "ERROR: Invalid checksum of downloaded file." + exit 1 + fi else echo "Skipping image download - file already exists." fi From 22f635252c341e74649b935d0910715fbc430956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Fri, 28 Aug 2020 16:51:08 +0200 Subject: [PATCH 46/80] Revert "added variable upf path" This reverts commit 6ea26d683defad8a070256d1934f84176be61819. --- network-functions/core-network/charts/upf/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index 3b7941a2..86474b6f 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -55,3 +55,4 @@ upf_extended: n4_if_name: "" # VF Interface name n4_pci_bus_addr: "" # full format of the PCI bus addr of the VF interface the UPF needs to be attached n4_uio_driver: "" # UIO driver used. Options are: igb_uio, vfio-pci + From f764bed38c019db4fcef6ba9ce9a2118880c7109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Fri, 28 Aug 2020 16:56:55 +0200 Subject: [PATCH 47/80] Revert "added variable upf path" This reverts commit ef73cdaf46ebe72ef318427bdf3eba5d192041d9. --- network-functions/core-network/5G/UPF/run_upf.sh | 2 +- network-functions/core-network/charts/upf/templates/5g-upf.yaml | 2 -- network-functions/core-network/charts/upf/values.yaml | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/network-functions/core-network/5G/UPF/run_upf.sh b/network-functions/core-network/5G/UPF/run_upf.sh index 79256943..bc34f24c 100755 --- a/network-functions/core-network/5G/UPF/run_upf.sh +++ b/network-functions/core-network/5G/UPF/run_upf.sh @@ -26,7 +26,7 @@ n4_ip=$CONFIG_N4_IP_ADDR # Gateway IP address n6_gw_ip=$CONFIG_N6_GW_IP_ADDR -upf_path=$CONFIG_UPF_PATH +upf_path=/root/upf uio_drv=$CONFIG_UIO_DRIVER diff --git a/network-functions/core-network/charts/upf/templates/5g-upf.yaml b/network-functions/core-network/charts/upf/templates/5g-upf.yaml index 9e345f8f..7f0c59d5 100644 --- a/network-functions/core-network/charts/upf/templates/5g-upf.yaml +++ b/network-functions/core-network/charts/upf/templates/5g-upf.yaml @@ -56,8 +56,6 @@ spec: value: {{ .Values.upf.n6_addr }} - name: CONFIG_N6_GW_IP_ADDR value: {{ .Values.upf.n6_gw_addr }} - - name: CONFIG_UPF_PATH - value: {{ .Values.upf.path }} - name: CONFIG_N3_IF_NAME value: {{ .Values.upf_extended.n3_if_name }} - name: CONFIG_N3_IF_MAC diff --git a/network-functions/core-network/charts/upf/values.yaml b/network-functions/core-network/charts/upf/values.yaml index 86474b6f..24ca9f74 100644 --- a/network-functions/core-network/charts/upf/values.yaml +++ b/network-functions/core-network/charts/upf/values.yaml @@ -40,7 +40,6 @@ upf: n4_addr: 192.179.120.170 # ip address of the n4 interface along with subnet n6_addr: 192.168.1.170/24 # ip address of the n6 interface along with subnet n6_gw_addr: 192.168.1.170 # ip address of the n6 gateway - path: /root/upf # upf path #auto-startup of upf upf_autostart: false # auto-start upf on default From adce25fb0c65e916f2edd84aafa1c0f5f1812230 Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Mon, 31 Aug 2020 15:14:30 +0800 Subject: [PATCH 48/80] modify README.md and add some steps --- applications/openvino/benchmark/README.md | 89 +++++++++++++++-------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/applications/openvino/benchmark/README.md b/applications/openvino/benchmark/README.md index d6353912..3a8fdc69 100644 --- a/applications/openvino/benchmark/README.md +++ b/applications/openvino/benchmark/README.md @@ -1,19 +1,25 @@ -# Optimize the OpenNESS with K8s Benchmark APP for CPU and HDDL-R mode +``` +SPDX-License-Identifier: Apache-2.0 +Copyright (c) 2020 Intel Corporation +``` -use k8s configMap to store all the parameters passed to `benchmark_app` command which is in the `do_benchmark.sh` shell script file. +# OpenVINO Benchmark Performance Test -### The details are as follows: +> Notes: The following all steps assumes that OpenNESS were installed through [OpenNESS playbooks](https://github.com/otcshare/specs/blob/master/doc/getting-started/network-edge/controller-edge-node-setup.md). -1. First, create a configMap named `cm-benchmark` defining a file containing all environment variables for the `benchmark_app` command executed in `docker` container.\ +- [Precondition](#precondition) +- [Build benchamrk image](#build-benchamrk-image) +- [Deploy](#deploy) +- [Troubleshooting](#troubleshooting) + +### Precondition + +1. **configuration preparation** + + edit `benchmark_job.yaml` file and modify the `data` field in `configMap` section. All the following environment variables is used by `benchmark_app` command. ```yaml - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: cm-benchmark - data: - env.txt: | + ... ... NIREQ=32 NITER=50000 # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable @@ -23,35 +29,56 @@ use k8s configMap to store all the parameters passed to `benchmark_app` command # API: sync/async API=async BATCH_SIZE=1 - --- + ... ... ``` -Insert the configMap content to the top of `benchmark_job.yaml` file. +### Build benchamrk image -2. modify the `benchmark_job.yaml` file, add a item to `volumeMounts` and `volumes` fields respectively and add a `--env-file` option to docker run command at args field. +1. Download edgeapp on both master and node, go to dir edgeapps/applications/openvino/benchmark/ + +2. Build benchmark on node by: + + ``` + sh build_image.sh + ``` + +3. change `parallelism: 1` to pod number in `benchmark_job.yaml` file on master. + +### Deploy + +1. Query which jobs are currently running. - ```yaml - ... ... - args: ["-c", "/usr/local/bin/docker run --rm --device-cgroup-rule='c 10:* rmw' --device-cgroup-rule='c 89:* rmw' --device-cgroup-rule='c 189:* rmw' --device-cgroup-rule='c 180:* rmw' -v /dev:/dev -v /var/tmp:/var/tmp --env-file /dockerenv/env.txt openvino-benchmark:1.0 /do_benchmark.sh"] - ... ... - volumeMounts: - - name: dockerenv - mountPath: /dockerenv - ... ... - volumes: - - name: dockerenv - configMap: - name: cm-benchmark ``` + kubectl get jobs + ``` + + If a job named `openvino-benchmark-job` in k8s cluster is exist, delete it. -3. Edit the `do_benchmark.sh` file and replace the parameter by environment variables defined at step 1. + ``` + kubectl delete jobs openvino-benchmark-job + ``` + +2. On master execute following command to start the job and get logs of each pod + + ``` + kubectl apply -f benchmark_job.yaml + ``` + +3. After `openvino-benchmark-job` job launched in step 2 is completed, on node `docker ps` and `docker logs` to get all deamon log + +### Troubleshooting + +In this section some issues in implementation process are covered + +1. Benchmark image build failed. When `sh build_image.sh` on node, interrupted by some errors such as following: ```sh ... ... - ./demo_squeezenet_download_convert_run.sh -d ${TARGET_DEVICE} - - /root/inference_engine_samples_build/intel64/Release/benchmark_app -i ${IMAGE} -m ${MODEL} -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api ${API} -b ${BATCH_SIZE} + E: Release file for http://archive.ubuntu.com/ubuntu/dists/bionic-backports/InRelease is not valid yet (invalid for another 5h 41min 27s). Updates for this repository will not be applied. + The command '/bin/sh -c apt-get update && apt-get install -y --no-install-recommends cpio sudo python3-pip python3-setuptools libboost-filesystem1.65 libboost-thread1.65 libboost-program-options1.65 lsb-release libjson-c-dev' returned a non-zero code: 100 ``` - + **Solution**: + + make sure that system time to be synchronized among all nodes and controllers in a system. From 3913ac44a1fe66ec75a3da4348435c1d145eb225 Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Mon, 31 Aug 2020 16:30:35 +0530 Subject: [PATCH 49/80] added script and steps for sending external rtsp stream data (#111) --- applications/eis-experience-kit/README.md | 81 +++++++++++++++++++ .../scripts/send_rtsp_stream_linux.sh | 27 +++++++ .../scripts/send_rtsp_stream_win.bat | 15 ++++ 3 files changed, 123 insertions(+) create mode 100755 applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh create mode 100755 applications/eis-experience-kit/scripts/send_rtsp_stream_win.bat diff --git a/applications/eis-experience-kit/README.md b/applications/eis-experience-kit/README.md index be8d92a7..ecf47958 100644 --- a/applications/eis-experience-kit/README.md +++ b/applications/eis-experience-kit/README.md @@ -26,6 +26,9 @@ Currently, `eis-experience-kit` supports EIS in version 2.2. - [Deploy Settings](#deploy-settings) - [Inventory](#inventory) - [Playbook Main File](#playbook-main-file) + - [EIS Demo Setting](#eis-demo-setting) + - [RTSP Stream Setting](#rtsp-stream-setting) + - [View Visualizer Setting](#view-visualizer-setting) - [Installation](#installation) - [Removal](#removal) - [References](#references) @@ -81,6 +84,84 @@ User needs to set the OpenNESS Master Node IP address. It can be done in `invent ### Playbook Main File The main file for playbook is `eis_pcb_demo.yml`. User can define here which roles should be run during the build & deployment. They can be switch by using comments for unnecessary roles. +### EIS Demo Setting +eis-experience-kit currently we can configure for Demo type as + +- PCB Demo +- Safety Demo + +Following flags controll for configuring demo type on `group_vars/all.yml +```sh +demo_type: "safety" -> for Safety Demo +demo_type: "pcb" -> for PCB Demo +``` +#### RTSP Stream Setting +Currently RTSP camera steam data can be received follwing source + - rtsp stream from camera-stream pod + - rtsp stream from Linux host + - rtsp stream from Window host + +on eis-experience-kit demo default rtsp strem will recive from camera-stream pod. +Follwing flags are contrl for receving receiving rtsp strem on `group_vars/all.yml` + +#### Enable rtsp stream from camera-stream pod(Default) +```sh + camera_stream_pod: true + rtsp_camera_stream_ip: "ia-camera-stream-service" + rtsp_camera_stream_port: 8554 +``` +#### Enable rtsp stream from extrnal Linux/Window host + ```sh + camera_stream_pod: false + rtsp_camera_stream_ip: "192.169.1.1" < update Linux/window external rtsp server IP> + rtsp_camera_stream_port: 8554 +``` + +#### Send rtsp stream from external Linux (CentOS) +```sh +yum install -y epel-release https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm +yum install -y vlc +sed -i 's/geteuid/getppid/' /usr/bin/vlc +./send_rtsp_stream_linux.sh +``` +#### Send rtsp stream from external Linux (ubuntu) + +```sh +apt-get install vlc +sed -i 's/geteuid/getppid/' /usr/bin/vlc +./send_rtsp_stream_linux.sh +``` +#### Send rtsp stream from external Windows Host + +```sh +#install vlc player +#update follwing varaible on send_rtsp_stream_linux.sh +set vlc_path="c:\Program Files (x86)\VideoLAN\VLC" +set file_name="c:\Data\Safety_Full_Hat_and_Vest.avi" < update Demo file name> +set port="8554" +#next run send_rtsp_stream_win.bat file +``` + +**Note**: Following script and demo video file should copied from ansible host machine + + `/eis-experience-kit/scripts/send_rtsp_stream_linux.sh` + `/eis-experience-kit/scripts/send_rtsp_stream_win.bat` + `/opt/eis_repo/IEdgeInsights/VideoIngestion/test_videos/pcb_d2000.avi` + `/opt/eis_repo/IEdgeInsights/VideoIngestion/test_videos/Safety_Full_Hat_and_Vest.avi` + + +### View Visualizer Setting + +Update IP Adddress of host where we want to see the GUI ouput, Visualizer container will expose the GUI output on display host. + +display_host_ip: "192.168.0.1" < Update Display Host IP> +display_no: "1" + +**Note**: +- Display host shoud have GUI/VNC access and check the Display by echo $DISPLAY +update the display on above `display_no` . +- configure `xhost +` on Display host for receiving video GUI + ## Installation After all the configuration is done, script `deploy_eis_pcb_demo.sh` needs to be executed to start the deployment process. No more actions are required, all the installation steps are fully automated. diff --git a/applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh b/applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh new file mode 100755 index 00000000..52573b3c --- /dev/null +++ b/applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +set -eu + +helpPrint() +{ + echo "" + echo "Usage: $0 " + echo -e "Example:" + echo -e " $0 /root/pcb_d2000.avi 8554" + echo -e " $0 /root/Safety_Full_Hat_and_Vest.avi 8554" + exit 1 # Exit with help +} + +if [[ $# -ne 2 ]] ; then + echo "No of argument are not correct" + helpPrint +fi +name=$1 +port=$2 + +pkill -9 cvlc +sleep 2 + +cvlc -vvv "$name" --sout ="#gather:rtp{sdp=rtsp://0.0.0.0:$port/}" --loop --sout-keep diff --git a/applications/eis-experience-kit/scripts/send_rtsp_stream_win.bat b/applications/eis-experience-kit/scripts/send_rtsp_stream_win.bat new file mode 100755 index 00000000..fecb1229 --- /dev/null +++ b/applications/eis-experience-kit/scripts/send_rtsp_stream_win.bat @@ -0,0 +1,15 @@ +::# SPDX-License-Identifier: Apache-2.0 +::# Copyright (c) 2020 Intel Corporation + +set vlc_path="c:\Program Files (x86)\VideoLAN\VLC" +set file_name="c:\Data\Safety_Full_Hat_and_Vest.avi" +set port="8554" + +echo "kill if any VLC process running" +taskkill /F /IM vlc.exe +timeout 5 + +echo "Start vlc rtsp stream" +cd %vlc_path% + +vlc.exe %file_name% --sout=#gather:rtp{sdp=rtsp://0.0.0.0:%port%/} --loop --sout-keep From ea44e7d9dba078fe9b4a0e7c915582555d54d816 Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Tue, 1 Sep 2020 10:02:56 +0200 Subject: [PATCH 50/80] CERA-1584: Automation of Getting EIS Images from Existing Cluster (#110) * CERA-1584: Automation of Getting EIS Images from Existing Cluster * Add missing etcd pod to the import list and fixed camera_stream image tagging * Removed unused functions + checking if ssh keys are setup on both ansible hosts + flag to skip building EIS base images * Undo proxy addresses replacement * Removed pcb_demo from scripts and playbooks names * Reversed port change in sample proxy address * Fixed shellckeck findings --- ..._pcb_demo.sh => cleanup_eis_deployment.sh} | 2 +- .../{deploy_eis_pcb_demo.sh => deploy_eis.sh} | 2 +- .../{eis_pcb_demo.yml => eis.yml} | 1 + ...s_pcb_demo_cleanup.yml => eis_cleanup.yml} | 0 .../host_vars/localhost.yml | 3 + .../roles/camera_stream/build/tasks/main.yml | 31 ++-- .../roles/etcd/build/tasks/main.yml | 6 + .../video_analytics/build/tasks/main.yml | 6 + .../video_ingestion/build/defaults/main.yml | 1 - .../video_ingestion/build/tasks/main.yml | 6 + .../roles/visualizer/build/tasks/main.yml | 6 + .../scripts/eis_import_images.sh | 134 ++++++++++++++++++ 12 files changed, 184 insertions(+), 14 deletions(-) rename applications/eis-experience-kit/{cleanup_eis_pcb_demo.sh => cleanup_eis_deployment.sh} (76%) rename applications/eis-experience-kit/{deploy_eis_pcb_demo.sh => deploy_eis.sh} (85%) rename applications/eis-experience-kit/{eis_pcb_demo.yml => eis.yml} (93%) rename applications/eis-experience-kit/{eis_pcb_demo_cleanup.yml => eis_cleanup.yml} (100%) create mode 100644 applications/eis-experience-kit/scripts/eis_import_images.sh diff --git a/applications/eis-experience-kit/cleanup_eis_pcb_demo.sh b/applications/eis-experience-kit/cleanup_eis_deployment.sh similarity index 76% rename from applications/eis-experience-kit/cleanup_eis_pcb_demo.sh rename to applications/eis-experience-kit/cleanup_eis_deployment.sh index 3608483a..6abd3efd 100755 --- a/applications/eis-experience-kit/cleanup_eis_pcb_demo.sh +++ b/applications/eis-experience-kit/cleanup_eis_deployment.sh @@ -5,5 +5,5 @@ source scripts/ansible-precheck.sh # check if ansibles are already installed # and ready to use -ansible-playbook -vv ./eis_pcb_demo_cleanup.yml --inventory inventory.ini +ansible-playbook -vv ./eis_cleanup.yml --inventory inventory.ini exit $? diff --git a/applications/eis-experience-kit/deploy_eis_pcb_demo.sh b/applications/eis-experience-kit/deploy_eis.sh similarity index 85% rename from applications/eis-experience-kit/deploy_eis_pcb_demo.sh rename to applications/eis-experience-kit/deploy_eis.sh index 69902439..c8d53917 100755 --- a/applications/eis-experience-kit/deploy_eis_pcb_demo.sh +++ b/applications/eis-experience-kit/deploy_eis.sh @@ -10,5 +10,5 @@ ansible-playbook -vv ./eis_sources.yml --inventory inventory.ini source scripts/eis_repo_config.sh . -ansible-playbook -vv ./eis_pcb_demo.yml --inventory inventory.ini +ansible-playbook -vv ./eis.yml --inventory inventory.ini exit $? diff --git a/applications/eis-experience-kit/eis_pcb_demo.yml b/applications/eis-experience-kit/eis.yml similarity index 93% rename from applications/eis-experience-kit/eis_pcb_demo.yml rename to applications/eis-experience-kit/eis.yml index 1c01348d..f61be09f 100644 --- a/applications/eis-experience-kit/eis_pcb_demo.yml +++ b/applications/eis-experience-kit/eis.yml @@ -9,6 +9,7 @@ - role: docker - role: docker_registry/localhost - role: common/build + when: build_eis_base_images | default(True) - role: camera_stream/build - role: etcd/build - role: video_ingestion/build diff --git a/applications/eis-experience-kit/eis_pcb_demo_cleanup.yml b/applications/eis-experience-kit/eis_cleanup.yml similarity index 100% rename from applications/eis-experience-kit/eis_pcb_demo_cleanup.yml rename to applications/eis-experience-kit/eis_cleanup.yml diff --git a/applications/eis-experience-kit/host_vars/localhost.yml b/applications/eis-experience-kit/host_vars/localhost.yml index faf92534..9c498c0e 100644 --- a/applications/eis-experience-kit/host_vars/localhost.yml +++ b/applications/eis-experience-kit/host_vars/localhost.yml @@ -26,3 +26,6 @@ release_package_path: "" # provide release package archive path when eis_sour eis_env_file: "{{ eis_sources_dir }}/build/.env" os_yum_base_packages: "jq,vim-common,curl,yum-utils,python2-pip,wget,git,python-devel,moreutils" + +docker_format_command: !unsafe 'docker images --format "{{.Repository}}:{{.Tag}}"' +build_eis_base_images: true diff --git a/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml b/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml index a224c353..c3cc1b6b 100644 --- a/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/camera_stream/build/tasks/main.yml @@ -3,19 +3,28 @@ --- -- name: Copy pcb streaming file from EIS repo to folder having Dockerfile - copy: - src: "{{ pcb_cam_stream_source }}" - dest: "{{ docker_src_folder }}" +- name: Check if camera_stream image is already built + shell: "{{ docker_format_command }} | grep {{ docker_image_tag }}" + register: image_built + ignore_errors: true -- name: Copy Safety Hat streaming file from EIS repo to folder having Dockerfile - copy: - src: "{{ safety_cam_stream_source }}" - dest: "{{ docker_src_folder }}" +- name: Prepare camera_stream image build + block: + - name: Copy pcb streaming file from EIS repo to folder having Dockerfile + copy: + src: "{{ pcb_cam_stream_source }}" + dest: "{{ docker_src_folder }}" + - name: Copy Safety Hat streaming file from EIS repo to folder having Dockerfile + copy: + src: "{{ safety_cam_stream_source }}" + dest: "{{ docker_src_folder }}" + - name: Build {{ docker_image_name }} docker image + command: "docker build --network=host -t {{ docker_image_tag }} roles/camera_stream/build/files" + changed_when: true + when: image_built is failed -- name: Build {{ docker_image_name }} docker image - command: "docker build --network=host -t {{ docker_image_tag }} -t {{ docker_registry_address }}/{{ docker_image_tag }} roles/camera_stream/build/files" - changed_when: true +- name: Tag camera_stream image + command: docker tag {{ docker_image_tag }} {{ docker_registry_address }}/{{ docker_image_tag }} - name: Push the image to the Docker Registry command: docker push {{ docker_registry_address }}/{{ docker_image_tag }} diff --git a/applications/eis-experience-kit/roles/etcd/build/tasks/main.yml b/applications/eis-experience-kit/roles/etcd/build/tasks/main.yml index acfcd299..31f36aa9 100644 --- a/applications/eis-experience-kit/roles/etcd/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/etcd/build/tasks/main.yml @@ -3,9 +3,15 @@ --- +- name: Check if ETCD image is already built + shell: "{{ docker_format_command }} | grep {{ docker_image_tag }}" + register: image_built + ignore_errors: true + - name: Build ETCD Docker Container command: "docker-compose -f {{ docker_compose_file }} build {{ docker_image_name }}" changed_when: true + when: image_built is failed - name: Add tag to the ETCD image command: "docker tag {{ docker_image_tag }} {{ docker_registry_address }}/{{ docker_image_tag }}" diff --git a/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml b/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml index 2dcf941a..8b65ad47 100644 --- a/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_analytics/build/tasks/main.yml @@ -3,11 +3,17 @@ --- +- name: Check if VideoAnalytics image is already built + shell: "{{ docker_format_command }} | grep {{ docker_image_tag }}" + register: image_built + ignore_errors: true + - name: Build {{ docker_image_name }} docker image shell: docker-compose build {{ docker_image_name }} args: chdir: "{{ eis_sources_dir }}/build" changed_when: true + when: image_built is failed - name: Add tag to the {{ docker_image_name }} image command: "docker tag {{ docker_image_tag }} {{ docker_registry_address }}/{{ docker_image_tag }}" diff --git a/applications/eis-experience-kit/roles/video_ingestion/build/defaults/main.yml b/applications/eis-experience-kit/roles/video_ingestion/build/defaults/main.yml index 9d4fc264..883d27b4 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/build/defaults/main.yml +++ b/applications/eis-experience-kit/roles/video_ingestion/build/defaults/main.yml @@ -5,4 +5,3 @@ docker_image_name: "ia_video_ingestion" docker_image_tag: "{{ docker_image_name }}:{{ eis_version }}" - diff --git a/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml b/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml index 2dcf941a..0c1718cf 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/video_ingestion/build/tasks/main.yml @@ -3,11 +3,17 @@ --- +- name: Check if VideoIngestion image is already built + shell: "{{ docker_format_command }} | grep {{ docker_image_tag }}" + register: image_built + ignore_errors: true + - name: Build {{ docker_image_name }} docker image shell: docker-compose build {{ docker_image_name }} args: chdir: "{{ eis_sources_dir }}/build" changed_when: true + when: image_built is failed - name: Add tag to the {{ docker_image_name }} image command: "docker tag {{ docker_image_tag }} {{ docker_registry_address }}/{{ docker_image_tag }}" diff --git a/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml b/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml index 2dcf941a..e0a6140d 100644 --- a/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml +++ b/applications/eis-experience-kit/roles/visualizer/build/tasks/main.yml @@ -3,11 +3,17 @@ --- +- name: Check if Visualizer image is already built + shell: "{{ docker_format_command }} | grep {{ docker_image_tag }}" + register: image_built + ignore_errors: true + - name: Build {{ docker_image_name }} docker image shell: docker-compose build {{ docker_image_name }} args: chdir: "{{ eis_sources_dir }}/build" changed_when: true + when: image_built is failed - name: Add tag to the {{ docker_image_name }} image command: "docker tag {{ docker_image_tag }} {{ docker_registry_address }}/{{ docker_image_tag }}" diff --git a/applications/eis-experience-kit/scripts/eis_import_images.sh b/applications/eis-experience-kit/scripts/eis_import_images.sh new file mode 100644 index 00000000..19a88b01 --- /dev/null +++ b/applications/eis-experience-kit/scripts/eis_import_images.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +if [[ "${#}" -ne 2 ]]; then + echo "Wrong arguments. Usage: ${0} " + exit 1 +fi + +ansible_host_ssh_address=$1 +eis_version=$2 +images_to_import=( + "ia_camera_stream:${eis_version}" + "ia_etcd:${eis_version}" + "ia_video_ingestion:${eis_version}" + "ia_video_analytics:${eis_version}" + "ia_visualizer:${eis_version}" +) + +export_docker_image () { + # $1 - Docker image repository and tag + echo "Exporting image ${1}..." + + # shellcheck disable=SC2029 + # variable ${1} is expected to be expanded on the client side + ssh "${ansible_host_ssh_address}" "if [ ! -f /tmp/eis_images_export/${1}.tar ]; then docker save -o /tmp/eis_images_export/${1}.tar ${1}; fi" + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not export image ${1}." + exit 1 + fi + echo "Exporting completed" +} + +download_docker_image () { +# $1 - Docker image repository and tag + echo "Downloading image ${1}..." + if [[ ! -f "/tmp/eis_images_import/${1}.tar" ]]; then + scp "${ansible_host_ssh_address}:/tmp/eis_images_export/${1}.tar" "/tmp/eis_images_import/" + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not download image ${1}." + exit 1 + fi + echo "Downloading completed" + else + echo "File /tmp/eis_images_import/${1} already exists, skipping..." + fi +} + +import_docker_image () { +# $1 - Docker image repository and tag + echo "Importing image ${1}..." + docker load -i "/tmp/eis_images_import/${1}.tar" + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not import image ${1}." + exit 1 + fi + echo "Importing completed" +} + +# Check root permissions +id -u 1>/dev/null +if [[ $? -ne 0 ]]; then + echo "ERROR: Script requires root permissions" + exit 1 +fi + +echo "Check if ssh connection is setup using ssh-keys" +ssh -o BatchMode=yes "${ansible_host_ssh_address}" 'exit' +if [[ ! $? -eq 0 ]]; then + echo "Connection is not prepared correctly. Please setup ssh connection to the old ansible" \ + "host before running this script." + exit 1 +fi + +# Install Docker if missing +command -v docker 1>/dev/null +if [[ $? -ne 0 ]]; then + echo "Docker not installed..." + yum install -y yum-utils + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not install yum-utils package." + exit 1 + fi + yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not add Docker CE repository. Check above for possible root cause" + exit 1 + fi + + echo "Installing docker-ce..." + yum install -y docker-ce docker-ce-cli containerd.io + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not install docker-ce packages" + exit 1 + fi +fi + +echo "Check if docker service is running and start if not" +command systemctl is-active --quiet docker.service +if [[ $? -ne 0 ]]; then + echo "Docker service is not running, try to start..." + command systemctl start docker + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not start Docker service. Exiting..." + exit 1 + fi +fi + +# Export Docker images on other Ansible host +ssh "${ansible_host_ssh_address}" "if [ ! -d /tmp/eis_images_export ]; then mkdir /tmp/eis_images_export; fi" +if [[ $? -ne 0 ]]; then + echo "ERROR: Could not create temporary folder /tmp/eis_images_export on remote" + exit 1 +fi +for pod_name in "${images_to_import[@]}"; do + export_docker_image "${pod_name}" +done + +# Download images to the current Ansible host +if [[ ! -d "/tmp/eis_images_import" ]]; then + mkdir /tmp/eis_images_import + if [[ $? -ne 0 ]]; then + echo "ERROR: Could not create temporary folder /tmp/eis_images_import" + exit 1 + fi +fi +for pod_name in "${images_to_import[@]}"; do + download_docker_image "${pod_name}" +done + +# Import Docker images on the current Ansible host +for pod_name in "${images_to_import[@]}"; do + import_docker_image "${pod_name}" +done From 96e0c76b59cab65b403d5318719bb9e227afcee2 Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Wed, 2 Sep 2020 09:58:54 +0800 Subject: [PATCH 51/80] Modify Troubleshooting section in README.md file --- applications/openvino/benchmark/README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/applications/openvino/benchmark/README.md b/applications/openvino/benchmark/README.md index 3a8fdc69..d0658a15 100644 --- a/applications/openvino/benchmark/README.md +++ b/applications/openvino/benchmark/README.md @@ -74,11 +74,16 @@ In this section some issues in implementation process are covered ```sh ... ... + E: Release file for http://security.ubuntu.com/ubuntu/dists/bionic-security/InRelease is not valid yet (invalid for another 5h 38min 40s). Updates for this repository will not be applied. + E: Release file for http://archive.ubuntu.com/ubuntu/dists/bionic-updates/InRelease is not valid yet (invalid for another 5h 39min 53s). Updates for this repository will not be applied. E: Release file for http://archive.ubuntu.com/ubuntu/dists/bionic-backports/InRelease is not valid yet (invalid for another 5h 41min 27s). Updates for this repository will not be applied. - The command '/bin/sh -c apt-get update && apt-get install -y --no-install-recommends cpio sudo python3-pip python3-setuptools libboost-filesystem1.65 libboost-thread1.65 libboost-program-options1.65 lsb-release libjson-c-dev' returned a non-zero code: 100 +The command '/bin/sh -c apt-get update && apt-get install -y --no-install-recommends cpio sudo python3-pip python3-setuptools libboost-filesystem1.65 libboost-thread1.65 libboost-program-options1.65 lsb-release libjson-c-dev' returned a non-zero code: 100 ``` - **Solution**: - - make sure that system time to be synchronized among all nodes and controllers in a system. - + Invalid time causes the server to be unable to use Ubuntu’s packaging system apt . + + > **The solution is to synchronize your clock manually or use a service (the better way)!** + + **Reference:** + + https://ahelpme.com/linux/ubuntu/ubuntu-apt-inrelease-is-not-valid-yet-invalid-for-another-151d-18h-5min-59s/ From c3273be30e492b026a020f812e1c20197d6a94aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Wed, 2 Sep 2020 17:37:12 +0200 Subject: [PATCH 52/80] added network policy for EIS --- .../host_vars/openness_controller.yaml | 1 + .../deploy/files/eis_network_policy.yaml | 18 +++++++++++++++++ .../roles/common/deploy/tasks/main.yml | 16 +++++++++++++++ .../files/eis_network_policy.yaml | 18 +++++++++++++++++ .../roles/eis_network_policy/tasks/main.yml | 20 +++++++++++++++++++ 5 files changed, 73 insertions(+) create mode 100644 applications/eis-experience-kit/roles/common/deploy/files/eis_network_policy.yaml create mode 100644 applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml create mode 100644 applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml diff --git a/applications/eis-experience-kit/host_vars/openness_controller.yaml b/applications/eis-experience-kit/host_vars/openness_controller.yaml index 80013cb2..e29815e3 100644 --- a/applications/eis-experience-kit/host_vars/openness_controller.yaml +++ b/applications/eis-experience-kit/host_vars/openness_controller.yaml @@ -10,6 +10,7 @@ eis_install_path: "/opt/eis" helm_charts_location: "{{ eis_install_path }}/charts" eis_integ_path: "{{ eis_install_path }}/etcd/eis_integ" etcd_certs_location: "{{ eis_install_path }}/etcd/Certificates" +eis_netork_policy_path: "{{ eis_install_path }}/network_policy" zmq_keys_gen_force: false del_zmq_keys: false diff --git a/applications/eis-experience-kit/roles/common/deploy/files/eis_network_policy.yaml b/applications/eis-experience-kit/roles/common/deploy/files/eis_network_policy.yaml new file mode 100644 index 00000000..7a83e41a --- /dev/null +++ b/applications/eis-experience-kit/roles/common/deploy/files/eis_network_policy.yaml @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-in-eis +spec: +# this selects all pods from choosen namespace (by -n option) + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: +# this allows ingress from all pods from choosen namespace (by -n option) + - podSelector: {} diff --git a/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml index 6ab3f0a9..2179a286 100644 --- a/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml @@ -12,3 +12,19 @@ - name: Create namespace {{ k8s_eis_namespace }} command: kubectl create namespace {{ k8s_eis_namespace }} when: get_ns_eis.rc == 1 + +- name: Create directory for network policy + file: + path: {{ eis_netork_policy_path }} + state: directory + recurse: yes + +- name: Copy network policy to host + copy: + src: "eis_network_policy.yaml" + dest: "{{ eis_netork_policy_path }}" + +- name: Apply EIS network policy + command: kubectl apply -n {{ k8s_eis_namespace }} -f eis_network_policy.yaml + args: + chdir: "{{ eis_netork_policy_path }}" diff --git a/applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml b/applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml new file mode 100644 index 00000000..7a83e41a --- /dev/null +++ b/applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-in-eis +spec: +# this selects all pods from choosen namespace (by -n option) + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: +# this allows ingress from all pods from choosen namespace (by -n option) + - podSelector: {} diff --git a/applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml b/applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml new file mode 100644 index 00000000..e1c4f108 --- /dev/null +++ b/applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +- name: Create directory for network policy + file: + path: {{ eis_netork_policy_path }} + state: directory + recurse: yes + +- name: Copy network policy to host + copy: + src: "eis_network_policy.yaml" + dest: "{{ cera_charts_path }}" + +- name: Apply EIS network policy + command: kubectl apply -n {{ k8s_eis_namespace }} -f eis_network_policy.yaml + args: + chdir: "{{ eis_netork_policy_path }}" From 9fc5908f8a14f817b1b038030b9a004dc26c87d8 Mon Sep 17 00:00:00 2001 From: lukaszXlesiecki <67093702+lukaszXlesiecki@users.noreply.github.com> Date: Wed, 2 Sep 2020 17:44:15 +0200 Subject: [PATCH 53/80] Delete main.yml --- .../roles/eis_network_policy/tasks/main.yml | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml diff --git a/applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml b/applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml deleted file mode 100644 index e1c4f108..00000000 --- a/applications/eis-experience-kit/roles/eis_network_policy/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - ---- - -- name: Create directory for network policy - file: - path: {{ eis_netork_policy_path }} - state: directory - recurse: yes - -- name: Copy network policy to host - copy: - src: "eis_network_policy.yaml" - dest: "{{ cera_charts_path }}" - -- name: Apply EIS network policy - command: kubectl apply -n {{ k8s_eis_namespace }} -f eis_network_policy.yaml - args: - chdir: "{{ eis_netork_policy_path }}" From fa104040d7294f22fb3ae5f6e8bf8551d0ee3881 Mon Sep 17 00:00:00 2001 From: lukaszXlesiecki <67093702+lukaszXlesiecki@users.noreply.github.com> Date: Wed, 2 Sep 2020 17:44:41 +0200 Subject: [PATCH 54/80] Delete eis_network_policy.yaml --- .../files/eis_network_policy.yaml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml diff --git a/applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml b/applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml deleted file mode 100644 index 7a83e41a..00000000 --- a/applications/eis-experience-kit/roles/eis_network_policy/files/eis_network_policy.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2020 Intel Corporation - ---- - -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-ingress-in-eis -spec: -# this selects all pods from choosen namespace (by -n option) - podSelector: {} - policyTypes: - - Ingress - ingress: - - from: -# this allows ingress from all pods from choosen namespace (by -n option) - - podSelector: {} From 2eb563808345b9beafbf1f3be510424d1aedbb46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Lesiecki?= Date: Fri, 4 Sep 2020 12:42:39 +0200 Subject: [PATCH 55/80] typos fixed --- .../eis-experience-kit/host_vars/openness_controller.yaml | 2 +- .../eis-experience-kit/roles/common/deploy/tasks/main.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/applications/eis-experience-kit/host_vars/openness_controller.yaml b/applications/eis-experience-kit/host_vars/openness_controller.yaml index e29815e3..7d800816 100644 --- a/applications/eis-experience-kit/host_vars/openness_controller.yaml +++ b/applications/eis-experience-kit/host_vars/openness_controller.yaml @@ -10,7 +10,7 @@ eis_install_path: "/opt/eis" helm_charts_location: "{{ eis_install_path }}/charts" eis_integ_path: "{{ eis_install_path }}/etcd/eis_integ" etcd_certs_location: "{{ eis_install_path }}/etcd/Certificates" -eis_netork_policy_path: "{{ eis_install_path }}/network_policy" +eis_network_policy_path: "{{ eis_install_path }}/network_policy" zmq_keys_gen_force: false del_zmq_keys: false diff --git a/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml index 2179a286..317f7bd7 100644 --- a/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml @@ -15,16 +15,16 @@ - name: Create directory for network policy file: - path: {{ eis_netork_policy_path }} + path: {{ eis_network_policy_path }} state: directory recurse: yes - name: Copy network policy to host copy: src: "eis_network_policy.yaml" - dest: "{{ eis_netork_policy_path }}" + dest: "{{ eis_network_policy_path }}" - name: Apply EIS network policy command: kubectl apply -n {{ k8s_eis_namespace }} -f eis_network_policy.yaml args: - chdir: "{{ eis_netork_policy_path }}" + chdir: "{{ eis_network_policy_path }}" From 0d2a0e21c19d46adade0b5b200ae3b9933c2ee9d Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Mon, 7 Sep 2020 16:09:15 +0800 Subject: [PATCH 56/80] Add other data model --- applications/openvino/benchmark/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/applications/openvino/benchmark/Dockerfile b/applications/openvino/benchmark/Dockerfile index 635a1930..51c2938f 100644 --- a/applications/openvino/benchmark/Dockerfile +++ b/applications/openvino/benchmark/Dockerfile @@ -45,6 +45,17 @@ RUN pip3 install networkx==2.3 WORKDIR /opt/intel/openvino/deployment_tools/demo RUN /bin/bash -c "source $INSTALL_DIR/bin/setupvars.sh && ./demo_squeezenet_download_convert_run.sh" +#Download OpenVINO pre-trained models + +RUN pip3 install pyyaml requests +WORKDIR /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader +RUN ./downloader.py --name face-detection-retail-0004 +RUN ./downloader.py --name squeezenet1.1 +RUN ./downloader.py --name semantic-segmentation-adas-0001 +RUN ./downloader.py --name ssd300 +RUN ./downloader.py --name ssd512 +RUN /bin/bash -c "source $INSTALL_DIR/bin/setupvars.sh && ./converter.py --name ssd300" +RUN /bin/bash -c "source $INSTALL_DIR/bin/setupvars.sh && ./converter.py --name ssd512" WORKDIR /root/inference_engine_samples_build RUN /bin/bash -c "source $INSTALL_DIR/bin/setupvars.sh && make" From 955d9efafd8f33145fb6172ea85ba7486150c6c0 Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Mon, 7 Sep 2020 19:24:55 +0530 Subject: [PATCH 57/80] updated EIS latest tag v2.3.1 (#118) * changes added for eis v2.3.1 PV release --- .../eis-experience-kit/group_vars/all.yml | 2 +- .../roles/eis_sources/defaults/main.yml | 2 +- .../roles/eis_sources/tasks/main.yml | 15 +++++++++++---- .../deploy/templates/video_ingestion_pcb.json.j2 | 2 +- .../templates/video_ingestion_safety.json.j2 | 2 +- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/applications/eis-experience-kit/group_vars/all.yml b/applications/eis-experience-kit/group_vars/all.yml index f8cdbfe1..683b8174 100644 --- a/applications/eis-experience-kit/group_vars/all.yml +++ b/applications/eis-experience-kit/group_vars/all.yml @@ -18,7 +18,7 @@ eis_repo_dir: "/opt/eis_repo" eis_sources_dir: "/opt/eis_repo/IEdgeInsights" -demo_type: "safety" # set "pcb" for pcp demo or "safety" for Safety Hat demo. +demo_type: "pcb" # set "pcb" for pcp demo or "safety" for Safety Hat demo. camera_stream_pod: true # enabled if rtsp stream receive from camera stream pod rtsp_camera_stream_ip: "ia-camera-stream-service" # set remote rtsp camera IP diff --git a/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml b/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml index 2b7ef59d..ead95cb1 100644 --- a/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml +++ b/applications/eis-experience-kit/roles/eis_sources/defaults/main.yml @@ -4,5 +4,5 @@ --- repo_tool: "https://storage.googleapis.com/git-repo-downloads/repo" eis_repo_url: "ssh://git@gitlab.devtools.intel.com:29418/Indu/IEdgeInsights/eis-manifests" -eis_repo_branch: "refs/tags/v2.3-Alpha-RC1" +eis_repo_branch: "refs/tags/v2.3.1" diff --git a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml index 9ea5ec14..68db25cd 100644 --- a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml +++ b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml @@ -9,7 +9,7 @@ state: directory mode: '0755' -- name: download repo tool for multi-repo codebase checkout. +- name: download repo tool for multi-repo codebase checkout shell: "{{ item }}" with_items: - curl {{ repo_tool }} > repo @@ -26,7 +26,6 @@ with_items: - repo init -u "{{ eis_repo_url }}" -b {{ eis_repo_branch }} - cd .repo/manifests; repo init -m video.xml; repo sync - - cd IEdgeInsights/build; pip3 install -r requirements.txt; python3 eis_builder.py args: chdir: "{{ eis_repo_dir }}" changed_when: true @@ -47,8 +46,8 @@ - name: Move source directory to the expected location shell: >- package_dir=({{ temp_dir.path }}/*) && - sources_dir={{ temp_dir.path }}/$(basename $package_dir)/IEdgeInsights && - mv $sources_dir/* {{ eis_sources_dir }} + sources_dir={{ temp_dir.path }}/$(basename $package_dir)/ && + mv $sources_dir/* {{ eis_repo_dir }} changed_when: true - name: Remove temporary directory file: @@ -56,3 +55,11 @@ state: absent when: temp_dir.path is defined when: eis_source == "release" + +- name: generate eis config build file + shell : "{{ item }}" + with_items: + - cd IEdgeInsights/build; pip3 install -r requirements.txt; python3 eis_builder.py + args: + chdir: "{{ eis_repo_dir }}" + changed_when: true diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 index d1188529..b3573ef9 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_pcb.json.j2 @@ -7,7 +7,7 @@ "ingestor": { "type": "opencv", "pipeline": "rtspsrc location=\"rtsp://{{ rtsp_camera_stream_ip }}:{{ rtsp_camera_stream_port }}/\" latency=100 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! appsink max_buffers=100 drop=true", - "loop_video": "true", + "loop_video": true, "queue_size": 10, "poll_interval": 0.2 }, diff --git a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 index feaf8879..39a6b668 100644 --- a/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 +++ b/applications/eis-experience-kit/roles/video_ingestion/deploy/templates/video_ingestion_safety.json.j2 @@ -7,7 +7,7 @@ "ingestor": { "type": "opencv", "pipeline": "rtspsrc location=\"rtsp://{{ rtsp_camera_stream_ip }}:{{ rtsp_camera_stream_port }}/\" latency=100 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! appsink max_buffers=100 drop=true", - "loop_video": "true", + "loop_video": true, "queue_size": 10, "poll_interval": 0.2 }, From d3e66f1897546d850dd09e34288db5d969d84ee5 Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Tue, 8 Sep 2020 21:42:57 +0530 Subject: [PATCH 58/80] integrated eis WebVisualizer (#120) --- applications/eis-experience-kit/eis.yml | 3 + .../eis-experience-kit/group_vars/all.yml | 14 ++-- .../web_visualizer/build/defaults/main.yml | 7 ++ .../web_visualizer/build/tasks/cleanup.yml | 9 ++ .../roles/web_visualizer/build/tasks/main.yml | 24 ++++++ .../web_visualizer/deploy/defaults/main.yml | 17 ++++ .../deploy/files/web-visualizer/Chart.yaml | 10 +++ .../templates/web-visualizer-etcd-secret.yaml | 12 +++ .../templates/web-visualizer-pod.yaml | 63 ++++++++++++++ .../templates/web-visualizer-service.yaml | 19 +++++ .../deploy/files/web-visualizer/values.yaml | 14 ++++ .../web_visualizer/deploy/tasks/cleanup.yaml | 42 ++++++++++ .../web_visualizer/deploy/tasks/main.yml | 83 +++++++++++++++++++ .../templates/web-visualizer-values.yaml.j2 | 18 ++++ .../deploy/templates/web_access.sh.j2 | 37 +++++++++ .../templates/web_visualizer_pcb.json.j2 | 13 +++ .../templates/web_visualizer_safety.json.j2 | 15 ++++ 17 files changed, 395 insertions(+), 5 deletions(-) create mode 100644 applications/eis-experience-kit/roles/web_visualizer/build/defaults/main.yml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/build/tasks/cleanup.yml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/build/tasks/main.yml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/defaults/main.yml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/Chart.yaml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-etcd-secret.yaml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-pod.yaml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-service.yaml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/values.yaml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/cleanup.yaml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/main.yml create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web-visualizer-values.yaml.j2 create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_access.sh.j2 create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_pcb.json.j2 create mode 100644 applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_safety.json.j2 diff --git a/applications/eis-experience-kit/eis.yml b/applications/eis-experience-kit/eis.yml index f61be09f..039c05ee 100644 --- a/applications/eis-experience-kit/eis.yml +++ b/applications/eis-experience-kit/eis.yml @@ -15,6 +15,7 @@ - role: video_ingestion/build - role: video_analytics/build - role: visualizer/build + - role: web_visualizer/build - hosts: controller_group roles: @@ -25,4 +26,6 @@ - role: etcd/deploy - role: video_analytics/deploy - role: visualizer/deploy + when: display_visualizer_host | default(False) + - role: web_visualizer/deploy - role: video_ingestion/deploy diff --git a/applications/eis-experience-kit/group_vars/all.yml b/applications/eis-experience-kit/group_vars/all.yml index 683b8174..d26c014b 100644 --- a/applications/eis-experience-kit/group_vars/all.yml +++ b/applications/eis-experience-kit/group_vars/all.yml @@ -10,6 +10,9 @@ docker_registry_ip: "{{ hostvars[groups['controller_group'][0]]['ansible_host'] docker_registry_address: "{{ docker_registry_ip }}:{{ docker_registry_port }}" docker_registry_dir: "/opt/docker-registry/" +web_visualizer_ip: "{{ hostvars[groups['controller_group'][0]]['ansible_host'] }}" +web_visualizer_port: 5050 + eis_version: "{{ lookup('env', 'EIS_VERSION') }}" etcd_version: "{{ lookup('env', 'ETCD_VERSION') }}" dev_mode: "{{ lookup('env', 'DEV_MODE') }}" @@ -18,12 +21,13 @@ eis_repo_dir: "/opt/eis_repo" eis_sources_dir: "/opt/eis_repo/IEdgeInsights" -demo_type: "pcb" # set "pcb" for pcp demo or "safety" for Safety Hat demo. -camera_stream_pod: true # enabled if rtsp stream receive from camera stream pod +demo_type: "pcb" # set "pcb" for pcp demo or "safety" for Safety Hat demo. +camera_stream_pod: true # enabled if rtsp stream receive from camera stream pod rtsp_camera_stream_ip: "ia-camera-stream-service" # set remote rtsp camera IP -rtsp_camera_stream_port: 8554 # set remote rtsp stream port no +rtsp_camera_stream_port: 8554 # set remote rtsp stream port no -display_host_ip: "192.168.0.1" # update ip for visualizer HOST GUI. -display_no: "1" # update DISPLAY number for visualizer GUI output +display_visualizer_host: false # enabled if visualizer output view on display host. +display_host_ip: "192.168.0.1" # update ip for visualizer HOST GUI. +display_no: "1" # update DISPLAY number for visualizer GUI output diff --git a/applications/eis-experience-kit/roles/web_visualizer/build/defaults/main.yml b/applications/eis-experience-kit/roles/web_visualizer/build/defaults/main.yml new file mode 100644 index 00000000..d73929c8 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/build/defaults/main.yml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +docker_image_name: "ia_web_visualizer" +docker_image_tag: "{{ docker_image_name }}:{{ eis_version }}" diff --git a/applications/eis-experience-kit/roles/web_visualizer/build/tasks/cleanup.yml b/applications/eis-experience-kit/roles/web_visualizer/build/tasks/cleanup.yml new file mode 100644 index 00000000..5d45afa8 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/build/tasks/cleanup.yml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +- name: Removing {{ docker_image_name }} docker image + command: docker rmi {{ docker_image_tag }} {{ docker_registry_address }}/{{ docker_image_tag }} + changed_when: true + ignore_errors: yes diff --git a/applications/eis-experience-kit/roles/web_visualizer/build/tasks/main.yml b/applications/eis-experience-kit/roles/web_visualizer/build/tasks/main.yml new file mode 100644 index 00000000..d336ca16 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/build/tasks/main.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- +- name: Check if WebVisualizer image is already built + shell: "{{ docker_format_command }} | grep {{ docker_image_tag }}" + register: image_built + ignore_errors: true + + +- name: Build {{ docker_image_name }} docker image + shell: docker-compose build {{ docker_image_name }} + args: + chdir: "{{ eis_sources_dir }}/build" + changed_when: true + when: image_built is failed + +- name: Add tag to the {{ docker_image_name }} image + command: "docker tag {{ docker_image_tag }} {{ docker_registry_address }}/{{ docker_image_tag }}" + changed_when: true + +- name: Push {{ docker_image_name }} image to Docker Registry + command: docker push {{ docker_registry_address }}/{{ docker_image_tag }} + changed_when: true diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/defaults/main.yml b/applications/eis-experience-kit/roles/web_visualizer/deploy/defaults/main.yml new file mode 100644 index 00000000..ca9d9bd0 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/defaults/main.yml @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +docker_name: "ia_web_visualizer" +docker_image: "{{ docker_registry_address }}/{{ docker_name }}:{{ eis_version }}" +camera_stream_results_port: 65013 +app_name: "WebVisualizer" +peer: "client" +cert_name: "etcd_{{ app_name }}_cert" +key_name: "etcd_{{ app_name }}_key" +publisher_pod_service: "ia-video-analytics-service" +helm_release_name: "web-visualizer-release" +helm_chart_web_visualizer: "{{ helm_charts_location }}/web-visualizer" +etcdctl_bin_path: "/opt/eis/etcd/eis_integ/bin" +etcd_port: 2379 diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/Chart.yaml b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/Chart.yaml new file mode 100644 index 00000000..79120a3e --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/Chart.yaml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +apiVersion: v2 +name: ia-web-visualizer +description: A Helm chart for Web Visualizer app +type: application +version: 2.2 diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-etcd-secret.yaml b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-etcd-secret.yaml new file mode 100644 index 00000000..81069157 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-etcd-secret.yaml @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secrets.etcdCertsSecretName }} +type: Opaque +data: +{{ (.Files.Glob "secrets/*").AsSecrets | indent 2 }} diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-pod.yaml b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-pod.yaml new file mode 100644 index 00000000..ef72be8e --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-pod.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "{{ .Chart.Name }}" + labels: + name: "{{ .Chart.Name }}" +spec: + replicas: 1 + selector: + matchLabels: + app: "{{ .Chart.Name }}" + template: + metadata: + labels: + app: "{{ .Chart.Name }}" + spec: + hostname: ia-web-visualizer + containers: + - name: "{{ .Chart.Name }}" + image: {{ .Values.image }} + volumeMounts: + - mountPath: "{{ .Values.eisSecretsPath }}" + name: etcd-certs + readOnly: true + env: + - name: AppName + value: "{{ .Values.appName }}" + - name: DEV_MODE + value: "{{ .Values.devMode }}" + - name: PROFILING_MODE + value: "false" + - name: ETCD_HOST + value: ia-etcd-service + - name: CertType + value: "zmq,pem" + - name: ZMQ_RECV_HWM + value: "50" + - name: SubTopics + value: "VideoAnalytics/camera1_stream_results" + - name: camera1_stream_results_cfg + value: "zmq_tcp,{{ .Values.publisherPodService }}:{{ .Values.cameraStreamResultsPort }}" + - name: CONFIGMGR_CERT + value: "{{ .Values.certPath }}" + - name: CONFIGMGR_KEY + value: "{{ .Values.keyPath }}" + - name: CONFIGMGR_CACERT + value: "{{ .Values.rootCACertPath }}" + - name: no_proxy + value: "{{ .Values.eis_no_proxy }}" + hostNetwork: false + volumes: + - name: etcd-certs + projected: + sources: + - secret: + name: {{ .Values.secrets.etcdCertsSecretName }} + - secret: + name: {{ .Values.secrets.rootCaSecretName }} diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-service.yaml b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-service.yaml new file mode 100644 index 00000000..bc17e629 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/templates/web-visualizer-service.yaml @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +apiVersion: v1 +kind: Service +metadata: + name: "{{ .Values.service.name }}" +spec: + type: {{ .Values.service.type }} + selector: + app: {{ .Chart.Name }} + ports: + - name: webvisualizeripport + protocol: TCP + port: {{ .Values.webStreamPort }} + externalIPs: + - {{ .Values.webStreamIP }} diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/values.yaml b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/values.yaml new file mode 100644 index 00000000..69c63333 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/files/web-visualizer/values.yaml @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +service: + name: ia-web-visualizer-service + type: ClusterIP + +secrets: + etcdCertsSecretName: webvisualizer + rootCaSecretName: rootca + etcdCertName: etcd_WebVisualizer_cert + etcdKeyName: etcd_WebVisualizer_key diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/cleanup.yaml b/applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/cleanup.yaml new file mode 100644 index 00000000..1b152b52 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/cleanup.yaml @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +- name: Remove {{ app_name }} deployment + command: "helm -n {{ k8s_eis_namespace }} uninstall {{ helm_release_name }}" + ignore_errors: yes + changed_when: true + +- name: Remove {{ app_name }} Helm charts + file: + path: "{{ helm_chart_web_visualizer }}" + state: absent + ignore_errors: yes + +- name: Remove {{ app_name }} ETCD certificates + file: + path: "{{ etcd_certs_location }}/{{ app_name }}" + state: absent + ignore_errors: yes + +- name: Get node IP + shell: kubectl get nodes -o jsonpath="{.items[0].status.addresses[?(@.type==\"InternalIP\")].address}" + register: nodeip + changed_when: false + +- name: Wait for delete deployment + shell: ssh {{ nodeip.stdout }} "docker ps | grep {{ docker_registry_address }}/{{ docker_name }}" + register: result + until: result.rc == 1 + retries: 6 + delay: 10 + delegate_to: localhost + changed_when: false + ignore_errors: yes + +- name: Remove {{ app_name }} docker image from the node + shell: ssh {{ nodeip.stdout }} 'docker rmi -f $(docker images -q {{ docker_registry_address }}/{{ docker_name }}) && docker system prune --volumes -f' + delegate_to: localhost + changed_when: true + ignore_errors: yes diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/main.yml new file mode 100644 index 00000000..827026fc --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/tasks/main.yml @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +- name: Create {{ app_name }} user in ETCD + command: ./etcd_users.py --create {{ app_name }} + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + +- name: Copy {{ app_name }} config file + template: + src: "web_visualizer_{{ demo_type }}.json.j2" + dest: "{{ eis_integ_path }}/configs/web_visualizer.json" + mode: 0744 + +- name: Initialize {{ app_name }} config in ETCD + command: ./etcd_write.py ./configs/web_visualizer.json + args: + chdir: "{{ eis_integ_path }}" + environment: + ETCDCTL_CACERT: "{{ rootca_cert }}" + ETCDCTL_CERT: "{{ root_client_cert }}" + ETCDCTL_KEY: "{{ root_client_key }}" + changed_when: true + +- name: Generate {{ app_name }} client certificates + include_tasks: ./roles/common/deploy/tasks/gen_etcd_client_cert.yml + +- name: Generate ZMQ keys and put them to the ETCD + include_tasks: ./roles/common/deploy/tasks/gen_zmq_keys.yaml + +- name: Create folder for Helm Charts + file: + path: "{{ helm_charts_location }}" + state: directory + mode: '0744' + +- name: Copy {{ app_name }} Helm Charts to the Master Node + copy: + src: "web-visualizer" + dest: "{{ helm_charts_location }}" + +- name: Generate values file for {{ app_name }} Helm Chart from template + template: + src: web-visualizer-values.yaml.j2 + dest: "{{ helm_chart_web_visualizer }}/web-visualizer-values.yaml" + mode: 0744 + +- name: Create folder for {{ app_name }} ETCD certificates + file: + path: "{{ helm_chart_web_visualizer }}/secrets" + state: directory + mode: '0744' + +- name: Copy certificates to the Helm Chart directory + copy: + src: "{{ etcd_certs_location }}/{{ app_name }}/" + dest: "{{ helm_chart_web_visualizer }}/secrets" + remote_src: yes + +- name: Copy web_access.sh file + template: + src: "web_access.sh.j2" + dest: "{{ helm_chart_web_visualizer }}/secrets/web_access.sh" + mode: 0744 + +- name: write web access key and cert in ETCD + command: ./web_access.sh + args: + chdir: "{{ helm_chart_web_visualizer }}/secrets" + changed_when: true + +- name: Deploy {{ app_name }} using Helm + command: > + helm install --namespace {{ k8s_eis_namespace }} -f {{ helm_chart_web_visualizer }}/web-visualizer-values.yaml {{ helm_release_name }} + {{ helm_chart_web_visualizer }} + changed_when: true diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web-visualizer-values.yaml.j2 b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web-visualizer-values.yaml.j2 new file mode 100644 index 00000000..c5c75197 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web-visualizer-values.yaml.j2 @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +--- + +image: "{{ docker_registry_address }}/{{ docker_name }}:{{ eis_version }}" +dockerRegistryEndpoint: "{{ docker_registry_address }}" +cameraStreamResultsPort: {{ camera_stream_results_port }} +webStreamPort: {{ web_visualizer_port }} +webStreamIP: "{{ web_visualizer_ip }}" +devMode: "{{ dev_mode }}" +eisInstallPath: "{{ eis_install_path }}" +publisherPodService: "{{ publisher_pod_service }}" +appName: "{{ app_name }}" +eisSecretsPath: "{{ eis_secrets_path }}" +certPath: "{{ eis_secrets_path }}/{{ cert_name }}" +keyPath: "{{ eis_secrets_path }}/{{ key_name }}" +rootCACertPath: "{{ eis_secrets_path }}/{{ rootca_cert_name }}" diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_access.sh.j2 b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_access.sh.j2 new file mode 100644 index 00000000..eabf0c72 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_access.sh.j2 @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2020 Intel Corporation + +mkdir -p RootCA +mkdir -p Server +mkdir -p Client + +EtcdctlPath={{ etcdctl_bin_path }} + +echo "Generating RootCA Key and Cert for WebVisualizer access " +openssl ecparam -genkey -name secp384r1 -out "./RootCA/ca.key" +openssl req -key "./RootCA/ca.key" -new -x509 -days 1000 -subj "/CN=WebVisualizer" -out "./RootCA/ca.crt" + +echo "Generate Server Key and Cert for WebVisualizer access" +openssl req -new -sha256 -nodes -out ./Server/server.csr -newkey rsa:2048 -keyout ./Server/server.key -subj "/CN=Server" +openssl x509 -req -in ./Server/server.csr -CA ./RootCA/ca.crt -CAkey ./RootCA/ca.key -days 1000 -out ./Server/server.cert -CAcreateserial + +echo "Generate Client Key and Cert for WebVisualizer access" +openssl req -new -sha256 -nodes -out ./Client/client.csr -newkey rsa:2048 -keyout ./Client/client.key -subj "/CN=Client" +openssl x509 -req -in client.csr -CA ./RootCA/ca.crt -CAkey ./RootCA/ca.key -days 1000 -out ./Client/client.cert -CAcreateserial + +etcd_ip=$(kubectl -n eis get service ia-etcd-service | grep ia-etcd-service |awk '{print $3}') + +echo "Access etcd for writing WebVisualizer web access certificate" +export ETCDCTL_CACERT={{ rootca_cert }} +export ETCDCTL_CERT={{ root_client_cert }} +export ETCDCTL_KEY={{ root_client_key }} +export ETCDCTL_ENDPOINTS=https://$etcd_ip:{{ etcd_port }} + +cd $EtcdctlPath +./etcdctl get "/GlobalEnv/" +cat {{ helm_chart_web_visualizer }}/secrets/Server/server.key | ./etcdctl put /WebVisualizer/server_key +cat {{ helm_chart_web_visualizer }}/secrets/Server/server.cert | ./etcdctl put /WebVisualizer/server_cert +cat {{ helm_chart_web_visualizer }}/secrets/RootCA/ca.crt | ./etcdctl put /WebVisualizer/ca_cert + diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_pcb.json.j2 b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_pcb.json.j2 new file mode 100644 index 00000000..60639960 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_pcb.json.j2 @@ -0,0 +1,13 @@ +{ + "/WebVisualizer/config": { + "labels": { + "camera1_stream_results": { + "0": "MISSING", + "1": "SHORT" + } + }, + "password": "admin@123", + "port": {{ web_visualizer_port }}, + "username": "admin" + } +} diff --git a/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_safety.json.j2 b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_safety.json.j2 new file mode 100644 index 00000000..252520f4 --- /dev/null +++ b/applications/eis-experience-kit/roles/web_visualizer/deploy/templates/web_visualizer_safety.json.j2 @@ -0,0 +1,15 @@ +{ + "/WebVisualizer/config": { + "labels": { + "camera1_stream_results": { + "1": "safety_helmet", + "2": "safety_jacket", + "3": "Safe", + "4": "Violation" + } + }, + "password": "admin@123", + "port": {{ web_visualizer_port }}, + "username": "admin" + } +} From 45dec127469a86091d1968554a6782245ec77041 Mon Sep 17 00:00:00 2001 From: cjnolan <47635874+cjnolan@users.noreply.github.com> Date: Wed, 9 Sep 2020 12:10:34 +0100 Subject: [PATCH 59/80] Update Video Analytics Sample App to run with service mesh (#108) * Update Video Analytics Sample App to run with service mesh * Address review comments * Update sample app Helm chart to work with service mesh * Update VAS sample app to use net/http for VAS API calls * Add logic for handling network faults when using REST APIs * Fix gofmt issues * Fix parsing of pipeline ID from HTTP Response * Update README --- applications/vas-sample-app/README.md | 52 ++++---- applications/vas-sample-app/build/Dockerfile | 3 - .../vas-sample-app/cmd/application.go | 118 ++++++++++++------ applications/vas-sample-app/cmd/main.go | 19 ++- .../helm/templates/vas-cons-app.yaml | 36 +++--- .../deployments/yaml/vas-cons-app.yaml | 36 +++--- 6 files changed, 170 insertions(+), 94 deletions(-) diff --git a/applications/vas-sample-app/README.md b/applications/vas-sample-app/README.md index d2a6e717..426ea04f 100644 --- a/applications/vas-sample-app/README.md +++ b/applications/vas-sample-app/README.md @@ -3,56 +3,63 @@ SPDX-License-Identifier: Apache-2.0 Copyright (c) 2020 Intel Corporation ``` -# VAS Sample Application in OpenNESS +# Video Analytics Services Sample Application in OpenNESS -This sample application demonstrates VAS consumer sample application deployment and execution on the OpenNESS edge platform with VAS enabled. +- [Introduction](#introduction) + - [Directory Structure](#directory-structure) +- [Quick Start](#quick-start) + - [Sample App Deployment](#sample-app-deployment) + - [Sample App Results](#sample-app-results) +## Introduction -# Introduction -## Directory Structure -- `/cmd` : Main applications inside. -- `/build` : Building scripts in the folder. -- `/deployments` : Helm Charts and K8s yaml files inside. +This sample application demonstrates the Video Analytics Services (VAS) consumer sample application deployment and execution on the OpenNESS edge platform with VAS enabled. +For more information on VAS itself, please see [OpenNESS Video Analytics Services](https://github.com/otcshare/specs/blob/master/doc/applications/openness_va_services.md). + + +### Directory Structure +- `/cmd` : Application source files. +- `/build` : Build scripts. +- `/deployments` : Helm Charts and K8s yaml files. ## Quick Start -### To build sample application container image: +To build the sample application container image, run the following command from the sample application folder: ```sh ./build/build-image.sh ``` -After build image, you can directly push the image to docker registry on the controller: +After building the image, you can push it directly to the docker registry on the controller with the following command: ```sh docker push :5000/vas-cons-app:1.0 ``` -Or you can directly build image on the edgenode. +The image can also be built and stored directly on the edgenode. +> **Note**: If the application image is pushed to the docker registry on the controller, you will need to edit the entry ```repository``` in ```deployments/helm/values.yaml``` or ```deployments/yaml/vas-cons-app.yaml``` to the IP address of the docker registry before deploying the application. -### To deploy and test the sample application: +### Sample App Deployment -- Make sure the sample application images built and reachable -- Two method for the deployment on the openness edge node - - Use the command ```kubectl apply``` with the yaml file ```vas-cons-app.yaml``` in the folder - deployments/yaml - - Or use helm charts in the folder - deployments/helm -- Check Results by ```kubectl logs``` -NOTE: If image pushed to docker registry on the controller, you need to edit: ```deployments/helm/values.yaml``` or ```deployments/yaml/vas-cons-app.yaml``` - change ```repository``` to IP address of Docker Registry . +- Make sure the sample application images are built and reachable by running ```docker image list | grep vas-cons-app``` on the edgenode. +- Two methods are available for deploying the sample application on the edgenode: + - Use the command ```kubectl apply -f vas-cons-app.yaml``` from the ```deployments/yaml``` folder. + - Use Helm to install the application using the files in the ```deployments/helm``` folder. -### To check test results: +### Sample App Results -1. check whether the application pod is successfully deployed. +1. Check whether the application pod is successfully deployed. ```sh # kubectl get pods -A | grep vas-cons-app -default vas-cons-app 0/1 Completed 0 30h +default vas-cons-app-7f8bf7c978-csfwj -c vas-cons-app 0/1 Completed 0 30h ``` -2. check the pod's logs as below expected results: +2. Check the pod's logs, results should look similar to the following: ```sh -# kubectl logs vas-cons-app +# kubectl logs vas-cons-app-7f8bf7c978-csfwj -c vas-cons-app 2020/06/29 01:37:25 Video-analytics-service Consumer Started 2020/06/29 01:37:25 CSR Started 2020/06/29 01:37:25 CSR creating certificate @@ -81,4 +88,3 @@ default vas-cons-app 0/1 Completed 0 "state": "COMPLETED" } ``` - diff --git a/applications/vas-sample-app/build/Dockerfile b/applications/vas-sample-app/build/Dockerfile index 014cf9fd..83fb9a98 100644 --- a/applications/vas-sample-app/build/Dockerfile +++ b/applications/vas-sample-app/build/Dockerfile @@ -20,9 +20,6 @@ RUN cd /tmp && \ mv go /usr/local && \ rm -rf /tmp/go* -# Get go dependencies -RUN go get github.com/levigross/grequests - WORKDIR /root COPY build/start.sh ./ diff --git a/applications/vas-sample-app/cmd/application.go b/applications/vas-sample-app/cmd/application.go index 21e5e93f..94cef16c 100644 --- a/applications/vas-sample-app/cmd/application.go +++ b/applications/vas-sample-app/cmd/application.go @@ -4,81 +4,121 @@ package main import ( + "bytes" "encoding/json" - "github.com/levigross/grequests" "log" - "strings" + "net/http" "time" ) -type Source struct { +type VASPostSource struct { URI string `json:"uri"` Type string `json:"type"` } -type Destination struct { +type VASPostDestination struct { Path string `json:"path"` Type string `json:"type"` Format string `json:"format"` } -type VASPostBody struct { - SRCE Source `json:"source"` - DEST Destination `json:"destination"` +type VASPostRequest struct { + Source *VASPostSource `json:"source"` + Destination *VASPostDestination `json:"destination"` } type VASInstanceStatus struct { - AvgFps float64 `json:"avg_fps"` + StartTime float64 `json:"start_time"` ElapsedTime float64 `json:"elapsed_time"` ID int `json:"id"` - StartTime float64 `json:"start_time"` State string `json:"state"` + AvgFPS float64 `json:"avg_fps"` } func postVAServingRequest(vaEndPoint string, vaPipeline string) error { // Interface: POST /pipelines/{name}/{version} for Start new pipeline instance - var endPointStr string - endPointStr = vaEndPoint + "/pipelines/" + vaPipeline - log.Println("Starting POST Request:" + endPointStr) - var src Source - src.URI = "https://github.com/intel-iot-devkit/sample-videos/blob/master/bottle-detection.mp4?raw=true" - src.Type = "uri" - var dst Destination - dst.Path = "/tmp/results.txt" - dst.Type = "file" - dst.Format = "json-lines" - var vasPost = VASPostBody{SRCE: src, DEST: dst} - var vasPostJson, _ = json.Marshal(vasPost) - var vasRequestOption = &grequests.RequestOptions{} - vasRequestOption.JSON = string(vasPostJson) - //resp, err = grequests.Post("http://localhost:8080/pipelines/object_detection/1",vasRequestOption) - resp, err := grequests.Post(endPointStr, vasRequestOption) + VASRequest := VASPostRequest{ + Source: &VASPostSource{ + URI: "https://github.com/intel-iot-devkit/sample-videos/blob/master/bottle-detection.mp4?raw=true", + Type: "uri", + }, + Destination: &VASPostDestination{ + Path: "/tmp/results.txt", + Type: "file", + Format: "json-lines", + }, + } + + endPointStr := vaEndPoint + "/pipelines/" + vaPipeline + log.Println("Starting POST Request: ", endPointStr) + + VASReqPayload, err := json.Marshal(VASRequest) if err != nil { - log.Fatalln("Post Failed with: ", err) - return err + log.Fatal(err) + } + + VASPostResp, err := http.Post(endPointStr, "application/json; charset=UTF-8", bytes.NewBuffer(VASReqPayload)) + for VASPostResp.StatusCode == http.StatusServiceUnavailable { + log.Println("Pipeline service is not currently available, trying again") + time.Sleep(time.Duration(5) * time.Second) + VASPostResp, err = http.Post(endPointStr, "", bytes.NewBuffer(VASReqPayload)) + } + if err != nil { + log.Fatal(err) + } + + var VASRespBody interface{} + err = json.NewDecoder(VASPostResp.Body).Decode(&VASRespBody) + if err != nil { + log.Fatal(err) + } + parsedRespBody, err := json.MarshalIndent(VASRespBody, "", "") + if err != nil { + log.Fatal(err) + } + instanceID := string(parsedRespBody) + + err = VASPostResp.Body.Close() + if err != nil { + log.Fatal(err) } - log.Println("Pipeline Instance Created:", resp.String()) - var instance = strings.TrimSpace(resp.String()) // Interface: GET /pipelines/{name}/{version}/{instance_id}/status for Return pipeline instance status. - // Loop status query until instantance completed. - var statusResp VASInstanceStatus for { - //resp, err := grequests.Get("http://localhost:8080/pipelines/object_detection/1/"+instance+"/status", nil) - resp, err := grequests.Get(endPointStr+"/"+instance+"/status", nil) + getEndPointStr := endPointStr + "/" + instanceID + "/status" + log.Println("Starting status check: ", getEndPointStr) + VASGetResp, err := http.Get(getEndPointStr) + for VASGetResp.StatusCode == http.StatusServiceUnavailable { + log.Println("Pipeline status service is not currently available, trying again") + time.Sleep(time.Duration(5) * time.Second) + VASGetResp, err = http.Get(getEndPointStr) + } if err != nil { - log.Fatalln("Get Failed with: ", err) - return err + log.Fatal(err) } - log.Println(resp.String()) - resp.JSON(&statusResp) + + var statusResp VASInstanceStatus + err = json.NewDecoder(VASGetResp.Body).Decode(&statusResp) + if err != nil { + log.Fatal(err) + } + + log.Printf("{\n avg_fps: %f,\n elapsed_time: %f\n id: %d\n start_time: %f\n state: %s\n}\n", + statusResp.AvgFPS, statusResp.ElapsedTime, statusResp.ID, statusResp.StartTime, statusResp.State) + if statusResp.State == "COMPLETED" { break + } else if statusResp.State == "ERROR" { + log.Fatal("Error with VAS pipeline instance") } - time.Sleep(time.Duration(10) * time.Second) + err = VASGetResp.Body.Close() + if err != nil { + log.Fatal(err) + } + time.Sleep(time.Duration(10) * time.Second) } - return nil + return nil } diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index a77c9dff..73e13a80 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -15,6 +15,7 @@ import ( "encoding/pem" "log" "net/http" + "time" ) // Connectivity constants @@ -65,6 +66,12 @@ func authenticate(prvKey *ecdsa.PrivateKey) (*x509.CertPool, tls.Certificate) { log.Println("CSR POST /auth") resp, err := http.Post("http://"+EAAServerName+":"+EAAServPort+"/auth", "", bytes.NewBuffer(reqBody)) + for resp.StatusCode == http.StatusServiceUnavailable { + log.Println("EAA service is not currently available, trying again") + time.Sleep(time.Duration(5) * time.Second) + resp, err = http.Post("http://"+EAAServerName+":"+EAAServPort+"/auth", + "", bytes.NewBuffer(reqBody)) + } if err != nil { log.Fatal(err) } @@ -75,6 +82,11 @@ func authenticate(prvKey *ecdsa.PrivateKey) (*x509.CertPool, tls.Certificate) { log.Fatal(err) } + err = resp.Body.Close() + if err != nil { + log.Fatal(err) + } + x509Encoded, err := x509.MarshalECPrivateKey(prvKey) if err != nil { log.Fatal(err) @@ -111,6 +123,11 @@ func discoverServices(client *http.Client) (ServiceList, error) { } resp, err := client.Do(req) + for resp.StatusCode == http.StatusServiceUnavailable { + log.Println("EAA service is not currently available, trying again") + time.Sleep(time.Duration(5) * time.Second) + resp, err = client.Do(req) + } if err != nil { log.Println("Service-discovery request failed:", err) return servList, err @@ -176,7 +193,7 @@ func main() { log.Fatal("abnormal services num") return } - log.Println("Discoverd serive:") + log.Println("Discovered service:") log.Println(" URN.ID: ", s.URN.ID) log.Println(" URN.Namespace:", s.URN.Namespace) log.Println(" Description: ", s.Description) diff --git a/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml b/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml index b7c7e62a..6cc81822 100644 --- a/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml +++ b/applications/vas-sample-app/deployments/helm/templates/vas-cons-app.yaml @@ -2,24 +2,32 @@ # Copyright (c) 2020 Intel Corporation --- -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: {{ .Release.Name }}-vas-cons-app - namespace: default - labels: - name: {{ .Release.Name }}-vas-cons-app spec: - restartPolicy: Never - containers: - - name: {{ .Release.Name }}-vas-cons-app - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.pullPolicy }} - volumeMounts: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-vas-cons-app + template: + metadata: + labels: + app: {{ .Release.Name }}-vas-cons-app + spec: + containers: + - name: {{ .Release.Name }}-vas-cons-app + image: "{{ .Values.image.repository }}: {{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.pullPolicy }} + ports: + - containerPort: 80 + - containerPort: 443 + volumeMounts: - name: tmp mountPath: /tmp - - volumes: + volumes: - name: tmp hostPath: - path: /tmp + path: /tmp + type: Directory diff --git a/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml b/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml index 1b8f1e7b..6f4f2c81 100644 --- a/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml +++ b/applications/vas-sample-app/deployments/yaml/vas-cons-app.yaml @@ -2,24 +2,32 @@ # Copyright (c) 2020 Intel Corporation --- -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: vas-cons-app - namespace: default - labels: - name: vas-cons-app spec: - restartPolicy: Never - containers: - - name: vas-cons-app - image: vas-cons-app:1.0 - imagePullPolicy: Never - volumeMounts: + replicas: 1 + selector: + matchLabels: + app: vas-cons-app + template: + metadata: + labels: + app: vas-cons-app + spec: + containers: + - name: vas-cons-app + image: vas-cons-app:1.0 + imagePullPolicy: Never + ports: + - containerPort: 80 + - containerPort: 443 + volumeMounts: - name: tmp mountPath: /tmp - - volumes: + volumes: - name: tmp hostPath: - path: /tmp + path: /tmp + type: Directory From e83d6918bc5ec5d891983d4dd926481f106a92f3 Mon Sep 17 00:00:00 2001 From: damiankopyto <48013534+damiankopyto@users.noreply.github.com> Date: Thu, 10 Sep 2020 15:46:15 +0100 Subject: [PATCH 60/80] OP-4602 Fix (#121) --- .../opentelemetry-agent/templates/agent-configmap.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/telemetry-sample-app/opentelemetry-agent/templates/agent-configmap.yaml b/applications/telemetry-sample-app/opentelemetry-agent/templates/agent-configmap.yaml index 65bed31c..4d85a8c9 100644 --- a/applications/telemetry-sample-app/opentelemetry-agent/templates/agent-configmap.yaml +++ b/applications/telemetry-sample-app/opentelemetry-agent/templates/agent-configmap.yaml @@ -11,7 +11,8 @@ metadata: data: otel-agent-config.yaml: | receivers: - opencensus: "{{ .Values.agent.receiver.opencensus.endpoint }}" + opencensus: + endpoint: "{{ .Values.agent.receiver.opencensus.endpoint }}" exporters: opencensus: From 83b6a4df84010d99c38acf53dc54d6e1d60f3172 Mon Sep 17 00:00:00 2001 From: Mariusz Szczepanik Date: Tue, 15 Sep 2020 09:50:52 +0200 Subject: [PATCH 61/80] Fix EdgeApps go version --- applications/sample-app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/sample-app/go.mod b/applications/sample-app/go.mod index 0dcc8170..17520fd5 100644 --- a/applications/sample-app/go.mod +++ b/applications/sample-app/go.mod @@ -1,6 +1,6 @@ module github.com/otcshare/edgeapps/applications/sample-app -go 1.14.2 +go 1.14 require ( github.com/gorilla/websocket v1.4.2 From 3760ac5407da0cbb6651c869c6e41db478b383b1 Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Thu, 17 Sep 2020 14:56:52 +0800 Subject: [PATCH 62/80] modify README.md file --- applications/openvino/benchmark/README.md | 6 +++--- applications/openvino/benchmark/benchmark_job.yaml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/applications/openvino/benchmark/README.md b/applications/openvino/benchmark/README.md index d0658a15..b5dc2a8b 100644 --- a/applications/openvino/benchmark/README.md +++ b/applications/openvino/benchmark/README.md @@ -34,7 +34,7 @@ Copyright (c) 2020 Intel Corporation ### Build benchamrk image -1. Download edgeapp on both master and node, go to dir edgeapps/applications/openvino/benchmark/ +1. Download edgeapp on both controller and node machine, go to dir edgeapps/applications/openvino/benchmark/ 2. Build benchmark on node by: @@ -42,7 +42,7 @@ Copyright (c) 2020 Intel Corporation sh build_image.sh ``` -3. change `parallelism: 1` to pod number in `benchmark_job.yaml` file on master. +3. change `parallelism: 1` to pod number in `benchmark_job.yaml` file on controller machine. ### Deploy @@ -58,7 +58,7 @@ Copyright (c) 2020 Intel Corporation kubectl delete jobs openvino-benchmark-job ``` -2. On master execute following command to start the job and get logs of each pod +2. On controller machine execute following command to start the job and get logs of each pod ``` kubectl apply -f benchmark_job.yaml diff --git a/applications/openvino/benchmark/benchmark_job.yaml b/applications/openvino/benchmark/benchmark_job.yaml index 41501d50..8f7db66d 100644 --- a/applications/openvino/benchmark/benchmark_job.yaml +++ b/applications/openvino/benchmark/benchmark_job.yaml @@ -8,12 +8,12 @@ metadata: name: cm-benchmark data: env.txt: | - NIREQ=32 - NITER=50000 + NIREQ=8 + NITER=100 # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable - TARGET_DEVICE=CPU + TARGET_DEVICE=HDDL IMAGE=/opt/intel/openvino/deployment_tools/demo/car.png - MODEL=/root/openvino_models/ir/public/squeezenet1.1/FP16/squeezenet1.1.xml + MODEL=/opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/public/ssd512/FP16/ssd512.xml # API: sync/async API=async BATCH_SIZE=1 @@ -25,7 +25,7 @@ metadata: labels: jobgroup: openvino-benchmark spec: - parallelism: 2 + parallelism: 1 template: metadata: labels: From 261cab1da1722d113cb831736ee399c2d7143bb8 Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Thu, 17 Sep 2020 15:01:50 +0800 Subject: [PATCH 63/80] modify some values of configMap parameter --- applications/openvino/benchmark/benchmark_job.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/openvino/benchmark/benchmark_job.yaml b/applications/openvino/benchmark/benchmark_job.yaml index 8f7db66d..8fe2efd3 100644 --- a/applications/openvino/benchmark/benchmark_job.yaml +++ b/applications/openvino/benchmark/benchmark_job.yaml @@ -8,8 +8,8 @@ metadata: name: cm-benchmark data: env.txt: | - NIREQ=8 - NITER=100 + NIREQ=32 + NITER=999999 # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable TARGET_DEVICE=HDDL IMAGE=/opt/intel/openvino/deployment_tools/demo/car.png From 418774f44921f29d40d8cd325bb32d4471008d5f Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Thu, 17 Sep 2020 15:37:40 +0800 Subject: [PATCH 64/80] modify NITER --- applications/openvino/benchmark/benchmark_job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/openvino/benchmark/benchmark_job.yaml b/applications/openvino/benchmark/benchmark_job.yaml index 8fe2efd3..fc6d66d3 100644 --- a/applications/openvino/benchmark/benchmark_job.yaml +++ b/applications/openvino/benchmark/benchmark_job.yaml @@ -9,7 +9,7 @@ metadata: data: env.txt: | NIREQ=32 - NITER=999999 + NITER=99999 # target_device: CPU, GPU, FPGA, HDDL or MYRIAD are acceptable TARGET_DEVICE=HDDL IMAGE=/opt/intel/openvino/deployment_tools/demo/car.png From 72acff8e6cae9a43b373c8fcf6ad098d5a145ebc Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Thu, 17 Sep 2020 15:55:39 +0800 Subject: [PATCH 65/80] add double quote for environment variables in do_benchmark.sh file --- applications/openvino/benchmark/do_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/openvino/benchmark/do_benchmark.sh b/applications/openvino/benchmark/do_benchmark.sh index 99231d2b..93568910 100755 --- a/applications/openvino/benchmark/do_benchmark.sh +++ b/applications/openvino/benchmark/do_benchmark.sh @@ -7,5 +7,5 @@ source /opt/intel/openvino/bin/setupvars.sh cd /opt/intel/openvino/deployment_tools/demo ./demo_squeezenet_download_convert_run.sh -d ${TARGET_DEVICE} -/root/inference_engine_samples_build/intel64/Release/benchmark_app -i ${IMAGE} -m ${MODEL} -d ${TARGET_DEVICE} -nireq ${NIREQ} -niter ${NITER} -api ${API} -b ${BATCH_SIZE} +/root/inference_engine_samples_build/intel64/Release/benchmark_app -i "${IMAGE}" -m "${MODEL}" -d "${TARGET_DEVICE}" -nireq "${NIREQ}" -niter "${NITER}" -api "${API}" -b "${BATCH_SIZE}" From 265ec68a8b49f87d89da0e7c8db0b67331aca6af Mon Sep 17 00:00:00 2001 From: zhaoyuex Date: Thu, 17 Sep 2020 16:01:37 +0800 Subject: [PATCH 66/80] add double quote for environment variables in do_benchmark.sh file --- applications/openvino/benchmark/do_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/openvino/benchmark/do_benchmark.sh b/applications/openvino/benchmark/do_benchmark.sh index 93568910..4deba7d1 100755 --- a/applications/openvino/benchmark/do_benchmark.sh +++ b/applications/openvino/benchmark/do_benchmark.sh @@ -5,7 +5,7 @@ source /opt/intel/openvino/bin/setupvars.sh cd /opt/intel/openvino/deployment_tools/demo -./demo_squeezenet_download_convert_run.sh -d ${TARGET_DEVICE} +./demo_squeezenet_download_convert_run.sh -d "${TARGET_DEVICE}" /root/inference_engine_samples_build/intel64/Release/benchmark_app -i "${IMAGE}" -m "${MODEL}" -d "${TARGET_DEVICE}" -nireq "${NIREQ}" -niter "${NITER}" -api "${API}" -b "${BATCH_SIZE}" From a32ad88a6fd6085c3b23004380f906e7fc9d757b Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Thu, 17 Sep 2020 10:22:31 +0200 Subject: [PATCH 67/80] Added amf/smf resizing to the vm image building script (#124) * Added amf/smf resizing to the vm image building script * Remove unnecessary blank signs --- .../core-network/5G/AMF_SMF/build_amf_smf_image.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh index 96da471c..dee16328 100755 --- a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh +++ b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh @@ -33,8 +33,16 @@ else echo "Skipping image download - file already exists." fi +qemu-img resize "${img_file}" +5G +if [[ ${?} -ne 0 ]]; then + echo "ERROR: Failed to resize the image." + exit 1 +fi + virt-customize -a "${img_file}" \ --root-password password:root \ + --run-command "sudo growpart /dev/sda 1" \ + --run-command "sudo resize2fs /dev/sda1" \ --update \ --install qemu-guest-agent,iputils-ping,iproute2,screen,libpcap-dev,tcpdump,libsctp-dev,apache2,python-pip,sudo,ssh \ --mkdir /root/amf-smf \ From fd3dbd90a15deaadbd3179df9222b817407a852a Mon Sep 17 00:00:00 2001 From: cjnolan <47635874+cjnolan@users.noreply.github.com> Date: Thu, 17 Sep 2020 09:52:11 +0100 Subject: [PATCH 68/80] Fix error checks on VAS sample application (#123) --- .../vas-sample-app/cmd/application.go | 18 ++++++++++------ applications/vas-sample-app/cmd/main.go | 21 ++++++++++++------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/applications/vas-sample-app/cmd/application.go b/applications/vas-sample-app/cmd/application.go index 94cef16c..e6b98e58 100644 --- a/applications/vas-sample-app/cmd/application.go +++ b/applications/vas-sample-app/cmd/application.go @@ -59,13 +59,16 @@ func postVAServingRequest(vaEndPoint string, vaPipeline string) error { } VASPostResp, err := http.Post(endPointStr, "application/json; charset=UTF-8", bytes.NewBuffer(VASReqPayload)) + if err != nil { + log.Fatal(err) + } for VASPostResp.StatusCode == http.StatusServiceUnavailable { log.Println("Pipeline service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) VASPostResp, err = http.Post(endPointStr, "", bytes.NewBuffer(VASReqPayload)) - } - if err != nil { - log.Fatal(err) + if err != nil { + log.Fatal(err) + } } var VASRespBody interface{} @@ -89,13 +92,16 @@ func postVAServingRequest(vaEndPoint string, vaPipeline string) error { getEndPointStr := endPointStr + "/" + instanceID + "/status" log.Println("Starting status check: ", getEndPointStr) VASGetResp, err := http.Get(getEndPointStr) + if err != nil { + log.Fatal(err) + } for VASGetResp.StatusCode == http.StatusServiceUnavailable { log.Println("Pipeline status service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) VASGetResp, err = http.Get(getEndPointStr) - } - if err != nil { - log.Fatal(err) + if err != nil { + log.Fatal(err) + } } var statusResp VASInstanceStatus diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index 73e13a80..9a33f199 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -66,14 +66,17 @@ func authenticate(prvKey *ecdsa.PrivateKey) (*x509.CertPool, tls.Certificate) { log.Println("CSR POST /auth") resp, err := http.Post("http://"+EAAServerName+":"+EAAServPort+"/auth", "", bytes.NewBuffer(reqBody)) + if err != nil { + log.Fatal(err) + } for resp.StatusCode == http.StatusServiceUnavailable { log.Println("EAA service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) resp, err = http.Post("http://"+EAAServerName+":"+EAAServPort+"/auth", "", bytes.NewBuffer(reqBody)) - } - if err != nil { - log.Fatal(err) + if err != nil { + log.Fatal(err) + } } var conCreds AuthCredentials @@ -123,14 +126,18 @@ func discoverServices(client *http.Client) (ServiceList, error) { } resp, err := client.Do(req) + if err != nil { + log.Println("Service-discovery request failed:", err) + return ServList, err + } for resp.StatusCode == http.StatusServiceUnavailable { log.Println("EAA service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) resp, err = client.Do(req) - } - if err != nil { - log.Println("Service-discovery request failed:", err) - return servList, err + if err != nil { + log.Println("Service-discovery request failed:", err) + return servList, err + } } // TODO check if service list is empty -> handle & exit program From aef62de9d5b45e70bad58d3985100717fe3e1f7a Mon Sep 17 00:00:00 2001 From: cjnolan <47635874+cjnolan@users.noreply.github.com> Date: Fri, 18 Sep 2020 12:10:49 +0100 Subject: [PATCH 69/80] Add reconnection attempt limit to fallback handler (#126) --- applications/vas-sample-app/cmd/application.go | 16 ++++++++++++++-- applications/vas-sample-app/cmd/main.go | 14 ++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/applications/vas-sample-app/cmd/application.go b/applications/vas-sample-app/cmd/application.go index e6b98e58..6e559ee0 100644 --- a/applications/vas-sample-app/cmd/application.go +++ b/applications/vas-sample-app/cmd/application.go @@ -62,7 +62,9 @@ func postVAServingRequest(vaEndPoint string, vaPipeline string) error { if err != nil { log.Fatal(err) } - for VASPostResp.StatusCode == http.StatusServiceUnavailable { + postReconnectTries := 0 + for VASPostResp.StatusCode == http.StatusServiceUnavailable && postReconnectTries < 10 { + postReconnectTries++ log.Println("Pipeline service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) VASPostResp, err = http.Post(endPointStr, "", bytes.NewBuffer(VASReqPayload)) @@ -71,6 +73,10 @@ func postVAServingRequest(vaEndPoint string, vaPipeline string) error { } } + if postReconnectTries == 10 { + log.Fatal("Number of connection retries to ", endPointStr, "exceeded, exiting") + } + var VASRespBody interface{} err = json.NewDecoder(VASPostResp.Body).Decode(&VASRespBody) if err != nil { @@ -95,7 +101,9 @@ func postVAServingRequest(vaEndPoint string, vaPipeline string) error { if err != nil { log.Fatal(err) } - for VASGetResp.StatusCode == http.StatusServiceUnavailable { + getReconnectTries := 0 + for VASGetResp.StatusCode == http.StatusServiceUnavailable && getReconnectTries < 10 { + getReconnectTries++ log.Println("Pipeline status service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) VASGetResp, err = http.Get(getEndPointStr) @@ -104,6 +112,10 @@ func postVAServingRequest(vaEndPoint string, vaPipeline string) error { } } + if getReconnectTries == 10 { + log.Fatal("Number of connection retries to ", getEndPointStr, "exceeded, exiting") + } + var statusResp VASInstanceStatus err = json.NewDecoder(VASGetResp.Body).Decode(&statusResp) if err != nil { diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index 9a33f199..e584108f 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -69,7 +69,9 @@ func authenticate(prvKey *ecdsa.PrivateKey) (*x509.CertPool, tls.Certificate) { if err != nil { log.Fatal(err) } - for resp.StatusCode == http.StatusServiceUnavailable { + reconnectTries := 0 + for resp.StatusCode == http.StatusServiceUnavailable && reconnectTries < 10 { + reconnectTries++ log.Println("EAA service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) resp, err = http.Post("http://"+EAAServerName+":"+EAAServPort+"/auth", @@ -78,6 +80,9 @@ func authenticate(prvKey *ecdsa.PrivateKey) (*x509.CertPool, tls.Certificate) { log.Fatal(err) } } + if reconnectTries == 10 { + log.Fatal("Number of connection retries to EAA Auth exceeded, exiting") + } var conCreds AuthCredentials err = json.NewDecoder(resp.Body).Decode(&conCreds) @@ -130,7 +135,9 @@ func discoverServices(client *http.Client) (ServiceList, error) { log.Println("Service-discovery request failed:", err) return ServList, err } - for resp.StatusCode == http.StatusServiceUnavailable { + reconnectTries := 0 + for resp.StatusCode == http.StatusServiceUnavailable && reconnectTries < 10 { + recoonectTries++ log.Println("EAA service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) resp, err = client.Do(req) @@ -139,6 +146,9 @@ func discoverServices(client *http.Client) (ServiceList, error) { return servList, err } } + if reconnectTries == 10 { + log.Fatal("Number of connection retries to EAA Service Discovery exceeded, exiting") + } // TODO check if service list is empty -> handle & exit program From 625fcd2b43e723320a8761d35aee5380a21e787c Mon Sep 17 00:00:00 2001 From: cjnolan <47635874+cjnolan@users.noreply.github.com> Date: Tue, 22 Sep 2020 10:53:48 +0100 Subject: [PATCH 70/80] Fix typo in VAS sample app (#127) --- applications/vas-sample-app/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index e584108f..98858be1 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -137,7 +137,7 @@ func discoverServices(client *http.Client) (ServiceList, error) { } reconnectTries := 0 for resp.StatusCode == http.StatusServiceUnavailable && reconnectTries < 10 { - recoonectTries++ + reconnectTries++ log.Println("EAA service is not currently available, trying again") time.Sleep(time.Duration(5) * time.Second) resp, err = client.Do(req) From 25863c7cb1aadf33ba2eea02bfc2fd73587e9476 Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Wed, 23 Sep 2020 15:52:58 +0200 Subject: [PATCH 71/80] Removed setting UPF pod namespace strictly to default (#128) --- network-functions/core-network/charts/upf/templates/5g-upf.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/network-functions/core-network/charts/upf/templates/5g-upf.yaml b/network-functions/core-network/charts/upf/templates/5g-upf.yaml index 7f0c59d5..25a2952c 100644 --- a/network-functions/core-network/charts/upf/templates/5g-upf.yaml +++ b/network-functions/core-network/charts/upf/templates/5g-upf.yaml @@ -6,7 +6,6 @@ apiVersion: v1 kind: Pod metadata: name: {{ .Release.Name }} - namespace: default annotations: k8s.v1.cni.cncf.io/networks: sriov-openness spec: From 7317b02cb6b664ec48f34bfbc72a439965891c8d Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Wed, 23 Sep 2020 20:18:03 +0530 Subject: [PATCH 72/80] updated for web-visualizer readme (#125) --- applications/eis-experience-kit/README.md | 42 ++++++++++++------- .../eis-experience-kit/group_vars/all.yml | 2 +- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/applications/eis-experience-kit/README.md b/applications/eis-experience-kit/README.md index ecf47958..75ef4abd 100644 --- a/applications/eis-experience-kit/README.md +++ b/applications/eis-experience-kit/README.md @@ -11,9 +11,8 @@ Edge Insights Software (EIS) is the framework for enabling smart manufacturing w More details about EIS: [https://www.intel.com/content/www/us/en/internet-of-things/industrial-iot/edge-insights-industrial.html](https://www.intel.com/content/www/us/en/internet-of-things/industrial-iot/edge-insights-industrial.html) -Currently, `eis-experience-kit` supports EIS in version 2.2. +Currently, `eis-experience-kit` supports EIS in version 2.3.1 -- [OpenVINO](#openvino) - [Pre-requisites](#pre-requisites) - [Installation Process](#installation-process) - [Getting The Sources](#getting-the-sources) @@ -30,13 +29,10 @@ Currently, `eis-experience-kit` supports EIS in version 2.2. - [RTSP Stream Setting](#rtsp-stream-setting) - [View Visualizer Setting](#view-visualizer-setting) - [Installation](#installation) +- [Web Visualizer Display](#web-visualizer-display) - [Removal](#removal) - [References](#references) -## OpenVINO -EIS requires OpenVINO Toolkit to be downloaded and installed. It is automated and no user input is required. - -More about OpenVINO Toolkit: [https://docs.openvinotoolkit.org/](https://docs.openvinotoolkit.org/) ## Pre-requisites EIS applications require Network Edge OpenNESS platform to be deployed and working. @@ -150,23 +146,39 @@ set port="8554" `/opt/eis_repo/IEdgeInsights/VideoIngestion/test_videos/Safety_Full_Hat_and_Vest.avi` -### View Visualizer Setting - -Update IP Adddress of host where we want to see the GUI ouput, Visualizer container will expose the GUI output on display host. - +### View Visualizer HOST Server +Currently default setting is enabled for **web_visualizer**. +This setting is `optional` only if we want to view visualizer on any HOST server, Update IP Adddress of host server where we want to see the GUI output, Visualizer container will expose the GUI output on display host. +```sh +display_visualizer_host: true display_host_ip: "192.168.0.1" < Update Display Host IP> -display_no: "1" +display_no: "1" +``` **Note**: - Display host shoud have GUI/VNC access and check the Display by echo $DISPLAY -update the display on above `display_no` . -- configure `xhost +` on Display host for receiving video GUI +update the display on above `display_no`. +- configure `xhost +` on Display host for receiving video GUI ## Installation -After all the configuration is done, script `deploy_eis_pcb_demo.sh` needs to be executed to start the deployment process. No more actions are required, all the installation steps are fully automated. +After all the configuration is done, script `deploy_eis.sh` needs to be executed to start the deployment process. No more actions are required, all the installation steps are fully automated. + +## Web Visualizer display + +After EIS deployed successfully output can be viewed using + +`https://:5050` + +username:`admin` + +password:`admin@123` + +**Note**: +Open Web Visualizer on google chrome browser, if Your connection is not private show, select Advanced option and proceed to. + ## Removal -To clean up the platform from EIS applications `cleanup_eis_pcb_demo.sh` script can be used. It runs Ansible playbook `eis_pcb_demo_cleanup.yml` and processes all the roles defined there. Inventory file is used for getting Controller Node IP. +To clean up the platform from EIS applications `cleanup_eis_deployment.sh` script can be used. It runs Ansible playbook `eis_cleanup.yml` and processes all the roles defined there. Inventory file is used for getting Controller Node IP. ## References - [Industrial Edge Insights Application on OpenNESS - Solution Overview](https://github.com/otcshare/specs/blob/master/doc/applications/openness_eis.md) diff --git a/applications/eis-experience-kit/group_vars/all.yml b/applications/eis-experience-kit/group_vars/all.yml index d26c014b..6d6e3e6b 100644 --- a/applications/eis-experience-kit/group_vars/all.yml +++ b/applications/eis-experience-kit/group_vars/all.yml @@ -21,7 +21,7 @@ eis_repo_dir: "/opt/eis_repo" eis_sources_dir: "/opt/eis_repo/IEdgeInsights" -demo_type: "pcb" # set "pcb" for pcp demo or "safety" for Safety Hat demo. +demo_type: "pcb" # set "pcb" for pcb demo or "safety" for Safety Hat demo. camera_stream_pod: true # enabled if rtsp stream receive from camera stream pod rtsp_camera_stream_ip: "ia-camera-stream-service" # set remote rtsp camera IP From ffa9427b3761c53e32a19e5240b26a34e72ccce6 Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Fri, 25 Sep 2020 11:30:54 +0200 Subject: [PATCH 73/80] Removed EIS Network Policy + minor fix in ansible syntax (#130) --- .../roles/common/deploy/tasks/main.yml | 16 ---------------- .../roles/eis_sources/tasks/main.yml | 1 - 2 files changed, 17 deletions(-) diff --git a/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml b/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml index 317f7bd7..6ab3f0a9 100644 --- a/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml +++ b/applications/eis-experience-kit/roles/common/deploy/tasks/main.yml @@ -12,19 +12,3 @@ - name: Create namespace {{ k8s_eis_namespace }} command: kubectl create namespace {{ k8s_eis_namespace }} when: get_ns_eis.rc == 1 - -- name: Create directory for network policy - file: - path: {{ eis_network_policy_path }} - state: directory - recurse: yes - -- name: Copy network policy to host - copy: - src: "eis_network_policy.yaml" - dest: "{{ eis_network_policy_path }}" - -- name: Apply EIS network policy - command: kubectl apply -n {{ k8s_eis_namespace }} -f eis_network_policy.yaml - args: - chdir: "{{ eis_network_policy_path }}" diff --git a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml index 68db25cd..8922dd0e 100644 --- a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml +++ b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml @@ -15,7 +15,6 @@ - curl {{ repo_tool }} > repo - mv repo -f /bin/ - chmod 755 /bin/repo - changed_when: true args: chdir: "{{ eis_repo_dir }}" changed_when: true From 9bd83536623e5a3d876b63c1034ec0570ef89d85 Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Fri, 25 Sep 2020 12:06:13 +0200 Subject: [PATCH 74/80] Renamed AMF/SMF build image script params and updated README file (#129) * Replace second param name * Updated amf-smf README file * Added info about exporting variable to the README --- .../core-network/5G/AMF_SMF/README.md | 16 ++++++++++++++-- .../5G/AMF_SMF/build_amf_smf_image.sh | 6 +++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/network-functions/core-network/5G/AMF_SMF/README.md b/network-functions/core-network/5G/AMF_SMF/README.md index 78f6872c..6c33c526 100644 --- a/network-functions/core-network/5G/AMF_SMF/README.md +++ b/network-functions/core-network/5G/AMF_SMF/README.md @@ -32,13 +32,25 @@ make make install ``` -It is also required to have FlexCORE binaries already downloaded. +Qemu by default can use files that have permissions set for user `qemu`. To make sure that qemu can use a file that has permissions set only for root please edit `/etc/libvirt/qemu.conf` and uncomment lines `user=root` and `group=root`. After that reload libvirtd service by: + +```sh +systemctl restart libvirtd +``` + +If the issue with `virt-customize` tool is observed, try with exporting the environment variable: + +```sh +export LIBGUESTFS_BACKEND=direct +``` + +It is also required to have FlexCORE binaries already downloaded. Openness does not provide the binaries of AMF/SMF applications. ### Running the script Script takes two parameters - one is the destination path for the image and second is the path to the FlexCORE binaries directory. Usage looks like: ```sh -./build_amf_smf_image.sh +./build_amf_smf_image.sh ``` As a result of successful run, the customized image (ready to be used by kubevirt virtual machine) should be created in destination path. diff --git a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh index dee16328..430ec77f 100755 --- a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh +++ b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh @@ -4,12 +4,12 @@ # Copyright (c) 2020 Intel Corporation if [[ ${#} -ne 2 ]]; then - echo "Wrong arguments passed. Usage: ${0} " + echo "Wrong arguments passed. Usage: ${0} " exit 1 fi img_file=${1} -astri_dir=${2} +bin_dir=${2} if [[ ! -f ${img_file} ]]; then curl https://cloud-images.ubuntu.com/minimal/releases/bionic/release/ubuntu-18.04-minimal-cloudimg-amd64.img -o "${img_file}" @@ -46,7 +46,7 @@ virt-customize -a "${img_file}" \ --update \ --install qemu-guest-agent,iputils-ping,iproute2,screen,libpcap-dev,tcpdump,libsctp-dev,apache2,python-pip,sudo,ssh \ --mkdir /root/amf-smf \ - --copy-in "${astri_dir}:/root/amf-smf" + --copy-in "${bin_dir}:/root/amf-smf" if [[ ${?} -ne 0 ]]; then echo "ERROR: Failed to customize the image." From b0fe39b9be80f12d6548ca5758ec888b9d278c2e Mon Sep 17 00:00:00 2001 From: Mateusz Szelest <47850727+mateusz-szelest@users.noreply.github.com> Date: Fri, 25 Sep 2020 12:18:00 +0200 Subject: [PATCH 75/80] Changed minimal cloud Ubuntu image to regular for AMF/SMF deployment (#131) * Changed minimal cloud Ubuntu image to regular for AMF/SMF deployment * Updated README --- network-functions/core-network/5G/AMF_SMF/README.md | 2 +- .../core-network/5G/AMF_SMF/build_amf_smf_image.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/network-functions/core-network/5G/AMF_SMF/README.md b/network-functions/core-network/5G/AMF_SMF/README.md index 6c33c526..f7749670 100644 --- a/network-functions/core-network/5G/AMF_SMF/README.md +++ b/network-functions/core-network/5G/AMF_SMF/README.md @@ -55,4 +55,4 @@ Script takes two parameters - one is the destination path for the image and seco As a result of successful run, the customized image (ready to be used by kubevirt virtual machine) should be created in destination path. -Image is based on *Ubuntu 18.04 LTS Minimal* provided by Ubuntu site. +Image is based on *Ubuntu 18.04 LTS (Bionic Beaver)* provided by Ubuntu site. diff --git a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh index 430ec77f..5ce7c4cb 100755 --- a/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh +++ b/network-functions/core-network/5G/AMF_SMF/build_amf_smf_image.sh @@ -12,14 +12,14 @@ img_file=${1} bin_dir=${2} if [[ ! -f ${img_file} ]]; then - curl https://cloud-images.ubuntu.com/minimal/releases/bionic/release/ubuntu-18.04-minimal-cloudimg-amd64.img -o "${img_file}" + curl https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img -o "${img_file}" if [[ ${?} -ne 0 ]]; then echo "ERROR: Failed to download Ubuntu image." exit 1 fi # Check SHA256 if downloaded file is correct - curl https://cloud-images.ubuntu.com/minimal/releases/bionic/release/SHA256SUMS -o SHA256SUMS + curl https://cloud-images.ubuntu.com/bionic/current/SHA256SUMS -o SHA256SUMS if [[ ${?} -ne 0 ]]; then echo "ERROR: Failed to download SHA256SUMS file." exit 1 From 0c92259b7cd27fe3596db1c7775ae8144f191702 Mon Sep 17 00:00:00 2001 From: relhwigi Date: Fri, 25 Sep 2020 11:54:43 +0100 Subject: [PATCH 76/80] fixed bugs which showed up during testing --- network-functions/xran/run_test.sh | 2 +- network-functions/xran/test_verification.py | 135 +------------------- 2 files changed, 5 insertions(+), 132 deletions(-) diff --git a/network-functions/xran/run_test.sh b/network-functions/xran/run_test.sh index 29ae89df..8e2fe465 100755 --- a/network-functions/xran/run_test.sh +++ b/network-functions/xran/run_test.sh @@ -300,7 +300,7 @@ verifyTests() echo "STEP 2: VERIFY TESTS RESULTS" - python "$XRAN_DIR/app/test_verification.py" --ran "$tech" --cat "$cat" --testcase "$tc" --mu "$mu" --b "$bw" --verbose "$TEST_VERBOSE" + python "$XRAN_DIR/app/test_verification.py" --ran "$tech" --cat "$cat" --testcase "$tc" --m_u "$mu" --b "$bw" --verbose "$TEST_VERBOSE" echo -e "\t$(date) xRAN sample app test and test verififaction completed" } diff --git a/network-functions/xran/test_verification.py b/network-functions/xran/test_verification.py index bc3f25a7..003de037 100755 --- a/network-functions/xran/test_verification.py +++ b/network-functions/xran/test_verification.py @@ -174,11 +174,6 @@ def compare_results(rantech, cat, m_u, xran_path, direction, context): else: comp = 0 - if "srsEanble" in context: - srs_enb = 'srsEanble' - else: - srs_enb = 0 - print("compare results: {} [compression {}]\n".format(DIC_DIR.get(direction), comp)) #if cat == 1: @@ -346,128 +341,6 @@ def compare_results(rantech, cat, m_u, xran_path, direction, context): #if (direction == 0) | (cat == 0) | (srs_enb == 0): #DL or Cat A #done - #return res - - print("compare results: {} [compression {}]\n".format('SRS', comp)) - - #srs - symb_mask = context["srsSym"] - try: - flow_id = context["ccNum"]*context["antElmTRx"] - for i in range(0, flow_id): - #read ref and test files - tst = [] - ref = [] - - if direction == 1: - # UL - nrb = n_uirb - file_tst = xran_path+"/results/"+"o-du-srs_log_ant"+str(i)+".txt" - file_ref = xran_path+"/results/logs/"+"o-ru-play_srs_ant"+str(i)+".txt" - -# file_tst = xran_path+"/app/logs/"+"o-du-srs_log_ant"+str(i)+".txt" -# file_ref = xran_path+"/app/logs/"+"o-ru-play_srs_ant"+str(i)+".txt" - else: - raise Exception('Direction is not supported %d'.format(direction)) - - print("test result :", file_tst) - print("test reference:", file_ref) - if os.path.exists(file_tst): - try: - file_tst = open(file_tst, 'r') - except OSError: - print("Could not open/read file:", file_tst) - sys.exit() - else: - print(file_tst, "doesn't exist") - res = -1 - return res - if os.path.exists(file_ref): - try: - file_ref = open(file_ref, 'r') - except OSError: - print("Could not open/read file:", file_ref) - sys.exit() - else: - print(file_tst, "doesn't exist") - res = -1 - return res - - tst = file_tst.readlines() - ref = file_ref.readlines() - - print(len(tst)) - print(len(ref)) - - file_tst.close() - file_ref.close() - - print(context["numSlots"]) - - for slot_idx in range(0, context["numSlots"]): - for sym_idx in range(0, 14): - if symb_mask & (1 << sym_idx): - print("SRS check sym ", sym_idx) - if context["nFrameDuplexType"] == 1: - #skip sym if TDD - if direction == 0: - #DL - sym_dir = slot_config[slot_idx%context["nTddPeriod"]][sym_idx] - if sym_dir != 0: - continue - elif direction == 1: - #UL - sym_dir = slot_config[slot_idx%context["nTddPeriod"]][sym_idx] - if sym_dir != 1: - continue - - #print("Check:","[",i,"]", slot_idx, sym_idx) - for line_idx in range(0, nrb*12): - offset = (slot_idx*nrb*12*14) + sym_idx*nrb*12 + line_idx - try: - line_tst = tst[offset].rstrip() - except IndexError: - res = -1 - print("FAIL:", "IndexError on tst: ant:[", i, "]:", - offset, slot_idx, sym_idx, line_idx, len(tst)) - raise GetOutOfLoops - try: - line_ref = ref[offset].rstrip() - except IndexError: - res = -1 - print("FAIL:", "IndexError on ref: ant:[", i, "]:", - offset, slot_idx, sym_idx, line_idx, len(ref)) - raise GetOutOfLoops - if False: #SRS sent as not compressed - #comp == 1: - # discard LSB bits as BFP compression is not Bit Exact - tst_i_value = int(line_tst.split(" ")[0]) & 0xFF80 - tst_q_value = int(line_tst.split(" ")[1]) & 0xFF80 - ref_i_value = int(line_ref.split(" ")[0]) & 0xFF80 - ref_q_value = int(line_ref.split(" ")[1]) & 0xFF80 - - print("check:", "ant:[", i, "]:", offset, slot_idx, sym_idx, - line_idx, ":", "tst: ", tst_i_value, " ", tst_q_value, " ", - "ref: ", ref_i_value, " ", ref_q_value, " ") - if (tst_i_value != ref_i_value) or (tst_q_value != ref_q_value): - print("FAIL:", "ant:[", i, "]:", offset, slot_idx, sym_idx, - line_idx, ":", "tst: ", tst_i_value, " ", tst_q_value, - " ", "ref: ", ref_i_value, " ", ref_q_value, " ") - res = -1 - raise GetOutOfLoops - else: - #if line_idx == 0: - #print("Check:", offset,"[",i,"]", slot_idx, sym_idx,":" - #, line_tst, line_ref) - if line_ref != line_tst: - print("FAIL:", "ant:[", i, "]:", offset, slot_idx, sym_idx, - line_idx, ":", "tst:", line_tst, "ref:", line_ref) - res = -1 - raise GetOutOfLoops - except GetOutOfLoops: - pass - - return res def parse_dat_file(test_cfg): @@ -498,17 +371,17 @@ def run_tcase(rantech, cat, m_u, b_w, tcase, xran_path): """ method for runing test cases""" if rantech == 1: #LTE if cat == 1: - test_config = xran_path+"/app/usecase/lte_b/m_u{0:d}_{1:d}mhz".format(m_u, b_w) + test_config = xran_path+"/app/usecase/lte_b/mu{0:d}_{1:d}mhz".format(m_u, b_w) elif cat == 0: - test_config = xran_path+"/app/usecase/lte_a/m_u{0:d}_{1:d}mhz".format(m_u, b_w) + test_config = xran_path+"/app/usecase/lte_a/mu{0:d}_{1:d}mhz".format(m_u, b_w) else: print("Incorrect cat arguments\n") return -1 elif rantech == 0: #5G NR if cat == 1: - test_config = xran_path+"/app/usecase/cat_b/m_u{0:d}_{1:d}mhz".format(m_u, b_w) + test_config = xran_path+"/app/usecase/cat_b/mu{0:d}_{1:d}mhz".format(m_u, b_w) elif cat == 0: - test_config = xran_path+"/app/usecase/m_u{0:d}_{1:d}mhz".format(m_u, b_w) + test_config = xran_path+"/app/usecase/mu{0:d}_{1:d}mhz".format(m_u, b_w) else: print("Incorrect cat argument\n") return -1 From a9b522f752b50b7fd7d19b736717cea29d79e944 Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Fri, 25 Sep 2020 18:16:31 +0530 Subject: [PATCH 77/80] delete eis-repo if exist (#132) --- .../eis-experience-kit/roles/eis_sources/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml index 8922dd0e..90523957 100644 --- a/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml +++ b/applications/eis-experience-kit/roles/eis_sources/tasks/main.yml @@ -2,6 +2,11 @@ # Copyright (c) 2020 Intel Corporation --- +- name: delete if any old eis_repo folder exist + file: + path: "{{ eis_repo_dir }}" + state: absent + ignore_errors: yes - name: create directory if not exist file: From 0caeaf5ac0634915bad99bb808ba9bd97ff76c94 Mon Sep 17 00:00:00 2001 From: cjnolan <47635874+cjnolan@users.noreply.github.com> Date: Mon, 28 Sep 2020 11:07:01 +0100 Subject: [PATCH 78/80] Fix build error for VAS sample app (#133) --- applications/vas-sample-app/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/vas-sample-app/cmd/main.go b/applications/vas-sample-app/cmd/main.go index 98858be1..ef8df462 100644 --- a/applications/vas-sample-app/cmd/main.go +++ b/applications/vas-sample-app/cmd/main.go @@ -133,7 +133,7 @@ func discoverServices(client *http.Client) (ServiceList, error) { resp, err := client.Do(req) if err != nil { log.Println("Service-discovery request failed:", err) - return ServList, err + return servList, err } reconnectTries := 0 for resp.StatusCode == http.StatusServiceUnavailable && reconnectTries < 10 { From e4fc7c423cd0b53925459c46f48944c061d16f7b Mon Sep 17 00:00:00 2001 From: Sunil Parida <54929839+sunil-parida@users.noreply.github.com> Date: Wed, 30 Sep 2020 01:07:08 +0530 Subject: [PATCH 79/80] updated eis rtsp stream script error (#134) --- .../eis-experience-kit/scripts/eis_import_images.sh | 1 + .../eis-experience-kit/scripts/send_rtsp_stream_linux.sh | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/applications/eis-experience-kit/scripts/eis_import_images.sh b/applications/eis-experience-kit/scripts/eis_import_images.sh index 19a88b01..1b2d159f 100644 --- a/applications/eis-experience-kit/scripts/eis_import_images.sh +++ b/applications/eis-experience-kit/scripts/eis_import_images.sh @@ -15,6 +15,7 @@ images_to_import=( "ia_video_ingestion:${eis_version}" "ia_video_analytics:${eis_version}" "ia_visualizer:${eis_version}" + "ia_web_visualizer:${eis_version}" ) export_docker_image () { diff --git a/applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh b/applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh index 52573b3c..0b07d468 100755 --- a/applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh +++ b/applications/eis-experience-kit/scripts/send_rtsp_stream_linux.sh @@ -10,7 +10,7 @@ helpPrint() echo "Usage: $0 " echo -e "Example:" echo -e " $0 /root/pcb_d2000.avi 8554" - echo -e " $0 /root/Safety_Full_Hat_and_Vest.avi 8554" + echo -e " $0 /root/Safety_Full_Hat_and_Vest.avi 8554" exit 1 # Exit with help } @@ -18,10 +18,8 @@ if [[ $# -ne 2 ]] ; then echo "No of argument are not correct" helpPrint fi -name=$1 -port=$2 -pkill -9 cvlc sleep 2 -cvlc -vvv "$name" --sout ="#gather:rtp{sdp=rtsp://0.0.0.0:$port/}" --loop --sout-keep +sout="#gather:rtp{sdp=rtsp://0.0.0.0:$2/}" +cvlc -vvv "$1" --sout "$sout" --loop --sout-keep From fac78573ac5b83fd0d95304339f05309cfefdd43 Mon Sep 17 00:00:00 2001 From: Christopher Nolan Date: Wed, 30 Sep 2020 17:10:15 +0100 Subject: [PATCH 80/80] Update file paths --- applications/cdn-caching/nginx/README.md | 2 +- applications/eis-experience-kit/README.md | 2 +- applications/eis-experience-kit/docs/whitepaper.md | 2 +- applications/openvino/README.md | 6 +++--- applications/openvino/benchmark/README.md | 2 +- applications/sample-app/README.md | 6 +++--- applications/sample-app/go.mod | 2 +- applications/sample-app/simpleEaaConsumer/consumer.go | 2 +- applications/sample-app/simpleEaaConsumer/main.go | 2 +- applications/sample-app/simpleEaaProducer/main.go | 2 +- applications/sample-app/simpleEaaProducer/producer.go | 2 +- applications/smart-city-app/README.md | 6 +++--- applications/vas-sample-app/README.md | 2 +- network-functions/core-network/4G/README.md | 2 +- network-functions/core-network/5G/AF/README.md | 2 +- network-functions/core-network/5G/NEF/README.md | 2 +- network-functions/core-network/5G/UPF/README.md | 2 +- network-functions/ran/4G/README.md | 2 +- network-functions/ran/5G/flexRAN-gnb/README.md | 2 +- 19 files changed, 25 insertions(+), 25 deletions(-) diff --git a/applications/cdn-caching/nginx/README.md b/applications/cdn-caching/nginx/README.md index 063729f1..dbd6577c 100644 --- a/applications/cdn-caching/nginx/README.md +++ b/applications/cdn-caching/nginx/README.md @@ -86,7 +86,7 @@ Extra setup is not required when using the control interface for data / content #### Setting up the SR-IOV interface -1. Please refer to the link for setting up the SR-IOV interface: [Setting up SR-IOV for network edge](https://github.com/otcshare/specs/blob/master/doc/enhanced-platform-awareness/openness-sriov-multiple-interfaces.md#sriov-for-network-edge) +1. Please refer to the link for setting up the SR-IOV interface: [Setting up SR-IOV for network edge](https://github.com/open-ness/specs/blob/master/doc/enhanced-platform-awareness/openness-sriov-multiple-interfaces.md#sriov-for-network-edge) 2. Set the "sriov" helm key to "True" to enable the sriov annotations and NFD features in the deployment specification. diff --git a/applications/eis-experience-kit/README.md b/applications/eis-experience-kit/README.md index 75ef4abd..6d08d506 100644 --- a/applications/eis-experience-kit/README.md +++ b/applications/eis-experience-kit/README.md @@ -181,5 +181,5 @@ Open Web Visualizer on google chrome browser, if Your connection is not private To clean up the platform from EIS applications `cleanup_eis_deployment.sh` script can be used. It runs Ansible playbook `eis_cleanup.yml` and processes all the roles defined there. Inventory file is used for getting Controller Node IP. ## References -- [Industrial Edge Insights Application on OpenNESS - Solution Overview](https://github.com/otcshare/specs/blob/master/doc/applications/openness_eis.md) +- [Industrial Edge Insights Application on OpenNESS - Solution Overview](https://github.com/open-ness/specs/blob/master/doc/applications/openness_eis.md) - [Intel’s Edge Insights for Industrial](https://www.intel.com/content/www/us/en/internet-of-things/industrial-iot/edge-insights-industrial.html) diff --git a/applications/eis-experience-kit/docs/whitepaper.md b/applications/eis-experience-kit/docs/whitepaper.md index c40b1804..4123d00f 100644 --- a/applications/eis-experience-kit/docs/whitepaper.md +++ b/applications/eis-experience-kit/docs/whitepaper.md @@ -113,7 +113,7 @@ The EIS application can be deployed through the OpenNESS architecture which requ - **Camera**: A set of cameras connected through the wireless network. -EIS installation and deployment on the OpenNESS Network Edge environment is available at [eis-experience-kit](https://github.com/otcshare/edgeapps/blob/master/applications/eis-experience-kit/README.md) +EIS installation and deployment on the OpenNESS Network Edge environment is available at [eis-experience-kit](https://github.com/open-ness/edgeapps/blob/master/applications/eis-experience-kit/README.md) *NOTE: `In the above diagram, the RTSP Camera Stream pod is used for sending the RTSP stream. But in LTE/5G Network real-time deployment, the camera stream will come from real IP camera as per below diagram`* diff --git a/applications/openvino/README.md b/applications/openvino/README.md index 07963a23..cab48d71 100644 --- a/applications/openvino/README.md +++ b/applications/openvino/README.md @@ -7,6 +7,6 @@ Copyright (c) 2020 Intel Corporation This sample application demonstrates OpenVINO object detection (pedestrian and vehicle detection) deployment and execution on the OpenNESS edge platform. -- [OpenVINO Sample Application White Paper](https://github.com/otcshare/specs/blob/master/doc/applications/openness_openvino.md) -- [OpenVINO Sample Application Onboarding Guide - Network Edge](https://github.com/otcshare/specs/blob/master/doc/applications-onboard/network-edge-applications-onboarding.md#onboarding-openvino-application) -- [OpenVINO Sample Application Onboarding Guide - On-premises](https://github.com/otcshare/specs/blob/master/doc/applications-onboard/on-premises-applications-onboarding.md#onboarding-openvino-applications) +- [OpenVINO Sample Application White Paper](https://github.com/open-ness/specs/blob/master/doc/applications/openness_openvino.md) +- [OpenVINO Sample Application Onboarding Guide - Network Edge](https://github.com/open-ness/specs/blob/master/doc/applications-onboard/network-edge-applications-onboarding.md#onboarding-openvino-application) +- [OpenVINO Sample Application Onboarding Guide - On-premises](https://github.com/open-ness/specs/blob/master/doc/applications-onboard/on-premises-applications-onboarding.md#onboarding-openvino-applications) diff --git a/applications/openvino/benchmark/README.md b/applications/openvino/benchmark/README.md index b5dc2a8b..ee55261f 100644 --- a/applications/openvino/benchmark/README.md +++ b/applications/openvino/benchmark/README.md @@ -5,7 +5,7 @@ Copyright (c) 2020 Intel Corporation # OpenVINO Benchmark Performance Test -> Notes: The following all steps assumes that OpenNESS were installed through [OpenNESS playbooks](https://github.com/otcshare/specs/blob/master/doc/getting-started/network-edge/controller-edge-node-setup.md). +> Notes: The following all steps assumes that OpenNESS were installed through [OpenNESS playbooks](https://github.com/open-ness/specs/blob/master/doc/getting-started/network-edge/controller-edge-node-setup.md). - [Precondition](#precondition) - [Build benchamrk image](#build-benchamrk-image) diff --git a/applications/sample-app/README.md b/applications/sample-app/README.md index c4acfb83..d89cabc2 100644 --- a/applications/sample-app/README.md +++ b/applications/sample-app/README.md @@ -7,6 +7,6 @@ Copyright (c) 2019-2020 Intel Corporation This application presents a simple and lightweight producer-consumer test application based on EAA APIs. -- [OpenNESS Application Development and Porting Guide](https://github.com/otcshare/specs/blob/master/doc/applications/openness_appguide.md) -- [Sample EAA Test Application Onboarding Guide - Network Edge](https://github.com/otcshare/specs/blob/master/doc/applications-onboard/network-edge-applications-onboarding.md#onboarding-sample-application) -- [Generic Application (container or VM) Onboarding Guide - On-premises](https://github.com/otcshare/specs/blob/master/doc/applications-onboard/on-premises-applications-onboarding.md#onboarding-applications) +- [OpenNESS Application Development and Porting Guide](https://github.com/open-ness/specs/blob/master/doc/applications/openness_appguide.md) +- [Sample EAA Test Application Onboarding Guide - Network Edge](https://github.com/open-ness/specs/blob/master/doc/applications-onboard/network-edge-applications-onboarding.md#onboarding-sample-application) +- [Generic Application (container or VM) Onboarding Guide - On-premises](https://github.com/open-ness/specs/blob/master/doc/applications-onboard/on-premises-applications-onboarding.md#onboarding-applications) diff --git a/applications/sample-app/go.mod b/applications/sample-app/go.mod index 17520fd5..9c5840e0 100644 --- a/applications/sample-app/go.mod +++ b/applications/sample-app/go.mod @@ -1,4 +1,4 @@ -module github.com/otcshare/edgeapps/applications/sample-app +module github.com/open-ness/edgeapps/applications/sample-app go 1.14 diff --git a/applications/sample-app/simpleEaaConsumer/consumer.go b/applications/sample-app/simpleEaaConsumer/consumer.go index 7d228748..2d5de186 100644 --- a/applications/sample-app/simpleEaaConsumer/consumer.go +++ b/applications/sample-app/simpleEaaConsumer/consumer.go @@ -13,7 +13,7 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/otcshare/edgeapps/applications/sample-app/common" + "github.com/open-ness/edgeapps/applications/sample-app/common" "github.com/pkg/errors" ) diff --git a/applications/sample-app/simpleEaaConsumer/main.go b/applications/sample-app/simpleEaaConsumer/main.go index a90191c0..c3046205 100644 --- a/applications/sample-app/simpleEaaConsumer/main.go +++ b/applications/sample-app/simpleEaaConsumer/main.go @@ -6,7 +6,7 @@ package main import ( "log" - "github.com/otcshare/edgeapps/applications/sample-app/common" + "github.com/open-ness/edgeapps/applications/sample-app/common" ) // A consumer shout create a secure connection the Edge Node. This can be diff --git a/applications/sample-app/simpleEaaProducer/main.go b/applications/sample-app/simpleEaaProducer/main.go index ab1cfed9..b3e7be19 100644 --- a/applications/sample-app/simpleEaaProducer/main.go +++ b/applications/sample-app/simpleEaaProducer/main.go @@ -8,7 +8,7 @@ import ( "log" "time" - "github.com/otcshare/edgeapps/applications/sample-app/common" + "github.com/open-ness/edgeapps/applications/sample-app/common" "github.com/pkg/errors" ) diff --git a/applications/sample-app/simpleEaaProducer/producer.go b/applications/sample-app/simpleEaaProducer/producer.go index bdeb99ba..dabb66cf 100644 --- a/applications/sample-app/simpleEaaProducer/producer.go +++ b/applications/sample-app/simpleEaaProducer/producer.go @@ -9,7 +9,7 @@ import ( "log" "net/http" - "github.com/otcshare/edgeapps/applications/sample-app/common" + "github.com/open-ness/edgeapps/applications/sample-app/common" "github.com/pkg/errors" ) diff --git a/applications/smart-city-app/README.md b/applications/smart-city-app/README.md index d9238457..95b11c40 100644 --- a/applications/smart-city-app/README.md +++ b/applications/smart-city-app/README.md @@ -15,7 +15,7 @@ Smart City application is a sample application that is built on top of the OpenV - [References](#references) ## Installing OpenNESS -The OpenNESS must be installed before going forward with Smart City application deployment. Installation is performed through [OpenNESS Deployment Flavors](https://github.com/otcshare/specs/blob/master/doc/flavors.md). +The OpenNESS must be installed before going forward with Smart City application deployment. Installation is performed through [OpenNESS Deployment Flavors](https://github.com/open-ness/specs/blob/master/doc/flavors.md). > **NOTE:** At the time of writing this guide, there was no [Network Policy for Kubernetes](https://kubernetes.io/docs/concepts/services-networking/network-policies/) defined yet for the Smart City application. So, it is advised to remove **all** the network policies existing in the `default` namespace such as: > ```shell @@ -70,7 +70,7 @@ This mode provide an easy and quick start with executing the application in the Visual Cloud Accelerator Card - Analytics (VCAC-A) is a PCIe add on card comprising of Intel Core i3-7100U Processor with Intel HD Graphics 620 and 12 Movidius VPUs. Provisioning the network edge with VCAC-A acceleration through OpenNESS Experience Kits enables dense and performant Smart City video analytics and transcoding pipelines. -1. Deploy the OpenNESS [Media Analytics Flavor with VCAC-A](https://github.com/otcshare/specs/blob/master/doc/flavors.md#media-analytics-flavor-with-vcac-a) and place the OpenNESS edge node hostname, that has the VCAC-A card(s) plugged-in, in `[edgenode_vca_group]` group in `inventory.ini` file of the openness-experience-kit. +1. Deploy the OpenNESS [Media Analytics Flavor with VCAC-A](https://github.com/open-ness/specs/blob/master/doc/flavors.md#media-analytics-flavor-with-vcac-a) and place the OpenNESS edge node hostname, that has the VCAC-A card(s) plugged-in, in `[edgenode_vca_group]` group in `inventory.ini` file of the openness-experience-kit. 2. Configure openness-experience-kit to deploy "Weave Net" CNI by editing `group_vars/all/10-default.yml` @@ -128,6 +128,6 @@ helm uninstall smart-city-app ## References -- [OpenNESS Smart City application whitepaper](https://github.com/otcshare/specs/blob/master/doc/applications/openness_ovc.md) +- [OpenNESS Smart City application whitepaper](https://github.com/open-ness/specs/blob/master/doc/applications/openness_ovc.md) - [Intel Open Visual Cloud Smart City reference pipeline](https://github.com/OpenVisualCloud/Smart-City-Sample) - [Intel Open Visual Cloud VCAC-A card media analytics software](https://github.com/OpenVisualCloud/VCAC-SW-Analytics/) diff --git a/applications/vas-sample-app/README.md b/applications/vas-sample-app/README.md index 426ea04f..3adef6a3 100644 --- a/applications/vas-sample-app/README.md +++ b/applications/vas-sample-app/README.md @@ -14,7 +14,7 @@ Copyright (c) 2020 Intel Corporation ## Introduction This sample application demonstrates the Video Analytics Services (VAS) consumer sample application deployment and execution on the OpenNESS edge platform with VAS enabled. -For more information on VAS itself, please see [OpenNESS Video Analytics Services](https://github.com/otcshare/specs/blob/master/doc/applications/openness_va_services.md). +For more information on VAS itself, please see [OpenNESS Video Analytics Services](https://github.com/open-ness/specs/blob/master/doc/applications/openness_va_services.md). ### Directory Structure diff --git a/network-functions/core-network/4G/README.md b/network-functions/core-network/4G/README.md index e0dcf1eb..848b2634 100644 --- a/network-functions/core-network/4G/README.md +++ b/network-functions/core-network/4G/README.md @@ -4,4 +4,4 @@ Copyright (c) 2020 Intel Corporation ``` ## Core Network 4G -OpenNESS support for 4G CUPS is documented here [openness_epc.md](https://github.com/otcshare/specs/blob/master/doc/core-network/openness_epc.md) +OpenNESS support for 4G CUPS is documented here [openness_epc.md](https://github.com/open-ness/specs/blob/master/doc/core-network/openness_epc.md) diff --git a/network-functions/core-network/5G/AF/README.md b/network-functions/core-network/5G/AF/README.md index 8fcd081d..0ffd1ec1 100644 --- a/network-functions/core-network/5G/AF/README.md +++ b/network-functions/core-network/5G/AF/README.md @@ -4,4 +4,4 @@ Copyright (c) 2020 Intel Corporation ``` ## Core Network 5G – AF -OpenNESS supports deployment of AF in Cloudnative environment using OpenNESS for more details please refer to [openness_ngc.md](https://github.com/otcshare/specs/blob/master/doc/core-network/openness_ngc.md) +OpenNESS supports deployment of AF in Cloudnative environment using OpenNESS for more details please refer to [openness_ngc.md](https://github.com/open-ness/specs/blob/master/doc/core-network/openness_ngc.md) diff --git a/network-functions/core-network/5G/NEF/README.md b/network-functions/core-network/5G/NEF/README.md index 276aae27..efd98a72 100644 --- a/network-functions/core-network/5G/NEF/README.md +++ b/network-functions/core-network/5G/NEF/README.md @@ -4,4 +4,4 @@ Copyright (c) 2020 Intel Corporation ``` ## Core Network 5G – NEF -OpenNESS supports deployment of NEF in Cloudnative environment using OpenNESS for more details please refer to [openness_ngc.md](https://github.com/otcshare/specs/blob/master/doc/core-network/openness_ngc.md) +OpenNESS supports deployment of NEF in Cloudnative environment using OpenNESS for more details please refer to [openness_ngc.md](https://github.com/open-ness/specs/blob/master/doc/core-network/openness_ngc.md) diff --git a/network-functions/core-network/5G/UPF/README.md b/network-functions/core-network/5G/UPF/README.md index d3bb7f5e..5bd46041 100644 --- a/network-functions/core-network/5G/UPF/README.md +++ b/network-functions/core-network/5G/UPF/README.md @@ -6,4 +6,4 @@ Copyright (c) 2020 Intel Corporation # 5g - UPF -OpenNESS supports deployment of 5G User Plane Functions in Cloudnative environment. For more details please refer to [openness_upf.md](https://github.com/otcshare/specs/blob/master/doc/core-network/openness_upf.md) +OpenNESS supports deployment of 5G User Plane Functions in Cloudnative environment. For more details please refer to [openness_upf.md](https://github.com/open-ness/specs/blob/master/doc/core-network/openness_upf.md) diff --git a/network-functions/ran/4G/README.md b/network-functions/ran/4G/README.md index 61847999..c383e314 100644 --- a/network-functions/ran/4G/README.md +++ b/network-functions/ran/4G/README.md @@ -4,5 +4,5 @@ Copyright (c) 2020 Intel Corporation ``` ## FlexRAN 4G – eNodeB -OpenNESS supports deployment of eNodeB in Cloudnative environment using OpenNESS for more details please refer to [openness_ran.md](https://github.com/otcshare/specs/blob/master/doc/ran/openness_ran.md) +OpenNESS supports deployment of eNodeB in Cloudnative environment using OpenNESS for more details please refer to [openness_ran.md](https://github.com/open-ness/specs/blob/master/doc/ran/openness_ran.md) diff --git a/network-functions/ran/5G/flexRAN-gnb/README.md b/network-functions/ran/5G/flexRAN-gnb/README.md index 971b842d..8b9fbcd1 100644 --- a/network-functions/ran/5G/flexRAN-gnb/README.md +++ b/network-functions/ran/5G/flexRAN-gnb/README.md @@ -4,4 +4,4 @@ Copyright (c) 2020 Intel Corporation ``` ## FlexRAN 5G – gNb -OpenNESS supports deployment of gNb (Sub6 and mmWave) in Cloudnative environment using OpenNESS for more details please refer to [openness_ran.md](https://github.com/otcshare/specs/blob/master/doc/ran/openness_ran.md) +OpenNESS supports deployment of gNb (Sub6 and mmWave) in Cloudnative environment using OpenNESS for more details please refer to [openness_ran.md](https://github.com/open-ness/specs/blob/master/doc/ran/openness_ran.md)