diff --git a/Makefile b/Makefile index 446f5cf7..edfbeca6 100644 --- a/Makefile +++ b/Makefile @@ -385,10 +385,9 @@ properclean: $(Q)$(RM) -Rfv $(DISTDIR)/* endif - +include $(KRAFTDIR)/contrib/Makefile include $(KRAFTDIR)/package/docker/Makefile - .PHONY: no-help: @echo "No help for target: $(strip $(subst help,,$(MAKECMDGOALS)))" diff --git a/contrib/Makefile b/contrib/Makefile new file mode 100644 index 00000000..68773f82 --- /dev/null +++ b/contrib/Makefile @@ -0,0 +1,9 @@ +CONTRIBDIR ?= $(KRAFTDIR)/contrib +GOTOOLS ?= $(CONTRIBDIR)/gotools +SRCS ?= $(GOTOOLS)/srcs +.DEFAULT_TARGET: all +all: gotools + +.PHONY: gotools +gotools: + include $(SRCS)/Makefile \ No newline at end of file diff --git a/contrib/gotools/configfiles/nginx.txt b/contrib/gotools/configfiles/nginx.txt new file mode 100644 index 00000000..e30056a3 --- /dev/null +++ b/contrib/gotools/configfiles/nginx.txt @@ -0,0 +1,6 @@ +nginx -h +nginx -v +nginx -V +nginx -t +nginx -T +nginx -Tq \ No newline at end of file diff --git a/contrib/gotools/srcs/Makefile b/contrib/gotools/srcs/Makefile new file mode 100644 index 00000000..39f5902b --- /dev/null +++ b/contrib/gotools/srcs/Makefile @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Authors: Gaulthier Gain +# +# Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# Program arguments +BINARY_NAME ?= gotools +BINARY_UNIX ?= $(BINARY_NAME)_unix +CONTAINER_NAME ?= unikraft/gotools:latest + +## Gotools +DOCKER ?= docker +TARGET ?= binary +GO ?= go +GOBUILD ?= $(GO) build +GOCLEAN ?= $(GO) clean +GOGET ?= $(GO) get + +# Targets +all: build +container: + $(DOCKER) build \ + -t $(CONTAINER_NAME) \ + -f Dockerfile \ + --target=$(TARGET) \ + . +build: deps + $(GOBUILD) -o $(BINARY_NAME) -v +clean: + $(GOCLEAN) + rm -f $(BINARY_NAME) + rm -f $(BINARY_UNIX) +run: + $(GOBUILD) -o $(BINARY_NAME) -v + ./$(BINARY_NAME) +deps: + $(GOGET) github.com/fatih/color + $(GOGET) github.com/akamensky/argparse + $(GOGET) github.com/awalterschulze/gographviz + $(GOGET) github.com/sergi/go-diff/... +# Cross compilation +build-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v diff --git a/contrib/gotools/srcs/buildtool/args.go b/contrib/gotools/srcs/buildtool/args.go new file mode 100644 index 00000000..477a2b3d --- /dev/null +++ b/contrib/gotools/srcs/buildtool/args.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package buildtool + +import ( + "os" + u "tools/srcs/common" + + "github.com/akamensky/argparse" +) + +const ( + programArg = "program" + unikraftArg = "unikraft" + sourcesArg = "sources" + makefileArg = "makefile" +) + +// ParseArguments parses arguments of the application. +// +// It returns an error if any, otherwise it returns nil. +func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { + + args.InitArgParse(p, args, u.STRING, "p", programArg, + &argparse.Options{Required: true, Help: "Program name"}) + + args.InitArgParse(p, args, u.STRING, "u", unikraftArg, + &argparse.Options{Required: false, Help: "Unikraft Path"}) + args.InitArgParse(p, args, u.STRING, "s", sourcesArg, + &argparse.Options{Required: true, Help: "App Sources " + + "Folder"}) + args.InitArgParse(p, args, u.STRING, "m", makefileArg, + &argparse.Options{Required: false, Help: "Add additional properties " + + "for Makefile"}) + + return u.ParserWrapper(p, os.Args) +} diff --git a/contrib/gotools/srcs/buildtool/kconfig_parser.go b/contrib/gotools/srcs/buildtool/kconfig_parser.go new file mode 100644 index 00000000..a27a8cb0 --- /dev/null +++ b/contrib/gotools/srcs/buildtool/kconfig_parser.go @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package buildtool + +import ( + "bufio" + "io" + "os" + "strings" + + u "tools/srcs/common" +) + +const ( + configLine = iota // Config = + commentedConfigLine // Commented config: # is not set + headerLine // Header: # <.*> + separatorLine // Separator: # + lineFeed // Line FEED: \n +) + +// Exported struct that represents a Kconfig entry. +type KConfig struct { + Config string + Value *string + Type int +} + +// writeConfig writes a '.config' file for the Unikraft build system. +// +// It returns an error if any, otherwise it returns nil. +func writeConfig(filename string, items []*KConfig) error { + + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + for _, kConfig := range items { + + var config string + switch kConfig.Type { + case configLine: + config = kConfig.Config + "=" + *kConfig.Value + case commentedConfigLine: + config = "# " + kConfig.Config + " is not set" + case headerLine: + config = kConfig.Config + case separatorLine: + config = "#" + case lineFeed: + config = "\n" + } + if _, err := f.Write([]byte(config + "\n")); err != nil { + u.PrintErr(err) + } + } + + return nil +} + +// parseConfig parses a '.config' file used by the Unikraft build system. +// +// It returns a list of KConfig and an error if any, otherwise it returns nil. +func parseConfig(filename string, kConfigMap map[string]*KConfig, + items []*KConfig, matchedLibs []string) ([]*KConfig, error) { + + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r := bufio.NewReader(f) + for { + line, err := r.ReadString(0x0A) + + items = addKConfig(line, kConfigMap, items, matchedLibs) + + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + } + + return items, nil +} + +// addKConfig adds a KConfig entry to adequate data structures. +// +// It returns a list of KConfig. This list will be saved into a '.config' +// file. +func addKConfig(line string, kConfigMap map[string]*KConfig, + items []*KConfig, matchedLibs []string) []*KConfig { + + var config string + var value *string + var typeConfig int + + switch { + case strings.HasPrefix(line, "#") && strings.Contains(line, + "CONFIG"): // Commented config: # is not set + + split := strings.Fields(line) + config = split[1] + value = nil + typeConfig = commentedConfigLine + case strings.HasPrefix(line, "#") && len(line) > 2: // Separator: # + config = strings.TrimSuffix(line, "\n") + value = nil + typeConfig = headerLine + case strings.HasPrefix(line, "#") && len(line) == 2: // Header: # <.*> + config, value = "#", nil + typeConfig = separatorLine + case strings.Contains(line, "="): // Config: = y + split := strings.Split(line, "=") + config = split[0] + word := strings.TrimSuffix(split[1], "\n") + value = &word + typeConfig = configLine + default: // Line FEED + config, value = "#", nil + typeConfig = lineFeed + } + + // Create KConfig + kConfig := &KConfig{ + config, + value, + typeConfig, + } + + // If config is not a comment, perform additional procedures + if config != "#" { + kConfigMap[config] = kConfig + items = append(items, kConfigMap[config]) + items = addInternalConfig(config, kConfigMap, items) + items = matchLibsKconfig(config, kConfigMap, items, matchedLibs) + } else { + items = append(items, kConfig) + } + + return items +} + +// updateConfig updates KConfig entries to particular values. +// +// It returns a list of KConfig. +func updateConfig(kConfigMap map[string]*KConfig, + items []*KConfig) []*KConfig { + v := "y" + var configs = []*KConfig{ + // CONFIG libs + {"CONFIG_HAVE_BOOTENTRY", &v, configLine}, + {"CONFIG_HAVE_SCHED", &v, configLine}, + {"CONFIG_LIBUKARGPARSE", &v, configLine}, + {"CONFIG_LIBUKBUS", &v, configLine}, + {"CONFIG_LIBUKSGLIST", &v, configLine}, + {"CONFIG_LIBUKTIMECONV", &v, configLine}, + + // CONFIG build + {"CONFIG_OPTIMIZE_NONE", &v, configLine}, + {"CONFIG_OPTIMIZE_PERF", nil, commentedConfigLine}, + } + + return SetConfig(configs, kConfigMap, items) +} + +// SetConfig updates a specific KConfig entry. +// +// It returns a list of KConfig. +func SetConfig(newConfigs []*KConfig, kConfigMap map[string]*KConfig, + items []*KConfig) []*KConfig { + + for _, conf := range newConfigs { + // If kConfigMap does not contains the value, add it + if _, ok := kConfigMap[conf.Config]; !ok { + if len(conf.Config) > 1 { + kConfigMap[conf.Config] = conf + items = append(items, kConfigMap[conf.Config]) + } else { + items = append(items, conf) + } + } else { + // Update only + newConfiguration := kConfigMap[conf.Config] + newConfiguration.Value = conf.Value + newConfiguration.Type = conf.Type + } + } + + return items +} + +// matchLibsKconfig performs the matching between Kconfig entries and micro-libs +// and updates the right Kconfig +// +// It returns a list of KConfig. +func matchLibsKconfig(conf string, kConfigMap map[string]*KConfig, + items []*KConfig, matchedLibs []string) []*KConfig { + + v := "y" + switch conf { + case "CONFIG_LIBPOSIX_PROCESS": + if u.Contains(matchedLibs, POSIXPROCESS) { + configs := []*KConfig{ + {"CONFIG_LIBPOSIX_PROCESS", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_LIBPOSIX_USER": + if u.Contains(matchedLibs, POSIXUSER) { + configs := []*KConfig{ + {"CONFIG_LIBPOSIX_USER", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_LIBSYSCALL_SHIM": + if u.Contains(matchedLibs, SYSCALLSHIM) { + configs := []*KConfig{ + {"CONFIG_LIBSYSCALL_SHIM", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_LIBUKTIME": + if u.Contains(matchedLibs, UKTIME) { + configs := []*KConfig{ + {"CONFIG_LIBUKTIME", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_UKSYSINFO": + if u.Contains(matchedLibs, UKSYSINFO) { + configs := []*KConfig{ + {"CONFIG_UKSYSINFO", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_POSIX_LIBDL": + if u.Contains(matchedLibs, POSIXLIBDL) { + configs := []*KConfig{ + {"CONFIG_POSIX_LIBDL", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_LIBVFSCORE": + if u.Contains(matchedLibs, VFSCORE) { + n := "16" + configs := []*KConfig{ + {"CONFIG_LIBVFSCORE", &v, configLine}, + {"CONFIG_LIBRAMFS", nil, commentedConfigLine}, + {"CONFIG_LIBDEVFS", &v, configLine}, + {"CONFIG_LIBDEVFS_USE_RAMFS", nil, commentedConfigLine}, + {"#", nil, separatorLine}, + {"# vfscore configuration", nil, headerLine}, + {"#", nil, separatorLine}, + {"CONFIG_LIBVFSCORE_PIPE_SIZE_ORDER", &n, configLine}, + {"CONFIG_LIBVFSCORE_AUTOMOUNT_ROOTFS", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_LIBNEWLIBC": + if u.Contains(matchedLibs, NEWLIB) { + configs := []*KConfig{ + {"CONFIG_HAVE_LIBC", &v, configLine}, + {"CONFIG_LIBNEWLIBC", &v, configLine}, + {"CONFIG_LIBNEWLIBM", &v, configLine}, + {"CONFIG_LIBNEWLIBC_WANT_IO_C99_FORMATS", nil, commentedConfigLine}, + {"CONFIG_LIBNEWLIBC_LINUX_ERRNO_EXTENSIONS", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_LIBPTHREAD_EMBEDDED": + if u.Contains(matchedLibs, PTHREADEMBEDDED) { + number := "32" + configs := []*KConfig{ + {"CONFIG_LIBPTHREAD_EMBEDDED", &v, configLine}, + {"CONFIG_LIBPTHREAD_EMBEDDED_MAX_SIMUL_THREADS", &number, configLine}, + {"CONFIG_LIBPTHREAD_EMBEDDED_MAX_TLS", &number, configLine}, + {"CONFIG_LIBPTHREAD_EMBEDDED_UTEST", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + case "CONFIG_LIBLWIP": + if u.Contains(matchedLibs, LWIP) { + seed, queues := "23", "1" + mss, dnsMaxServer, dnsTableSize := "1460", "2", "32" + configs := []*KConfig{ + {"CONFIG_VIRTIO_NET", &v, configLine}, + // + {"CONFIG_LIBUKMPI", &v, configLine}, + {"CONFIG_LIBUKMPI_MBOX", &v, configLine}, + // + {"CONFIG_LIBUKSWRAND", &v, configLine}, + {"CONFIG_LIBUKSWRAND_MWC", &v, configLine}, + {"CONFIG_LIBUKSWRAND_INITIALSEED", &seed, configLine}, + // + {"CONFIG_LIBUKNETDEV", &v, configLine}, + {"CONFIG_LIBUKNETDEV_MAXNBQUEUES", &queues, configLine}, + {"CONFIG_LIBUKNETDEV_DISPATCHERTHREADS", &v, configLine}, + // + {"CONFIG_LIBLWIP", &v, configLine}, + {"#", nil, separatorLine}, + {"# Netif drivers", nil, headerLine}, + {"#", nil, separatorLine}, + {"CONFIG_LWIP_UKNETDEV", &v, configLine}, + {"CONFIG_LWIP_AUTOIFACE", &v, configLine}, + {"CONFIG_LWIP_NOTHREADS", nil, commentedConfigLine}, + {"CONFIG_LWIP_THREADS", &v, configLine}, + {"CONFIG_LWIP_HEAP", &v, configLine}, + {"CONFIG_LWIP_NETIF_EXT_STATUS_CALLBACK", &v, configLine}, + {"CONFIG_LWIP_NETIF_STATUS_PRINT", &v, configLine}, + {"CONFIG_LWIP_IPV4", &v, configLine}, + {"CONFIG_LWIP_IPV6", nil, commentedConfigLine}, + {"CONFIG_LWIP_UDP", &v, configLine}, + {"CONFIG_LWIP_TCP", &v, configLine}, + {"CONFIG_LWIP_TCP_MSS", &mss, configLine}, + {"CONFIG_LWIP_WND_SCALE", &v, configLine}, + {"CONFIG_LWIP_TCP_KEEPALIVE", nil, commentedConfigLine}, + {"CONFIG_LWIP_TCP_TIMESTAMPS", nil, commentedConfigLine}, + {"CONFIG_LWIP_ICMP", &v, configLine}, + {"CONFIG_LWIP_IGMP", nil, commentedConfigLine}, + {"CONFIG_LWIP_SNMP", nil, commentedConfigLine}, + {"CONFIG_LWIP_DHCP", nil, commentedConfigLine}, + {"CONFIG_LWIP_DNS", &v, configLine}, + {"CONFIG_LWIP_DNS_MAX_SERVERS", &dnsMaxServer, configLine}, + {"CONFIG_LWIP_DNS_TABLE_SIZE", &dnsTableSize, configLine}, + {"CONFIG_LWIP_SOCKET", &v, configLine}, + {"CONFIG_LWIP_DEBUG", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + } + + return items +} + +// matchLibsKconfig performs the matching between Kconfig entries and micro-libs +// and updates the right Kconfigs +// +// It returns a list of KConfig. +func addInternalConfig(conf string, kConfigMap map[string]*KConfig, + items []*KConfig) []*KConfig { + v := "y" + switch conf { + case "CONFIG_PLAT_XEN": + configs := []*KConfig{ + {"CONFIG_PLAT_XEN", &v, configLine}, + {"CONFIG_XEN_HVMLITE", nil, commentedConfigLine}, + {"", nil, lineFeed}, + {"#", nil, separatorLine}, + {"# Console Options", nil, headerLine}, + {"#", nil, separatorLine}, + {"CONFIG_XEN_KERNEL_HV_CONSOLE", &v, configLine}, + {"CONFIG_XEN_KERNEL_EMG_CONSOLE", nil, commentedConfigLine}, + {"CONFIG_XEN_DEBUG_HV_CONSOLE", &v, configLine}, + {"CONFIG_XEN_DEBUG_EMG_CONSOLE", nil, commentedConfigLine}, + {"CONFIG_XEN_PV_BUILD_P2M", &v, configLine}, + {"CONFIG_XEN_GNTTAB", &v, configLine}, + {"CONFIG_XEN_XENBUS", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_PLAT_KVM": + configs := []*KConfig{ + {"CONFIG_PLAT_KVM", &v, configLine}, + {"", nil, lineFeed}, + {"#", nil, separatorLine}, + {"# Console Options", nil, headerLine}, + {"#", nil, separatorLine}, + {"CONFIG_KVM_KERNEL_SERIAL_CONSOLE", &v, configLine}, + {"CONFIG_KVM_KERNEL_VGA_CONSOLE", &v, configLine}, + {"CONFIG_KVM_DEBUG_SERIAL_CONSOLE", &v, configLine}, + {"CONFIG_KVM_DEBUG_VGA_CONSOLE", &v, configLine}, + {"CONFIG_KVM_PCI", &v, configLine}, + {"CONFIG_VIRTIO_BUS", &v, configLine}, + {"", nil, lineFeed}, + {"#", nil, separatorLine}, + {"# Virtio", nil, headerLine}, + {"#", nil, separatorLine}, + {"CONFIG_VIRTIO_PCI", nil, commentedConfigLine}, + {"CONFIG_VIRTIO_NET", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_PLAT_LINUXU": + heapSize := "4" + configs := []*KConfig{ + {"CONFIG_PLAT_LINUXU", &v, configLine}, + {"CONFIG_LINUXU_DEFAULT_HEAPMB", &heapSize, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBUKBOOT": + var number = "60" + configs := []*KConfig{ + {"CONFIG_LIBUKBOOT", &v, configLine}, + {"CONFIG_LIBUKBOOT_BANNER", &v, configLine}, + {"CONFIG_LIBUKBOOT_MAXNBARGS", &number, configLine}, + {"CONFIG_LIBUKBOOT_INITALLOC", &v, configLine}, + {"CONFIG_LIBUKDEBUG", &v, configLine}, + {"CONFIG_LIBUKDEBUG_PRINTK", &v, configLine}, + {"CONFIG_LIBUKDEBUG_PRINTK_INFO", &v, configLine}, + + {"CONFIG_LIBUKDEBUG_PRINTK_WARN", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_PRINTK_ERR", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_PRINTK_CRIT", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_PRINTD", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_NOREDIR", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_REDIR_PRINTD", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_REDIR_PRINTK", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_PRINT_TIME", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_PRINT_STACK", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_ENABLE_ASSERT", nil, commentedConfigLine}, + {"CONFIG_LIBUKDEBUG_TRACEPOINTS", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBNOLIBC": + configs := []*KConfig{ + {"CONFIG_LIBNOLIBC", nil, commentedConfigLine}, + {"CONFIG_LIBNOLIBC_UKDEBUG_ASSERT", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBUKALLOC": + configs := []*KConfig{ + {"CONFIG_LIBUKALLOC", &v, configLine}, + {"CONFIG_LIBUKALLOC_IFPAGES", &v, configLine}, + {"CONFIG_LIBUKALLOC_IFSTATS", nil, commentedConfigLine}, + {"CONFIG_LIBUKALLOCBBUDDY", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBUKSCHED": + configs := []*KConfig{ + {"CONFIG_LIBUKSCHED", &v, configLine}, + {"CONFIG_LIBUKSCHEDCOOP", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBUKMPI": + configs := []*KConfig{ + {"CONFIG_LIBUKMPI", nil, commentedConfigLine}, + {"CONFIG_LIBUKMPI_MBOX", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBUKSWRAND": + configs := []*KConfig{ + {"CONFIG_LIBUKSWRAND_MWC", nil, commentedConfigLine}, + {"CONFIG_LIBUKSWRAND_INITIALSEED", nil, commentedConfigLine}, + {"CONFIG_DEV_RANDOM", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBUKNETDEV": + configs := []*KConfig{ + {"CONFIG_LIBUKNETDEV_MAXNBQUEUES", nil, commentedConfigLine}, + {"CONFIG_LIBUKNETDEV_DISPATCHERTHREADS", nil, commentedConfigLine}, + } + items = SetConfig(configs, kConfigMap, items) + case "CONFIG_LIBUKLOCK": + configs := []*KConfig{ + {"CONFIG_LIBUKLOCK", &v, configLine}, + {"CONFIG_LIBUKLOCK_SEMAPHORE", &v, configLine}, + {"CONFIG_LIBUKLOCK_MUTEX", &v, configLine}, + } + items = SetConfig(configs, kConfigMap, items) + } + + return items +} diff --git a/contrib/gotools/srcs/buildtool/makefile_process.go b/contrib/gotools/srcs/buildtool/makefile_process.go new file mode 100644 index 00000000..2b3e1847 --- /dev/null +++ b/contrib/gotools/srcs/buildtool/makefile_process.go @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package buildtool + +import ( + "path/filepath" + "strings" + + u "tools/srcs/common" +) + +// ----------------------------Generate Makefile-------------------------------- + +// generateMakefile generates a 'Makefile' file for the Unikraft build system. +// +// It returns an error if any, otherwise it returns nil. +func generateMakefile(filename, unikraftPath, appFolder string, + matchedLibs []string, externalLibs map[string]string) error { + + var sb strings.Builder + + // Set unikraft root and libs workspace + sb.WriteString("UK_ROOT ?= " + unikraftPath + "unikraft\n" + + "UK_LIBS ?= " + unikraftPath + "libs\n") + + var libC = "" + // Add external libs + sb.WriteString("LIBS := ") + if len(matchedLibs) > 0 { + for _, lib := range matchedLibs { + // Only write external libs + if _, ok := externalLibs[lib]; ok { + if strings.Compare(NEWLIB, lib) == 0 || + strings.Compare(MUSL, lib) == 0 { + libC = lib + } else { + sb.WriteString("$(UK_LIBS)/" + lib + ":") + } + } + } + } + + // Write libC at the end to avoid conflicts + if len(libC) > 0 { + sb.WriteString("$(UK_LIBS)/" + libC) + } + + sb.WriteString("\n\n") + + // Bind UK_ROOT to make + sb.WriteString("all:\n" + + "\t@make -C $(UK_ROOT) A=" + appFolder + " L=$(LIBS)\n\n" + + "$(MAKECMDGOALS):\n" + + "\t@make -C $(UK_ROOT) A=" + appFolder + " L=$(LIBS) $(MAKECMDGOALS)\n") + + // Save the content to Makefile + return u.WriteToFile(filename, []byte(sb.String())) +} + +// typeFile determines the type of a given file. +// +// It returns a string that represents the used language. +func typeFile(filename string) string { + var extension = filepath.Ext(filename) + var flag string + switch extension { + case ".c": + flag = "C" + case ".cc": + case ".cpp": + flag = "CXX" + case ".S": + case ".asm": + flag = "AS" + } + return flag +} + +// generateMakefileUK generates a 'Makefile.uk' file for the Unikraft build +// system. +// +// It returns an error if any, otherwise it returns nil. +func generateMakefileUK(filename, programName, filetype string, + makefileLines string, sourceFiles []string) error { + + var sb strings.Builder + + // Add app registration + sb.WriteString("########################################" + + "########################################\n" + + "# App registration\n" + + "########################################" + + "########################################\n" + + "$(eval $(call addlib,app" + strings.ToLower(programName) + "))\n\n") + + // Add app includes (headers) + sb.WriteString("########################################" + + "########################################\n" + + "# App includes\n" + + "########################################" + + "########################################\n" + + "CINCLUDES-y += -I$(APP" + strings.ToUpper(programName) + "_BASE)" + + "/include\n\n") + + // Add app global flags + sb.WriteString("########################################" + + "########################################\n" + + "# Global flags\n" + + "########################################" + + "########################################\n" + + "# Suppress some warnings to make the build process look neater\n" + + "SUPPRESS_FLAGS += -Wno-unused-parameter " + + "-Wno-unused-variable -Wno-nonnull \\\n" + + "-Wno-unused-but-set-variable -Wno-unused-label " + + "-Wno-char-subscripts \\\n-Wno-unused-function " + + "-Wno-missing-field-initializers -Wno-uninitialized \\\n" + + "-Wno-array-bounds -Wno-maybe-uninitialized " + + "-Wno-pointer-sign -Wno-unused-value \\\n" + + "-Wno-unused-macros -Wno-parentheses " + + "-Wno-implicit-function-declaration \\\n" + + "-Wno-missing-braces -Wno-endif-labels " + + "-Wno-unused-but-set-variable \\\n" + + "-Wno-implicit-function-declaration -Wno-type-limits " + + "-Wno-sign-compare\n\n") + + // Add SUPPRESS Flags + sb.WriteString("APP" + strings.ToUpper(programName) + "_" + + typeFile(filetype) + "FLAGS-y +=" + "$(SUPPRESS_FLAGS)\n\n" + + "# ADD the flags of your app HERE\n\n") + + // Add additional lines + if len(makefileLines) > 0 { + b, _ := u.OpenTextFile(makefileLines) + for _, line := range strings.Split(string(b), "\n") { + if len(line) > 0 { + sb.WriteString("APP" + strings.ToUpper(programName) + + "_CFLAGS-y += " + line + "\n") + } + } + } + + // Add source files + sb.WriteString("########################################" + + "########################################\n" + + "# " + programName + "sources\n" + + "########################################" + + "########################################\n") + + for _, s := range sourceFiles { + sb.WriteString("APP" + strings.ToUpper(programName) + + "_SRCS-y += $(APP" + strings.ToUpper(programName) + + "_BASE)/" + s + "\n") + } + + // Save the content to Makefile.uk + return u.WriteToFile(filename, []byte(sb.String())) +} diff --git a/contrib/gotools/srcs/buildtool/microlibs_definitions.go b/contrib/gotools/srcs/buildtool/microlibs_definitions.go new file mode 100644 index 00000000..9ce4d6f6 --- /dev/null +++ b/contrib/gotools/srcs/buildtool/microlibs_definitions.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package buildtool + +// Exported const representing micro-libs definitions. +const ( + CARES = "c-ares" + COMPILERRT = "compiler-rt" + DEVFS = "devfs" + DPDK = "dpdk" + EIGEN = "eigen" + FP16 = "fp16" + FXDIV = "fxdiv" + INTELINTRINSICS = "intel-intrinsics" + LIBAXTLS = "libaxtls" + LIBBLOCK = "libblock" + LIBCXX = "libcxx" + LIBCXXABI = "libcxxabi" + LIBGO = "libgo" + LIBJVM = "libjvm" + LIBPROC = "libproc" + LIBRUBY = "libruby" + LIBRUST = "librust" + LIBUKSCHEDPREEMPT = "libukschedpreempt" + LIBUNWIND = "libunwind" + LIBUUID = "libuuid" + LIBUV = "libuv" + LIBV8 = "libv8" + LWIP = "lwip" + MICROPYTHON = "micropython" + MUSL = "musl" + NEWLIB = "newlib" + NOBLIM = "noblim" + NOLIBC = "nolibc" + OPENSSL = "openssl" + PFS9 = "9pfs" + POSIXLIBDL = "posix-libdl" + POSIXPROCESS = "posix-process" + POSIXUSER = "posix-user" + PTHREADEMBEDDED = "pthread-embedded" + PTHREADPOOL = "pthreadpool" + PYTHON = "python" + RAMFS = "ramfs" + SYSCALLSHIM = "syscallshim" + UKALLOC = "ukalloc" + UKALLOCBBUDDY = "ukallocbbuddy" + UKARGPARSE = "ukargparse" + UKBOOT = "ukboot" + UKBUS = "ukbus" + UKDEBUG = "ukdebug" + UKLOCK = "uklock" + UKMPI = "ukmpi" + UKNETDEV = "uknetdev" + UKPCI = "ukpci" + UKSCHED = "uksched" + UKSCHEDCOOP = "ukschedcoop" + UKSGLIST = "uksglist" + UKSYSINFO = "uksysinfo" + UKSWRAND = "ukswrand" + UKTIMECONV = "uktimeconv" + UKTIME = "uktime" + VFSCORE = "vfscore" + ZLIB = "zlib" +) diff --git a/contrib/gotools/srcs/buildtool/microlibs_process.go b/contrib/gotools/srcs/buildtool/microlibs_process.go new file mode 100644 index 00000000..39a85490 --- /dev/null +++ b/contrib/gotools/srcs/buildtool/microlibs_process.go @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package buildtool + +import ( + "io/ioutil" + "strings" + "sync" + u "tools/srcs/common" +) + +const ( + exportFile = "exportsyms.uk" + prefixUrl = "http://xenbits.xen.org/gitweb/?p=unikraft/libs/" + suffixUrl = ";a=blob_plain;f=exportsyms.uk;hb=refs/heads/staging" +) + +// -----------------------------Match micro-libs-------------------------------- + +// processSymbols adds symbols within the 'exportsyms.uk' file into a map. +// +func processSymbols(microLib, output string, mapSymbols map[string][]string) { + + lines := strings.Split(output, "\n") + for _, line := range lines { + if len(line) > 0 && !strings.Contains(line, "#") && + strings.Compare(line, "none") != 0 { + mapSymbols[line] = append(mapSymbols[line], microLib) + } + } +} + +// fetchSymbolsInternalLibs fetches all symbols within 'exportsyms.uk' files +// from Unikraft's internal libs and add them into a map. +// +// It returns an error if any, otherwise it returns nil. +func fetchSymbolsInternalLibs(unikraftLibs string, + microLibs map[string][]string) error { + + // Read files within the Unikraft directory + files, err := ioutil.ReadDir(unikraftLibs) + if err != nil { + return err + } + + // Read Unikraft internal libs symbols (exportsyms.uk) + for _, f := range files { + if f.IsDir() { + export := unikraftLibs + f.Name() + u.SEP + exportFile + if exists, _ := u.Exists(export); exists { + u.PrintInfo("Retrieving symbols of internal lib: " + f.Name()) + b, _ := u.OpenTextFile(export) + processSymbols(f.Name(), string(b), microLibs) + } + } + } + return nil +} + +// fetchSymbolsExternalLibs fetches all symbols within 'exportsyms.uk' files +// from Unikraft's external libs and add them into a map. +// +// It returns a list of symbols and an error if any, otherwise it returns nil. +func fetchSymbolsExternalLibs(url string, + microLibs map[string][]string) (map[string]string, error) { + + var externalLibs map[string]string + if body, err := u.DownloadFile(url); err != nil { + return nil, err + } else { + externalLibs = u.GitFindExternalLibs(*body) + + var wg sync.WaitGroup + wg.Add(len(externalLibs)) + // Iterate through all external libs to parse 'exportsyms.uk' file + for lib, git := range externalLibs { + // Use go routine to get better efficiency + go func(lib, git string, microLibs map[string][]string) { + defer wg.Done() + u.PrintInfo("Retrieving symbols of external lib: " + lib) + if symbols, err := u.DownloadFile(prefixUrl + git + suffixUrl); err != nil { + u.PrintWarning(err) + } else { + processSymbols(lib, *symbols, microLibs) + } + }(lib, git, microLibs) + } + wg.Wait() + } + return externalLibs, nil +} + +// matchSymbols performs the matching between Unikraft's micro-libs and +// libraries used by a given application based on the list of symbols that both +// contain. +// +// It returns a list of micro-libs that are required by the application +func matchSymbols(matchedLibs []string, data map[string]string, + microLibs map[string][]string) []string { + for key := range data { + if values, ok := microLibs[key]; ok { + for _, value := range values { + + // todo remove + if strings.Compare(NOLIBC, value) == 0 { + value = NEWLIB + } + // remove above + + if !u.Contains(matchedLibs, value) { + matchedLibs = append(matchedLibs, value) + } + } + } + } + + return matchedLibs +} + +// matchLibs performs the matching between Unikraft's micro-libs and +// libraries used by a given application +// +// It returns a list of micro-libs that are required by the application and an +// error if any, otherwise it returns nil. +func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string, error) { + + mapSymbols := make(map[string][]string) + + matchedLibs := make([]string, 0) + if err := fetchSymbolsInternalLibs(unikraftLibs, mapSymbols); err != nil { + return nil, nil, err + } + + // Get list of libs from xenbits + url := "http://xenbits.xen.org/gitweb/?pf=unikraft/libs" + externalLibs, err := fetchSymbolsExternalLibs(url, mapSymbols) + if err != nil { + return nil, nil, err + } + + // Perform the matching symbols on static data + matchedLibs = matchSymbols(matchedLibs, data.StaticData.Symbols, mapSymbols) + + // Perform the matching symbols on dynamic data + matchedLibs = matchSymbols(matchedLibs, data.DynamicData.Symbols, mapSymbols) + + return matchedLibs, externalLibs, nil +} + +// -----------------------------Clone micro-libs-------------------------------- + +// cloneGitRepo clones a specific git repository that hosts an external +// micro-libs on http://xenbits.xen.org/ +// +// It returns an error if any, otherwise it returns nil. +func cloneGitRepo(url, unikraftPathLibs string) error { + + u.PrintInfo("Clone git repository " + url) + if _, _, err := u.GitCloneRepository(url, unikraftPathLibs, true); err != nil { + return err + } + u.PrintOk("Git repository " + url + " has been cloned into " + + unikraftPathLibs) + + u.PrintInfo("Git branch " + url) + if _, _, err := u.GitBranchStaging(unikraftPathLibs, true); err != nil { + return err + } + + return nil +} + +// cloneLibsFolders clones all the needed micro-libs that are needed by a +// given application +// +func cloneLibsFolders(unikraftPath string, matchedLibs []string, + externalLibs map[string]string) { + + for _, lib := range matchedLibs { + if _, ok := externalLibs[lib]; ok { + exists, _ := u.Exists(unikraftPath + u.LIBSFOLDER + lib) + if !exists { + // If the micro-libs is not in the local host, clone it + if err := cloneGitRepo("git://xenbits.xen.org/unikraft/"+ + "libs/"+lib+".git", unikraftPath+u.LIBSFOLDER); err != nil { + u.PrintWarning(err) + } + } else { + u.PrintInfo("Library " + lib + " already exists in folder" + + unikraftPath + u.LIBSFOLDER) + } + } + } +} diff --git a/contrib/gotools/srcs/buildtool/run_buildtool.go b/contrib/gotools/srcs/buildtool/run_buildtool.go new file mode 100644 index 00000000..20cdf8b6 --- /dev/null +++ b/contrib/gotools/srcs/buildtool/run_buildtool.go @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package buildtool + +import ( + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + + u "tools/srcs/common" +) + +// STATES +const ( + compilerError = iota + linkingError + success +) + +// -----------------------------Generate Config--------------------------------- + +// generateConfigUk generates a 'Config.uk' file for the Unikraft build system. +// +// It returns an error if any, otherwise it returns nil. +func generateConfigUk(filename, programName string, matchedLibs []string) error { + + var sb strings.Builder + sb.WriteString("### Invisible option for dependencies\n" + + "config APP" + programName + "_DEPENDENCIES\n" + "\tbool\n" + + "\tdefault y\n") + + for _, lib := range matchedLibs { + sb.WriteString("\tselect " + lib + "\n") + } + + // Save the content to Makefile.uk + return u.WriteToFile(filename, []byte(sb.String())) +} + +// ---------------------------Process make output------------------------------- + +// checkMakeOutput checks if errors or warning are displayed during the +// execution of the 'make' command. +// +// It returns an integer that defines the result of 'make': +// +func checkMakeOutput(appFolder string, stderr *string) int { + + if stderr == nil { + return success + } + + // Linking errors during make + if strings.Contains(*stderr, "undefined") { + + str := parseMakeOutput(*stderr) + if len(str) > 0 { + if err := u.WriteToFile(appFolder+"stub.c", []byte(str)); err != nil { + u.PrintWarning(err) + } + } + + return linkingError + } + + // Compiler errors during make + if strings.Contains(*stderr, "error:") { + + return compilerError + } + + return success +} + +// parseMakeOutput parses the output of the 'make' command. +// +// It returns a string that contains stubs of undefined function(s). +func parseMakeOutput(output string) string { + + var sb strings.Builder + sb.WriteString("#include \n") + + undefinedSymbols := make(map[string]*string) + var re = regexp.MustCompile(`(?mi).*undefined reference to\s\x60(.*)'`) + for _, match := range re.FindAllStringSubmatch(output, -1) { + if _, ok := undefinedSymbols[match[1]]; !ok { + sb.WriteString("void ") + sb.WriteString(match[1]) + sb.WriteString("(void){\n\tprintf(\"STUB\\n\");\n}\n\n") + undefinedSymbols[match[1]] = nil + u.PrintInfo("Add stub to function: " + match[1]) + } + } + + return sb.String() +} + +// -------------------------------------Run------------------------------------- + +// RunBuildTool runs the automatic build tool to build a unikernel of a +// given application. +// +func RunBuildTool(homeDir string, data *u.Data) { + + // Init and parse local arguments + args := new(u.Arguments) + p, err := args.InitArguments() + if err != nil { + u.PrintErr(err) + } + if err := parseLocalArguments(p, args); err != nil { + u.PrintErr(err) + } + + // Get program Name + programName := *args.StringArg[programArg] + + // Take base path if absolute path is used + if filepath.IsAbs(programName) { + programName = filepath.Base(programName) + } + + var unikraftPath string + if len(*args.StringArg[unikraftArg]) == 0 { + path, err := setUnikraftFolder(homeDir + u.SEP) + if err != nil { + u.PrintErr(err) + } + unikraftPath = *path + } else { + unikraftPath = *args.StringArg[unikraftArg] + } + + // Check if sources argument is set + if len(*args.StringArg[sourcesArg]) == 0 { + u.PrintErr("sources argument '-s' must be set") + } + + // Check if the unikraft folder contains the 3 required folders + if _, err := ioutil.ReadDir(unikraftPath); err != nil { + u.PrintErr(err) + } else { + path, err := setUnikraftSubFolders(homeDir + u.SEP + u.UNIKRAFTFOLDER) + if err != nil { + u.PrintErr(err) + } + unikraftPath = *path + } + + // If data is not initialized, read output from dependency analysis tool + if data == nil { + u.PrintInfo("Initialize data") + outFolder := homeDir + u.SEP + programName + "_" + u.OUTFOLDER + if data, err = u.ReadDataJson(outFolder+programName, data); err != nil { + u.PrintErr(err) + } + } + + // Create unikraft application path + appFolderPtr, err := createUnikraftApp(programName, unikraftPath) + if err != nil { + u.PrintErr(err) + } + appFolder := *appFolderPtr + + // Create the folder 'include' if it does not exist + includeFolder, err := createIncludeFolder(appFolder) + if err != nil { + u.PrintErr(err) + } + + // Get sources files + sourcesPath := *args.StringArg[sourcesArg] + + // Copy all .h into the include folder + sourceFiles, includesFiles := make([]string, 0), make([]string, 0) + + // Move source files to Unikraft folder + if sourceFiles, err = processSourceFiles(sourcesPath, appFolder, *includeFolder, + sourceFiles, includesFiles); err != nil { + u.PrintErr(err) + } + + // Match micro-libs + matchedLibs, externalLibs, err := matchLibs(unikraftPath+"unikraft"+u.SEP+ + "lib"+u.SEP, data) + if err != nil { + u.PrintErr(err) + } + + // Clone the external git repositories + cloneLibsFolders(unikraftPath, matchedLibs, externalLibs) + + // Match internal dependencies between micro-libs + if err := searchInternalDependencies(unikraftPath, &matchedLibs, + externalLibs); err != nil { + u.PrintErr(err) + } + + for _, lib := range matchedLibs { + u.PrintOk("Match lib: " + lib) + } + + // Clone the external git repositories (if changed) + cloneLibsFolders(unikraftPath, matchedLibs, externalLibs) + + // Generate Makefiles + if err := generateMake(programName, appFolder, unikraftPath, *args.StringArg[makefileArg], + matchedLibs, sourceFiles, externalLibs); err != nil { + u.PrintErr(err) + } + + // Delete Build folder + deleteBuildFolder(appFolder) + + // Initialize config files + initConfig(appFolder, matchedLibs) + + // Run make + runMake(programName, appFolder) +} + +func searchInternalDependencies(unikraftPath string, matchedLibs *[]string, + externalLibs map[string]string) error { + + for _, lib := range *matchedLibs { + // Consider only external libs + if _, ok := externalLibs[lib]; ok { + + // Get and read Config.UK from external lib + configUk := unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk" + lines, err := u.ReadLinesFile(configUk) + if err != nil { + return err + } + + // Process Config.UK file + mapConfig := make(map[string][]string) + u.ProcessConfigUK(lines, false, mapConfig, nil) + + for config := range mapConfig { + + // Remove LIB prefix + if strings.Contains(config, "LIB") { + config = strings.TrimPrefix(config, "LIB") + } + + // Replace underscore by dash + if strings.Contains(config, "_") { + config = strings.ReplaceAll(config, "_", "-") + } + + // Check if matchedLibs already contains the lib + config = strings.ToLower(config) + if !u.Contains(*matchedLibs, config) { + *matchedLibs = append(*matchedLibs, config) + } + } + } + } + + return nil +} + +func generateMake(programName, appFolder, unikraftPath, makefile string, + matchedLibs, sourceFiles []string, externalLibs map[string]string) error { + // Generate Makefile + if err := generateMakefile(appFolder+"Makefile", unikraftPath, + appFolder, matchedLibs, externalLibs); err != nil { + return err + } + + // Generate Config.uk + if err := generateConfigUk(appFolder+"Config.uk", + strings.ToUpper(programName), matchedLibs); err != nil { + return err + } + + // Get the file type for Unikraft flag + fileType := languageUsed() + + // Generate Makefile.uk + if err := generateMakefileUK(appFolder+"Makefile.uk", programName, + fileType, makefile, sourceFiles); err != nil { + return err + } + + return nil +} + +func deleteBuildFolder(appFolder string) { + // Delete build folder if already exists + if file, err := u.OSReadDir(appFolder); err != nil { + u.PrintWarning(err) + } else { + for _, f := range file { + if f.IsDir() && f.Name() == "build" { + u.PrintWarning("build folder already exists. Delete it.") + if err := os.RemoveAll(appFolder + "build"); err != nil { + u.PrintWarning(err) + } + } + } + } +} + +func initConfig(appFolder string, matchedLibs []string) { + + // Run make allNoConfig to generate a .config file + if strOut, strErr, err := u.ExecuteWaitCommand(appFolder, "make", + "allnoconfig"); err != nil { + u.PrintErr(err) + } else if len(*strErr) > 0 { + u.PrintErr("error during generating .config: " + *strErr) + } else if len(*strOut) > 0 && !strings.Contains(*strOut, + "configuration written") { + u.PrintWarning("Default .config cannot be generated") + } + + // Parse .config + kConfigMap := make(map[string]*KConfig) + items := make([]*KConfig, 0) + items, err := parseConfig(appFolder+".config", kConfigMap, items, + matchedLibs) + if err != nil { + u.PrintErr(err) + } + + // Update .config + items = updateConfig(kConfigMap, items) + + // Write .config + if err := writeConfig(appFolder+".config", items); err != nil { + u.PrintErr(err) + } +} + +func runMake(programName, appFolder string) { + // Run make + stdout, stderr, _ := u.ExecuteRunCmd("make", appFolder, true) + + // Check the state of the make command + state := checkMakeOutput(appFolder, stderr) + if state == linkingError { + + // Add new stub.c in Makefile.uk + d := "APP" + strings.ToUpper(programName) + + "_SRCS-y += $(APP" + strings.ToUpper(programName) + + "_BASE)/stub.c" + if err := u.UpdateFile(appFolder+"Makefile.uk", []byte(d)); err != nil { + u.PrintErr(err) + } + + // Run make a second time + stdout, stderr, _ = u.ExecuteRunCmd("make", appFolder, true) + + // Check the state of the make command + checkMakeOutput(appFolder, stderr) + } + + out := appFolder + programName + + // Save make output into warnings.txt if warnings are here + if stderr != nil && strings.Contains(*stderr, "warning:") { + if err := u.WriteToFile(out+"_warnings.txt", []byte(*stderr)); err != nil { + u.PrintWarning(err) + } else { + u.PrintInfo("Warnings are written in file: " + out + "_warnings.txt") + } + } + + // Save make output into output.txt + if stdout != nil { + if err := u.WriteToFile(out+"_output.txt", []byte(*stdout)); err != nil { + u.PrintWarning(err) + } else { + u.PrintInfo("Output is written in file: " + out + "_output.txt") + } + } + + if state == compilerError { + u.PrintErr("Fix compilation errors") + } else if state == success { + u.PrintOk("Unikernel created in Folder: 'build/'") + } +} diff --git a/contrib/gotools/srcs/buildtool/unikraft_files_process.go b/contrib/gotools/srcs/buildtool/unikraft_files_process.go new file mode 100644 index 00000000..3c4d881d --- /dev/null +++ b/contrib/gotools/srcs/buildtool/unikraft_files_process.go @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package buildtool + +import ( + "bufio" + "errors" + "fmt" + "os" + "path/filepath" + + u "tools/srcs/common" +) + +// ---------------------------Create Include Folder----------------------------- + +func createIncludeFolder(appFolder string) (*string, error) { + + includeFolder := appFolder + u.INCLUDEFOLDER + if _, err := u.CreateFolder(includeFolder); err != nil { + return nil, err + } + + return &includeFolder, nil +} + +// ----------------------------Set UNIKRAFT Folders----------------------------- +func setUnikraftFolder(homeDir string) (*string, error) { + + unikraftFolder := homeDir + u.UNIKRAFTFOLDER + + created, err := u.CreateFolder(unikraftFolder) + if err != nil { + return nil, err + } + + if created { + setUnikraftSubFolders(unikraftFolder) + } else { + u.PrintInfo("Unikraft folder already exists") + return &unikraftFolder, nil + } + + return &unikraftFolder, nil +} + +func setUnikraftSubFolders(unikraftFolder string) (*string, error) { + + u.PrintInfo("Create Unikraft folder with apps and libs subfolders") + + // Create 'apps' and 'libs' subfolders + if _, err := u.CreateFolder(unikraftFolder + u.APPSFOLDER); err != nil { + return nil, err + } + + if _, err := u.CreateFolder(unikraftFolder + u.LIBSFOLDER); err != nil { + return nil, err + } + + // Download git repo of unikraft + if _, _, err := u.GitCloneRepository("git://xenbits.xen.org/unikraft/unikraft.git", + unikraftFolder, true); err != nil { + return nil, err + } + + // Use staging branch + if _, _, err := u.GitBranchStaging(unikraftFolder+"unikraft", true); err != nil { + return nil, err + } + + return &unikraftFolder, nil +} + +// ---------------------------Check UNIKRAFT Folder----------------------------- + +func containsUnikraftFolders(files []os.FileInfo) bool { + + if len(files) == 0 { + return false + } + + m := make(map[string]bool) + m[u.APPSFOLDER], m[u.LIBSFOLDER], m[u.UNIKRAFTFOLDER] = false, false, false + + var folderName string + for _, f := range files { + folderName = f.Name() + u.SEP + if _, ok := m[folderName]; ok { + m[folderName] = true + } + } + + return m[u.APPSFOLDER] == true && m[u.LIBSFOLDER] && m[u.UNIKRAFTFOLDER] +} + +// ---------------------------UNIKRAFT APP FOLDER------------------------------- + +func createUnikraftApp(programName, unikraftPath string) (*string, error) { + + var appFolder string + if unikraftPath[len(unikraftPath)-1] != os.PathSeparator { + appFolder = unikraftPath + u.SEP + u.APPSFOLDER + programName + u.SEP + } else { + appFolder = unikraftPath + u.APPSFOLDER + programName + u.SEP + } + + created, err := u.CreateFolder(appFolder) + if err != nil { + return nil, err + } + + if !created { + u.PrintWarning(appFolder + " already exists.") + handleCreationApp(&appFolder) + } + + return &appFolder, nil +} + +// -----------------------------Create App folder------------------------------- + +func handleCreationApp(appFolder *string) { + fmt.Println("Make your choice:\n1: Copy and overwrite files\n2: " + + "Enter manually the name of the folder\n3: exit program") + var input int + for true { + fmt.Print("Please enter your choice (0 to exit): ") + if _, err := fmt.Scanf("%d", &input); err != nil { + u.PrintWarning("Choice must be numeric! Try again") + } else { + switch input { + case 1: + return + case 2: + fmt.Print("Enter text: ") + reader := bufio.NewReader(os.Stdin) + text, _ := reader.ReadString('\n') + appFolder = &text + return + case 3: + os.Exit(1) + default: + u.PrintWarning("Invalid input! Try again") + } + } + } +} + +// -------------------------MOVE FILES TO APP FOLDER---------------------------- + +var srcLanguages = map[string]int{ + ".c": 0, + ".cpp": 0, + ".cc": 0, + ".S": 0, + ".asm": 0, + //".py": 0, + //".go": 0, +} + +func processSourceFiles(sourcesPath, appFolder, includeFolder string, + sourceFiles, includesFiles []string) ([]string, error) { + + err := filepath.Walk(sourcesPath, func(path string, info os.FileInfo, + err error) error { + + if !info.IsDir() { + extension := filepath.Ext(info.Name()) + if _, ok := srcLanguages[extension]; ok { + // Add source files to sourceFiles list + sourceFiles = append(sourceFiles, info.Name()) + + // Count the number of extension + srcLanguages[extension] += 1 + + // Copy source files to the appFolder + if err = u.CopyFileContents(path, appFolder+info.Name()); err != nil { + return err + } + } else if extension == ".h" { + // Add source files to includesFiles list + includesFiles = append(includesFiles, info.Name()) + + // Copy header files to the INCLUDEFOLDER + if err = u.CopyFileContents(path, includeFolder+info.Name()); err != nil { + return err + } + } else { + u.PrintWarning("Unsupported extension for file: " + info.Name()) + } + } + + return nil + }) + + if err != nil { + return nil, err + } + + // If no source file, exit the program + if len(sourceFiles) == 0 { + return nil, errors.New("unable to find source files") + } + + return sourceFiles, nil +} + +func languageUsed() string { + + max := -1 + var mostUsedFiles string + for key, value := range srcLanguages { + if max < value { + max = value + mostUsedFiles = key + } + } + + return mostUsedFiles +} diff --git a/contrib/gotools/srcs/common/arguments.go b/contrib/gotools/srcs/common/arguments.go new file mode 100644 index 00000000..6b262e22 --- /dev/null +++ b/contrib/gotools/srcs/common/arguments.go @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +import ( + "errors" + "os" + "strings" + + "github.com/akamensky/argparse" +) + +// Exported constants to determine arguments type. +const ( + INT = iota + BOOL + STRING +) + +// Exported constants to determine which tool is used. +const ( + CRAWLER = "crawler" + DEP = "dep" + BUILD = "build" + VERIF = "verif" + PERF = "perf" +) + +const ( + unknownArgs = "unknown arguments" +) + +// Exported constants to represent different types of arguments. +type Arguments struct { + IntArg map[string]*int + BoolArg map[string]*bool + StringArg map[string]*string +} + +// InitArguments allows to initialize the parser in order to parse given +// arguments. +// +// It returns a parser as well as an error if any, otherwise it returns nil. +func (args *Arguments) InitArguments() (*argparse.Parser, error) { + + args.IntArg = make(map[string]*int) + args.BoolArg = make(map[string]*bool) + args.StringArg = make(map[string]*string) + + p := argparse.NewParser("UNICORE toolchain", + "The UNICORE toolchain allows to build unikernels") + + return p, nil +} + +// ParserWrapper parses arguments of the application and skips unknownArgs +// error in order to use different levels of arguments parsing. +// +// It returns an error if any, otherwise it returns nil. +func ParserWrapper(p *argparse.Parser, args []string) error { + + err := p.Parse(args) + if err != nil && strings.Contains(err.Error(), unknownArgs) { + return nil + } + + return err +} + +// ParseArguments parses arguments of the application. +// +// It returns an error if any, otherwise it returns nil. +func (*Arguments) ParseMainArguments(p *argparse.Parser, args *Arguments) error { + + if args == nil { + return errors.New("args structure should be initialized") + } + + args.InitArgParse(p, args, BOOL, "", CRAWLER, + &argparse.Options{Required: false, Default: false, + Help: "Execute the crawler unikraft tool"}) + args.InitArgParse(p, args, BOOL, "", DEP, + &argparse.Options{Required: false, Default: false, + Help: "Execute only the dependency analysis tool"}) + args.InitArgParse(p, args, BOOL, "", BUILD, + &argparse.Options{Required: false, Default: false, + Help: "Execute only the automatic build tool"}) + args.InitArgParse(p, args, BOOL, "", VERIF, + &argparse.Options{Required: false, Default: false, + Help: "Execute only the verification tool"}) + args.InitArgParse(p, args, BOOL, "", PERF, + &argparse.Options{Required: false, Default: false, + Help: "Execute only the performance tool"}) + + // Parse only the two first arguments + if len(os.Args) > 2 { + return ParserWrapper(p, os.Args[:2]) + } else { + p.Parse(os.Args) + } + + return nil +} + +// InitArgParse initializes the Arguments structure depending the type of +// the variable. +func (*Arguments) InitArgParse(p *argparse.Parser, args *Arguments, typeVar int, + short, long string, options *argparse.Options) { + switch typeVar { + case INT: + args.IntArg[long] = new(int) + args.IntArg[long] = p.Int(short, long, options) + case BOOL: + args.BoolArg[long] = new(bool) + args.BoolArg[long] = p.Flag(short, long, options) + case STRING: + args.StringArg[long] = new(string) + args.StringArg[long] = p.String(short, long, options) + } +} diff --git a/contrib/gotools/srcs/common/data.go b/contrib/gotools/srcs/common/data.go new file mode 100644 index 00000000..6cf35a0a --- /dev/null +++ b/contrib/gotools/srcs/common/data.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +// Exported struct that represents static and dynamic data. +type Data struct { + StaticData StaticData `json:"static_data"` + DynamicData DynamicData `json:"dynamic_data"` +} + +// Exported struct that represents data for static dependency analysis. +type StaticData struct { + Dependencies map[string][]string `json:"dependencies"` + SharedLibs map[string][]string `json:"shared_libs"` + SystemCalls map[string]string `json:"system_calls"` + Symbols map[string]string `json:"symbols"` +} + +// Exported struct that represents data for dynamic dependency analysis. +type DynamicData struct { + SharedLibs map[string][]string `json:"shared_libs"` + SystemCalls map[string]string `json:"system_calls"` + Symbols map[string]string `json:"symbols"` +} diff --git a/contrib/gotools/srcs/common/file_process.go b/contrib/gotools/srcs/common/file_process.go new file mode 100644 index 00000000..72b237f2 --- /dev/null +++ b/contrib/gotools/srcs/common/file_process.go @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +import ( + "bufio" + "io" + "io/ioutil" + "os" +) + +// Exported constants for folder management +const ( + SEP = string(os.PathSeparator) + OUTFOLDER = "output" + SEP + PERM = 0755 +) + +// OpenTextFile opens a file named by filename. +// +// It returns a slice of bytes which represents its content and an error if +// any, otherwise it returns nil. +func OpenTextFile(filename string) ([]byte, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + return ioutil.ReadAll(file) +} + +// UpdateFile updates a file named by filename by adding new bytes at the end. +// +// It returns a slice of bytes which represents its content and an error if +// any, otherwise it returns nil. +func UpdateFile(filename string, dataByte []byte) error { + input, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + + result := append(input, dataByte...) + + return WriteToFile(filename, result) +} + +// WriteToFile creates and writes bytes content to a new file named by filename. +// +// It returns an error if any, otherwise it returns nil. +func WriteToFile(filename string, dataByte []byte) error { + err := ioutil.WriteFile(filename, dataByte, PERM) + return err +} + +// OSReadDir reads the content of a folder named by root. +// +// It returns a slice of FileInfo values and an error if any, otherwise it +// returns nil. +func OSReadDir(root string) ([]os.FileInfo, error) { + var files []os.FileInfo + f, err := os.Open(root) + if err != nil { + return files, err + } + fileInfo, err := f.Readdir(-1) + f.Close() + if err != nil { + return files, err + } + + for _, file := range fileInfo { + files = append(files, file) + } + return files, nil +} + +// Exists checks if a given file exists. +// +// It returns true if the file exists and an error if any, otherwise it +// returns nil. +func Exists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + + if os.IsNotExist(err) { + return false, nil + } + + return true, err +} + +// CreateFolder creates a folder if it does not exist. +// +// It returns true if the folder is created and an error if any, otherwise it +// returns nil. +func CreateFolder(path string) (bool, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + if err = os.Mkdir(path, PERM); err != nil { + return false, err + } + return true, nil + } + return false, nil +} + +// ReadLinesFile Reads a file line by line and saves its content into a slice. +// +// It returns a slice of string which represents each line of a file and an +// error if any, otherwise it returns nil. +func ReadLinesFile(path string) ([]string, error) { + + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + rd := bufio.NewReader(f) + + var lines []string + for { + line, err := rd.ReadString('\n') + + if len(line) == 0 { + break + } + + // End of file, break the reading + if err == io.EOF { + lines = append(lines, line) + break + } + + if err != nil { + return nil, err + } + lines = append(lines, line) + } + + return lines, err +} + +// CopyFileContents copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. +// +// It returns an error if any, otherwise it returns nil. +func CopyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cErr := out.Close() + if err == nil { + err = cErr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} diff --git a/contrib/gotools/srcs/common/git.go b/contrib/gotools/srcs/common/git.go new file mode 100644 index 00000000..f67e1cb4 --- /dev/null +++ b/contrib/gotools/srcs/common/git.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +import ( + "regexp" + "strings" +) + +const branch = "staging" + +// GitCloneRepository clones a git repository at the the given url. +// +// It returns two pointers of string which are respectively stdout and stderr +// and an error if any, otherwise it returns nil. +func GitCloneRepository(url, dir string, v bool) (*string, *string, error) { + return ExecuteRunCmd("git", dir, v, "clone", url) +} + +// GitBranchStaging updates the current branch of a git repository to the +// 'staging' branch. +// +// It returns two pointers of string which are respectively stdout and stderr +// and an error if any, otherwise it returns nil. +func GitBranchStaging(dir string, v bool) (*string, *string, error) { + strOut, strErr, err := ExecuteRunCmd("git", dir, v, "branch", "-r") + if err != nil { + return strOut, strErr, err + } + + if strings.Contains(*strOut, branch) || strings.Contains(*strErr, branch) { + PrintInfo("Checkout to " + branch) + return ExecuteRunCmd("git", dir, v, "checkout", branch) + } + + return strOut, strErr, err +} + +// GitPull pulls the current git repository. +// +// It returns two pointers of string which are respectively stdout and stderr +// and an error if any, otherwise it returns nil. +func GitPull(dir string, v bool) (*string, *string, error) { + return ExecuteRunCmd("git", dir, v, "pull") +} + +// GitFindExternalLibs finds all the external libraries of Unikraft which are +// hosted on Xenbits. +// +// It returns a map of all the external libs of Unikraft. +func GitFindExternalLibs(output string) map[string]string { + var re = regexp.MustCompile( + `(?m).*`) + + matches := re.FindAllStringSubmatch(output, -1) + externalLibs := make(map[string]string, len(matches)) + for _, match := range matches { + git := strings.Split(match[1], "/") + lib := strings.Split(git[len(git)-1], ".git") + externalLibs[lib[0]] = git[len(git)-1] + } + return externalLibs +} diff --git a/contrib/gotools/srcs/common/graph_process.go b/contrib/gotools/srcs/common/graph_process.go new file mode 100644 index 00000000..d28324f4 --- /dev/null +++ b/contrib/gotools/srcs/common/graph_process.go @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +import ( + "math/rand" + "os" + + "github.com/awalterschulze/gographviz" +) + +const letters = "0123456789ABCDEF" + +// RandStringBytes generates random string of size n. +// +// It returns a random string of a particular length. +func RandStringBytes(n int) string { + + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} + +// ColorGenerator generates a color in RGB format. +// +// It returns a string which represents a random string formatted as RGB color. +func ColorGenerator() string { + return "#" + RandStringBytes(6) +} + +// CreateGraphLabel creates a graph from a map. +// +// It returns a graph which represents all the direct and no-direct dependencies +// of a given application and an error if any, otherwise it returns nil. +func CreateGraphLabel(name string, data map[string][]string, + mapLabel map[string]string) (*gographviz.Escape, error) { + + graph := gographviz.NewEscape() + + if err := graph.SetName(name); err != nil { + return nil, err + } + + // Directed graph + if err := graph.SetDir(true); err != nil { + return nil, err + } + + // Create graph from map + for key, values := range data { + + colorsMap := map[string]string{} + + // Generate a random color + if _, in := colorsMap[key]; !in { + colorsMap[key] = ColorGenerator() + } + + attributes := map[string]string{"color": colorsMap[key]} + + // Create nodes + if err := graph.AddNode("\""+key+"\"", "\""+key+"\"", + attributes); err != nil { + return nil, err + } + + if values != nil { + + // Add edges + for _, v := range values { + + if label, ok := mapLabel[v]; ok { + attributes["label"] = label + } + + if err := graph.AddEdge("\""+key+"\"", "\""+v+"\"", true, + attributes); err != nil { + return nil, err + } + + // Delete label attributes if necessary + if _, ok := mapLabel[v]; ok { + delete(attributes, "label") + } + } + } + } + + return graph, nil +} + +// SaveGraphToFile saves a given graph to a file. +// +// It returns an error if any, otherwise it returns nil. +func SaveGraphToFile(filename string, graph *gographviz.Escape) error { + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + _, err = file.WriteString(graph.String()) + if err != nil { + return err + } + + return nil +} + +// GenerateGraph generates a given graph to a '.dot' and '.png' files. +// +// It returns an error if any, otherwise it returns nil. +func GenerateGraph(programName, fullPathName string, data map[string][]string, + mapLabel map[string]string) { + // Create graph + graph, err := CreateGraphLabel(programName, data, mapLabel) + + // Save graph as '.dot' file + if err = SaveGraphToFile(fullPathName+".dot", graph); err != nil { + PrintWarning(err) + } + + // Save graph as '.png' file + if _, err := ExecuteCommand("dot", []string{"-Tpng", + fullPathName + ".dot", "-o", fullPathName + ".png"}); err != nil { + PrintWarning(err) + PrintWarning("Open the following website to display the graph:" + + " https://dreampuf.github.io/GraphvizOnline/") + } else { + PrintOk("Graph saved into " + fullPathName + ".png") + } +} diff --git a/contrib/gotools/srcs/common/output_colors.go b/contrib/gotools/srcs/common/output_colors.go new file mode 100644 index 00000000..26e4340e --- /dev/null +++ b/contrib/gotools/srcs/common/output_colors.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +import ( + "fmt" + "log" + + "github.com/fatih/color" +) + +// PrintHeader1 prints a big header formatted string on stdout. +func PrintHeader1(v ...interface{}) { + header := color.New(color.FgBlue, color.Bold, color.Underline).SprintFunc() + fmt.Printf("%v\n", header(v)) +} + +// PrintHeader2 prints a small header formatted string on stdout. +func PrintHeader2(v ...interface{}) { + magenta := color.New(color.FgMagenta).SprintFunc() + fmt.Printf("%v\n", magenta(v)) +} + +// PrintWarning prints a warning formatted string on stdout. +func PrintWarning(v ...interface{}) { + yellow := color.New(color.FgYellow, color.Bold).SprintFunc() + fmt.Printf("[%s] %v\n", yellow("WARNING"), v) +} + +// PrintOk prints a success formatted string on stdout. +func PrintOk(v ...interface{}) { + green := color.New(color.FgGreen, color.Bold).SprintFunc() + fmt.Printf("[%s] %v\n", green("SUCCESS"), v) +} + +// PrintInfo prints an info formatted string on stdout. +func PrintInfo(v ...interface{}) { + blue := color.New(color.FgBlue, color.Bold).SprintFunc() + fmt.Printf("[%s] %v\n", blue("INFO"), v) +} + +// PrintErr prints an error formatted string on stdout and exits the app. +func PrintErr(v ...interface{}) { + red := color.New(color.FgRed).SprintFunc() + log.Fatalf("[%s] %v\n", red("ERROR"), v) +} diff --git a/contrib/gotools/srcs/common/process.go b/contrib/gotools/srcs/common/process.go new file mode 100644 index 00000000..33e85976 --- /dev/null +++ b/contrib/gotools/srcs/common/process.go @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +package common + +import ( + "bytes" + "context" + "errors" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" + "time" + "unicode" +) + +const TIMEOUT = 5 //5 secs + +// ExecutePipeCommand executes a piped command. +// +// It returns a string which represents stdout and an error if any, otherwise +// it returns nil. +func ExecutePipeCommand(command string) (string, error) { + + ctx, cancel := context.WithTimeout(context.Background(), TIMEOUT*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "/bin/bash", "-c", command) + out, err := cmd.Output() + if err != nil { + return "", err + } + + if ctx.Err() == context.DeadlineExceeded { + return string(out), errors.New("Time out during with: " + command) + } + + return string(out), nil +} + +// ExecuteRunCmd runs a command and display the output to stdout and stderr. +// +// It returns two pointers of string which are respectively stdout and stderr +// and an error if any, otherwise it returns nil. +func ExecuteRunCmd(name, dir string, v bool, args ...string) (*string, *string, + error) { + + cmd := exec.Command(name, args...) + cmd.Dir = dir + bufOut, bufErr := &bytes.Buffer{}, &bytes.Buffer{} + if v { + cmd.Stderr = io.MultiWriter(bufErr, os.Stderr) + cmd.Stdout = io.MultiWriter(bufOut, os.Stdout) + } else { + cmd.Stderr = bufErr + cmd.Stdout = bufOut + } + cmd.Stdin = os.Stdin + _ = cmd.Run() + + strOut, strErr := bufOut.String(), bufErr.String() + + return &strOut, &strErr, nil +} + +// ExecuteRunCmdStdin runs a command and saves stdout and stderr as bytes. +// +// It returns two byte arrays which are respectively stdout and stderr +// and an error if any, otherwise it returns nil. +func ExecuteRunCmdStdin(name string, stdinArgs []byte, args ...string) ([]byte, + []byte) { + + bufOut, bufErr := &bytes.Buffer{}, &bytes.Buffer{} + + var buffer bytes.Buffer + if len(stdinArgs) > 0 { + buffer = bytes.Buffer{} + buffer.Write(stdinArgs) + } + + ctx, cancel := context.WithTimeout(context.Background(), TIMEOUT*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, name, args...) + if len(stdinArgs) > 0 { + cmd.Stdin = &buffer + } + cmd.Stdout = bufOut + cmd.Stderr = bufErr + + _ = cmd.Run() + + if ctx.Err() == context.DeadlineExceeded { + PrintWarning("Time out during executing: " + cmd.String()) + return bufOut.Bytes(), bufErr.Bytes() + } + + return bufOut.Bytes(), bufErr.Bytes() +} + +// ExecuteCommand a single command without displaying the output. +// +// It returns a string which represents stdout and an error if any, otherwise +// it returns nil. +func ExecuteCommand(command string, arguments []string) (string, error) { + out, err := exec.Command(command, arguments...).CombinedOutput() + if err != nil { + return "", err + } + return string(out), nil +} + +// ExecuteWaitCommand runs command and waits to its termination without +// displaying the output. +// +// It returns a string which represents stdout and an error if any, otherwise +// it returns nil. +func ExecuteWaitCommand(dir, command string, args ...string) (*string, *string, + error) { + + cmd := exec.Command(command, args...) + cmd.Dir = dir + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + bufOut, bufErr := &bytes.Buffer{}, &bytes.Buffer{} + cmd.Stdout = io.MultiWriter(bufOut) // Add os.Stdin to record on stdout + cmd.Stderr = io.MultiWriter(bufErr) // Add os.Stdin to record on stderr + cmd.Stdin = os.Stdin + + if err := cmd.Start(); err != nil { + return nil, nil, err + } + + PrintInfo("Waiting command: " + command + " " + strings.Join(args, " ")) + + // Ignore error + _ = cmd.Wait() + + strOut, strErr := bufOut.String(), bufErr.String() + + return &strOut, &strErr, nil +} + +// PKill kills a given running process with a particular signal +// +// It returns an error if any, otherwise it returns nil. +func PKill(programName string, sig syscall.Signal) error { + if len(programName) == 0 { + return errors.New("program name should not be empty") + } + re, err := regexp.Compile(programName) + if err != nil { + return err + } + + pids := getPids(re) + if len(pids) == 0 { + return nil + } + + current := os.Getpid() + for _, pid := range pids { + if current != pid { + _ = syscall.Kill(pid, sig) + } + } + + return nil +} + +// PidOf gets PIDs of a particular process. +// +// It returns a list of integer which represents the pids of particular process +// and an error if any, otherwise it returns nil. +func PidOf(name string) ([]int, error) { + if len(name) == 0 { + return []int{}, errors.New("name should not be empty") + } + re, err := regexp.Compile("(^|/)" + name + "$") + if err != nil { + return []int{}, err + } + return getPids(re), nil +} + +// getPids gets PIDs of a particular process. +// +// It returns a list of integer which represents the pids of particular process. +func getPids(re *regexp.Regexp) []int { + var pids []int + + dirFD, err := os.Open("/proc") + if err != nil { + return nil + } + defer dirFD.Close() + + for { + // Read a small number at a time in case there are many entries, we don't want to + // allocate a lot here. + ls, err := dirFD.Readdir(10) + if err == io.EOF { + break + } + if err != nil { + return nil + } + + for _, entry := range ls { + if !entry.IsDir() { + continue + } + + // If the directory is not a number (i.e. not a PID), skip it + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + + cmdline, err := ioutil.ReadFile(filepath.Join("/proc", entry.Name(), "cmdline")) + if err != nil { + println("Error reading file %s: %+v", filepath.Join("/proc", + entry.Name(), "cmdline"), err) + continue + } + + // The bytes we read have '\0' as a separator for the command line + parts := bytes.SplitN(cmdline, []byte{0}, 2) + if len(parts) == 0 { + continue + } + // Split the command line itself we are interested in just the first part + exe := strings.FieldsFunc(string(parts[0]), func(c rune) bool { + return unicode.IsSpace(c) || c == ':' + }) + if len(exe) == 0 { + continue + } + // Check if the name of the executable is what we are looking for + if re.MatchString(exe[0]) { + // Grab the PID from the directory path + pids = append(pids, pid) + } + } + } + + return pids +} diff --git a/contrib/gotools/srcs/common/ukconfig.go b/contrib/gotools/srcs/common/ukconfig.go new file mode 100644 index 00000000..ac03551f --- /dev/null +++ b/contrib/gotools/srcs/common/ukconfig.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +import ( + "regexp" + "strings" +) + +const CONFIGUK = "Config.uk" +const MENUCONFIG = "menuconfig" +const CONFIG = "config" +const SELECT = "select" + +// processConfigUK processes each line of a "Config.uk" file. +// +// // +func ProcessConfigUK(lines []string, fullSelect bool, + mapConfig map[string][]string, mapLabel map[string]string) { + + var libName string + var otherConfig = false + + for i, line := range lines { + parseConfigUK(i, line, &libName, fullSelect, &otherConfig, + mapConfig, mapLabel) + } +} + +// parseConfigUK parses each line of to detect selected libraries (dependencies) +// +// // +func parseConfigUK(index int, line string, libName *string, fullSelect bool, + otherConfig *bool, mapConfig map[string][]string, mapLabel map[string]string) { + + space := regexp.MustCompile(`\s+|\t+`) + line = space.ReplaceAllString(line, " ") + line = strings.TrimSpace(line) + + switch { + case strings.Contains(line, MENUCONFIG), + strings.Contains(line, CONFIG) && index == 0: + { + // First case: get the name of the lib + + // Split the line to retrieve the name of the lib + split := strings.Split(line, " ") + if len(split) < 2 { + break + } + + *libName = strings.TrimSuffix(split[1], "\n") + } + case strings.Contains(line, CONFIG) && index > 0: + { + // Second case: check if other Config lines + *otherConfig = true + } + case strings.Contains(line, SELECT) && index > 0: + { + // Third case: add select libs + + // if there are other Config flag, check the dependencies if + // specified (fullDep), otherwise break + if !*otherConfig && !fullSelect { + break + } + + // Split the line to retrieve the name of the dependency + split := strings.Split(line, " ") + var library string + if len(split) < 2 { + break + } else if len(split) > 2 { + // If we have complex select (e.g., select LIBUKTIME if + // !HAVE_LIBC && ARCH_X86_64) + var re = regexp.MustCompile(`(?m)select\s(\w*)\sif\s([a-zA-Z0-9!_& ]*)`) + match := re.FindAllStringSubmatch(line, -1) + if len(match) > 0 { + library = match[0][1] + if mapLabel != nil { + mapLabel[library] = match[0][2] + } + } else { + break + } + } else { + library = split[1] + } + + // Current selected library + selectedLib := strings.TrimSuffix(library, "\n") + + // Links between current lib and its dependencies + mapConfig[*libName] = append(mapConfig[*libName], selectedLib) + + // Add selected lib in the map in order to generate a node + // if it does not exist + if _, ok := mapConfig[selectedLib]; !ok { + mapConfig[selectedLib] = nil + } + } + } +} diff --git a/contrib/gotools/srcs/common/unikraft_path.go b/contrib/gotools/srcs/common/unikraft_path.go new file mode 100644 index 00000000..46445a0c --- /dev/null +++ b/contrib/gotools/srcs/common/unikraft_path.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +// Exported constants for folder management +const ( + APPSFOLDER = "apps" + SEP + UNIKRAFTFOLDER = "unikraft" + SEP + BUILDFOLDER = "build" + SEP + LIBSFOLDER = "libs" + SEP + INCLUDEFOLDER = "include" + SEP + + KVM_IMAGE = "_kvm-x86_64" +) diff --git a/contrib/gotools/srcs/common/utils.go b/contrib/gotools/srcs/common/utils.go new file mode 100644 index 00000000..6f7b6658 --- /dev/null +++ b/contrib/gotools/srcs/common/utils.go @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package common + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" +) + +// Contains checks if a given slice contains a particular string. +// +// It returns true if the given contains the searched string. +func Contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +// DownloadFile downloads a file from an URL and reads its content. +// +// It returns a pointer to a string that represents the file content and an +// error if any, otherwise it returns nil. +func DownloadFile(url string) (*string, error) { + + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var bodyString string + if resp.StatusCode == http.StatusOK { + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + bodyString = string(bodyBytes) + } + return &bodyString, err +} + +// GetProgramPath returns the absolute path of a given program. +// +// It returns a string that represents the absolute path of a program and an +// error if any, otherwise it returns nil. +func GetProgramPath(programName *string) (string, error) { + var programPath string + println(*programName) + + if ok, err := Exists(*programName); err != nil { + return programPath, err + } else if ok { + // Program (binary) is installed + if filepath.IsAbs(*programName) { + programPath = *programName + *programName = filepath.Base(programPath) + } else if programPath, err = filepath.Abs(*programName); err != nil { + return programPath, err + } + } else { + // Run 'which' command to determine if program has a symbolic name + out, err := ExecuteCommand("which", []string{*programName}) + if err != nil { + return programPath, err + } else { + // Check if out is a valid path + if _, err := os.Stat(out); err == nil { + PrintWarning("Unknown Program -> will skip gathering" + + " symbols, system calls and shared libs process") + } else { + programPath = strings.TrimSpace(out) + } + } + } + return programPath, nil +} + +// ---------------------------------Record Data--------------------------------- + +// RecordDataTxt saves data into a text file named by filename. +// +// It returns an error if any, otherwise it returns nil. +func RecordDataTxt(filename string, headers []string, data interface{}) error { + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + v := reflect.ValueOf(data) + values := make([]interface{}, v.NumField()) + + for i := 0; i < v.NumField(); i++ { + values[i] = v.Field(i).Interface() + if err := WriteMapToFile(file, headers[i], values[i]); err != nil { + return err + } + } + + return nil +} + +// WriteMapToFile saves map into a text file named by filename. +// +// It returns an error if any, otherwise it returns nil. +func WriteMapToFile(file *os.File, headerName string, in interface{}) error { + header := "----------------------------------------------\n" + + headerName + "\n----------------------------------------------\n" + + if _, err := file.WriteString(header); err != nil { + return err + } + + switch v := in.(type) { + case map[string]string: + for key, value := range v { + + var str string + if len(value) > 0 { + str = key + "@" + value + } else { + str = key + } + + if _, err := file.WriteString(str + "\n"); err != nil { + return err + } + } + case map[string][]string: + for key, values := range v { + + var str string + if len(values) > 0 { + str = key + "->" + strings.Join(values, ",") + } else { + str = key + } + + if _, err := file.WriteString(str + "\n"); err != nil { + return err + } + } + } + + return nil +} + +// -------------------------------------JSON------------------------------------ + +// RecordDataJson saves json into a json file named by filename. +// +// It returns an error if any, otherwise it returns nil. +func RecordDataJson(filename string, data *Data) error { + + b, err := json.Marshal(data) + if err != nil { + return err + } + + if err = ioutil.WriteFile(filename+".json", b, os.ModePerm); err != nil { + return err + } + + return nil +} + +// ReadDataJson load json from a json file named by filename. +// +// It returns a Data structure initialized and an error if any, otherwise it +// returns nil. +func ReadDataJson(filename string, data *Data) (*Data, error) { + + jsonFile, err := os.Open(filename + ".json") + if err != nil { + return nil, err + } + defer jsonFile.Close() + + byteValue, err := ioutil.ReadAll(jsonFile) + if err != nil { + return nil, err + } + + if err = json.Unmarshal(byteValue, &data); err != nil { + return nil, err + } + + return data, nil +} diff --git a/contrib/gotools/srcs/crawlertool/args.go b/contrib/gotools/srcs/crawlertool/args.go new file mode 100644 index 00000000..411d6b1e --- /dev/null +++ b/contrib/gotools/srcs/crawlertool/args.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package crawlertool + +import ( + "os" + u "tools/srcs/common" + + "github.com/akamensky/argparse" +) + +const ( + fullLibsArg = "full" + outputArg = "output" + libsArg = "libraries" + repoArg = "repository" +) + +// ParseArguments parses arguments of the application. +// +// It returns an error if any, otherwise it returns nil. +func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { + + args.InitArgParse(p, args, u.BOOL, "f", fullLibsArg, + &argparse.Options{Required: false, Default: false, + Help: "Take all the selected libraries"}) + + args.InitArgParse(p, args, u.STRING, "o", outputArg, + &argparse.Options{Required: true, Help: "Output folder that will " + + "contain dependencies graph and images"}) + args.InitArgParse(p, args, u.STRING, "l", libsArg, + &argparse.Options{Required: false, Help: "Path of the file that " + + "contains libs"}) + args.InitArgParse(p, args, u.STRING, "r", repoArg, + &argparse.Options{Required: false, Help: "Path of the repository"}) + + return u.ParserWrapper(p, os.Args) +} diff --git a/contrib/gotools/srcs/crawlertool/run_crawlertool.go b/contrib/gotools/srcs/crawlertool/run_crawlertool.go new file mode 100644 index 00000000..0907e424 --- /dev/null +++ b/contrib/gotools/srcs/crawlertool/run_crawlertool.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package crawlertool + +import ( + "os" + "path/filepath" + "strings" + "time" + u "tools/srcs/common" +) + +// RunCrawler allows to run the crawler analyser tool (which is out of the +// UNICORE toolchain). +func RunCrawler() { + + mapLabel := make(map[string]string) + mapConfig := make(map[string][]string) + + // Init and parse local arguments + args := new(u.Arguments) + p, err := args.InitArguments() + if err != nil { + u.PrintErr(err) + } + if err := parseLocalArguments(p, args); err != nil { + u.PrintErr(err) + } + + // Used to select all libraries (even those below another Config fields) + fullSelect := *args.BoolArg[fullLibsArg] + + var path string + if len(*args.StringArg[repoArg]) > 0 { + // Only one folder + path = *args.StringArg[repoArg] + u.PrintInfo("Parse folder: " + path) + if err := searchConfigUK(path, fullSelect, mapConfig, mapLabel); err != nil { + u.PrintErr() + } + + } else if len(*args.StringArg[libsArg]) > 0 { + + // Several folders within a list + lines, err := u.ReadLinesFile(*args.StringArg[libsArg]) + if err != nil { + u.PrintErr(err) + } + + // Process Config.uk of each process + for _, line := range lines { + path = strings.TrimSuffix(line, "\n") + u.PrintInfo("Parse folder: " + path) + if err := searchConfigUK(path, fullSelect, mapConfig, mapLabel); err != nil { + u.PrintErr(err) + } + } + } else { + u.PrintErr("You must specify either -r (--repository) or -l (libs)") + } + + // Generate the out folder + outFolder := *args.StringArg[outputArg] + if outFolder[len(outFolder)-1:] != string(os.PathSeparator) { + outFolder += string(os.PathSeparator) + } + + outputPath := outFolder + + "output_" + time.Now().Format("20060102150405") + + // Create the dependencies graph + u.GenerateGraph("Unikraft Crawler", outputPath, mapConfig, + mapLabel) + + u.PrintOk(".dot file is saved: " + outputPath) +} + +// searchConfigUK performs a look-up to find "Config.uk" files. +// +// It returns an error if any, otherwise it returns nil. +func searchConfigUK(path string, fullSelect bool, + mapConfig map[string][]string, mapLabel map[string]string) error { + + err := filepath.Walk(path, func(path string, info os.FileInfo, + err error) error { + if err != nil { + return err + } + + // Consider only CONFIGUK files + if !info.IsDir() && info.Name() == u.CONFIGUK { + lines, err := u.ReadLinesFile(path) + if err != nil { + return err + } + u.ProcessConfigUK(lines, fullSelect, mapConfig, mapLabel) + } + + return nil + }) + + if err != nil { + return err + } + + return nil +} diff --git a/contrib/gotools/srcs/dependtool/args.go b/contrib/gotools/srcs/dependtool/args.go new file mode 100644 index 00000000..89a5e9c3 --- /dev/null +++ b/contrib/gotools/srcs/dependtool/args.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package dependtool + +import ( + "os" + u "tools/srcs/common" + + "github.com/akamensky/argparse" +) + +const ( + programArg = "program" + testFileArg = "testFile" + configFileArg = "configFile" + optionsArg = "options" + waitTimeArg = "waitTime" + saveOutputArg = "saveOutput" + fullDepsArg = "fullDeps" +) + +// parseLocalArguments parses arguments of the application. +func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { + + args.InitArgParse(p, args, u.STRING, "p", programArg, + &argparse.Options{Required: true, Help: "Program name"}) + args.InitArgParse(p, args, u.STRING, "t", testFileArg, + &argparse.Options{Required: false, Help: "Path of the test file"}) + args.InitArgParse(p, args, u.STRING, "c", configFileArg, + &argparse.Options{Required: false, Help: "Path of the config file"}) + args.InitArgParse(p, args, u.STRING, "o", optionsArg, + &argparse.Options{Required: false, Default: "", Help: "Extra options for " + + "launching program"}) + + args.InitArgParse(p, args, u.INT, "w", waitTimeArg, + &argparse.Options{Required: false, Default: 60, Help: "Time wait (" + + "sec) for external tests (default: 60 sec)"}) + + args.InitArgParse(p, args, u.BOOL, "", saveOutputArg, + &argparse.Options{Required: false, Default: false, + Help: "Save results as TXT file and graphs as PNG file"}) + args.InitArgParse(p, args, u.BOOL, "", fullDepsArg, + &argparse.Options{Required: false, Default: false, + Help: "Show dependencies of dependencies"}) + + return u.ParserWrapper(p, os.Args) +} diff --git a/contrib/gotools/srcs/dependtool/dynamic_analyser.go b/contrib/gotools/srcs/dependtool/dynamic_analyser.go new file mode 100644 index 00000000..ac030f16 --- /dev/null +++ b/contrib/gotools/srcs/dependtool/dynamic_analyser.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package dependtool + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "strconv" + "strings" + "syscall" + u "tools/srcs/common" +) + +// Exported struct that represents the arguments for dynamic analysis. +type DynamicArgs struct { + waitTime int + fullDeps, saveOutput bool + testFile string + options []string +} + +const ( + systrace = "strace" + libtrace = "ltrace" +) + +// ---------------------------------Read Json----------------------------------- + +// readTestFileJson load Testing json from a json file named by filename. +// +// It returns a Testing structure initialized and an error if any, otherwise it +// returns nil. +func readTestFileJson(filename string) (*Testing, error) { + + testingStruct := &Testing{} + jsonFile, err := os.Open(filename) + if err != nil { + return nil, err + } + defer jsonFile.Close() + + byteValue, err := ioutil.ReadAll(jsonFile) + if err != nil { + return nil, err + } + + if err = json.Unmarshal(byteValue, &testingStruct); err != nil { + return nil, err + } + + if testingStruct.ListCommands == nil || len(testingStruct.ListCommands) == 0 { + return nil, errors.New("ListCommands cannot be empty") + } + + return testingStruct, nil +} + +// --------------------------------Gather Data---------------------------------- + +// gatherDataAux gathers symbols and system calls of a given application (helper +// function. +// +// It returns true if a command must be run with sudo. +func gatherDataAux(command, programPath, programName, option string, + data *u.DynamicData, dArgs DynamicArgs) bool { + + testingStruct := &Testing{} + if len(dArgs.testFile) > 0 { + var err error + testingStruct, err = readTestFileJson(dArgs.testFile) + if err != nil { + u.PrintWarning("Cannot find test file: " + err.Error()) + } + } + _, errStr := runCommandTester(programPath, programName, command, option, + testingStruct, dArgs, data) + + ret := false + if command == systrace { + ret = parseTrace(errStr, data.SystemCalls) + } else { + ret = parseTrace(errStr, data.Symbols) + } + return ret +} + +// gatherData gathers symbols and system calls of a given application. +// +func gatherData(command, programPath, programName string, + data *u.DynamicData, dArgs DynamicArgs) { + + if len(dArgs.options) > 0 { + // Iterate through configs present in config file + for _, option := range dArgs.options { + // Check if program name is used in config file + if strings.Contains(option, programName) { + // If yes, take only arguments + split := strings.Split(option, programName) + option = strings.TrimSuffix(strings.Replace(split[1], + " ", "", -1), "\n") + } + + u.PrintInfo("Run " + programName + " with option: '" + + option + "'") + if requireSudo := gatherDataAux(command, programPath, programName, + option, data, dArgs); requireSudo { + u.PrintErr(programName + " requires superuser " + + "privileges: Run command with sudo") + } + } + } else { + // Run without option/config + if requireSudo := gatherDataAux(command, programPath, programName, + "", data, dArgs); requireSudo { + u.PrintErr(programName + " requires superuser " + + "privileges: Run command with sudo") + } + } +} + +// gatherDynamicSharedLibs gathers shared libraries of a given application. +// +// It returns an error if any, otherwise it returns nil. +func gatherDynamicSharedLibs(programName string, pid int, data *u.DynamicData, + fullDeps bool) error { + + // Get the pid + pidStr := strconv.Itoa(pid) + u.PrintInfo("PID '" + programName + "' : " + pidStr) + + // Use 'lsof' to get open files and thus .so files + if output, err := u.ExecutePipeCommand( + "lsof -p " + pidStr + " | uniq | awk '{print $9}'"); err != nil { + return err + } else { + // Parse 'lsof' output + if err := parseLsof(output, data, fullDeps); err != nil { + u.PrintErr(err) + } + } + + // Use 'cat /proc/pid' to get open files and thus .so files + if output, err := u.ExecutePipeCommand( + "cat /proc/" + pidStr + "/maps | awk '{print $6}' | " + + "grep '\\.so' | sort | uniq"); err != nil { + return err + } else { + // Parse 'cat' output + if err := parseLsof(output, data, fullDeps); err != nil { + u.PrintErr(err) + } + } + + return nil +} + +// ------------------------------------ARGS------------------------------------- + +// getDArgs returns a DynamicArgs struct which contains arguments specific to +// the dynamic dependency analysis. +// +// It returns a DynamicArgs struct. +func getDArgs(args *u.Arguments, options []string) DynamicArgs { + return DynamicArgs{*args.IntArg[waitTimeArg], + *args.BoolArg[fullDepsArg], *args.BoolArg[saveOutputArg], + *args.StringArg[testFileArg], options} +} + +// -------------------------------------RUN------------------------------------- + +// RunDynamicAnalyser runs the dynamic analysis to get shared libraries, +// system calls and library calls of a given application. +// +func dynamicAnalyser(args *u.Arguments, data *u.Data, programPath string) { + + // Check options + var configs []string + if len(*args.StringArg[configFileArg]) > 0 { + // Multi-lines options (config) + var err error + configs, err = u.ReadLinesFile(*args.StringArg[configFileArg]) + if err != nil { + u.PrintWarning(err) + } + } else if len(*args.StringArg[optionsArg]) > 0 { + // Single option (config) + configs = append(configs, *args.StringArg[optionsArg]) + } + + // Get dynamic structure + dArgs := getDArgs(args, configs) + programName := *args.StringArg[programArg] + + // Kill process if it is already launched + u.PrintInfo("Kill '" + programName + "' if it is already launched") + if err := u.PKill(programName, syscall.SIGINT); err != nil { + u.PrintErr(err) + } + + // Init dynamic data + dynamicData := &data.DynamicData + dynamicData.SharedLibs = make(map[string][]string) + dynamicData.SystemCalls = make(map[string]string) + dynamicData.Symbols = make(map[string]string) + + // Run strace + u.PrintHeader2("(*) Gathering system calls from ELF file") + gatherData(systrace, programPath, programName, dynamicData, dArgs) + + // Kill process if it is already launched + u.PrintInfo("Kill '" + programName + "' if it is already launched") + if err := u.PKill(programName, syscall.SIGINT); err != nil { + u.PrintErr(err) + } + + // Run ltrace + u.PrintHeader2("(*) Gathering symbols from ELF file") + gatherData(libtrace, programPath, programName, dynamicData, dArgs) +} diff --git a/contrib/gotools/srcs/dependtool/elf_file.go b/contrib/gotools/srcs/dependtool/elf_file.go new file mode 100644 index 00000000..b7cfb56b --- /dev/null +++ b/contrib/gotools/srcs/dependtool/elf_file.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package dependtool + +import ( + "debug/elf" + "os" +) + +// getElf reads and decodes an ELF file. +// +// It returns a pointer to an ELF file and an error if any, otherwise it +// returns nil. +func getElf(filename string) (*elf.File, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + _elf, err := elf.NewFile(f) + if err != nil { + return nil, err + } + + // Read and decode ELF identifier + var ident [16]uint8 + _, err = f.ReadAt(ident[0:], 0) + if err != nil { + return nil, err + } + + // Check the type + if ident[0] != '\x7f' || ident[1] != 'E' || ident[2] != 'L' || ident[3] != 'F' { + return nil, nil + } + + return _elf, nil +} + +// GetElfArchitecture gets the ELF architecture. +// +// It returns a string that defines the ELF class and a string that defines the +// Machine type. +func GetElfArchitecture(elf *elf.File) (string, string) { + var arch, mach string + + switch elf.Class.String() { + case "ELFCLASS64": + arch = "64 bits" + case "ELFCLASS32": + arch = "32 bits" + } + + switch elf.Machine.String() { + case "EM_AARCH64": + mach = "ARM64" + case "EM_386": + mach = "x86" + case "EM_X86_64": + mach = "x86_64" + } + + return arch, mach +} diff --git a/contrib/gotools/srcs/dependtool/parser.go b/contrib/gotools/srcs/dependtool/parser.go new file mode 100644 index 00000000..988c6d86 --- /dev/null +++ b/contrib/gotools/srcs/dependtool/parser.go @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package dependtool + +import ( + "fmt" + "regexp" + "strings" + + u "tools/srcs/common" +) + +const levelDeps = 5 + +type recursiveData struct { + data, glMap, printDep map[string][]string + cmd, line string + listStr []string + level int +} + +// --------------------------------Static Output-------------------------------- + +// parseReadELF parses the output of the 'readelf' command. +// +func parseReadELF(output string, data *u.StaticData) { + types := map[string]bool{"FUNC": true, "FILE": true, "OBJECT": true} + + // Check the output of 'readelf' command + for _, line := range strings.Split(output, "\n") { + words := strings.Fields(line) + + if len(words) > 8 && types[words[3]] { + symbol := strings.Split(words[7], "@") + data.Symbols[symbol[0]] = symbol[1] + } + } +} + +// parseNM parses the output of the 'nm' command. +// +func parseNM(output string, data *u.StaticData) { + // Get the list of system calls + systemCalls := initSystemCalls() + + // Check the output of 'nm' command + var re = regexp.MustCompile(`(?m)([U|T|B|D]\s)(.*)\s*`) + for _, match := range re.FindAllStringSubmatch(output, -1) { + // Add to system calls map if symbol is a system call + if _, isSyscall := systemCalls[match[2]]; isSyscall { + data.SystemCalls[match[2]] = "" + } else { + data.Symbols[match[2]] = "" + } + } +} + +// parsePackagesName parses the output of the 'apt-cache pkgnames' command. +// +// It returns a string which represents the name of application used by the +// package manager (apt, ...). +func parsePackagesName(output string) string { + + var i = 1 + lines := strings.Split(output, "\n") + for _, line := range lines { + if len(line) > 0 { + fmt.Printf("%d) %s\n", i, line) + i++ + } + } + + var input int + for true { + fmt.Print("Please enter your choice (0 to exit): ") + if _, err := fmt.Scanf("%d", &input); err != nil { + u.PrintWarning("Choice must be numeric! Try again") + } else { + if input == 0 { + u.PrintWarning("Abort dependencies analysis from apt-cache") + return "" + } else if (input >= 0) && (input <= i) { + return lines[input-1] + } else { + u.PrintWarning("Invalid input! Try again") + } + } + } + return "" +} + +// parseDependencies parses the output of the 'apt-cache depends' command. +// +// It returns a slice of strings which represents all the dependencies of +// a particular package. +func parseDependencies(output string, data, dependenciesMap, + printDep map[string][]string, fullDeps bool, level int) []string { + listDep := make([]string, 0) + for _, line := range strings.Split(output, "\n") { + if len(line) > 0 && !strings.Contains(line, "<") { + + if _, in := printDep[line]; !in { + fmt.Println(line) + printDep[line] = nil + } + + if fullDeps && level < levelDeps { + rd := recursiveData{ + data: data, + glMap: dependenciesMap, + printDep: printDep, + cmd: "apt-cache depends " + line + + " | awk '/Depends/ { print $2 }'", + line: line, + listStr: listDep, + level: level, + } + listDep = append(listDep, line) + parseRecursive(rd) + } + } else { + data[line] = nil + } + } + return listDep +} + +// parseLDD parses the output of the 'ldd' command. +// +// It returns a slice of strings which represents all the shared libs of +// a particular package. +func parseLDD(output string, data map[string][]string, lddMap map[string][]string, + fullDeps bool) []string { + + listLdd := make([]string, 0) + for _, line := range strings.Split(output, "\n") { + words := strings.Fields(line) + + if len(words) == 2 { + + lib, path := words[0], words[1] + + // Execute ldd only if fullDeps mode is set + if fullDeps { + rd := recursiveData{ + data: data, + glMap: lddMap, + printDep: nil, + cmd: "ldd " + path + " | awk '/ => / { print $1,$3 }'", + line: lib, + listStr: listLdd, + level: -1, + } + listLdd = append(listLdd, lib) + parseRecursive(rd) + } else { + data[lib] = nil + } + } + } + return listLdd +} + +// parseRecursive is used by parseDependencies and parseLDD to factorize code. +// +func parseRecursive(rD recursiveData) { + + if _, in := rD.glMap[rD.line]; in { + // Use additional map to avoid executing again ldd + rD.data[rD.line] = rD.glMap[rD.line] + } else { + + var libsAcc []string + out, err := u.ExecutePipeCommand(rD.cmd) + if err != nil { + u.PrintErr(err) + } + + if rD.printDep == nil { + libsAcc = parseLDD(out, rD.data, rD.glMap, true) + } else { + libsAcc = parseDependencies(out, rD.data, rD.glMap, rD.printDep, + true, rD.level+1) + } + + // Add return libsAcc to map + rD.data[rD.line] = libsAcc + rD.glMap[rD.line] = libsAcc + } +} + +// ------------------------------Dynamic Output -------------------------------- + +// detectPermissionDenied detects if "Permission denied" substring is +// present within dynamic analysis output. +// +// It returns true if it "Permission denied" is present, otherwise false. +func detectPermissionDenied(str string) bool { + if strings.Contains(str, "EACCESS (Permission denied)") || + strings.Contains(str, "13: Permission denied") { + return true + } + return false +} + +// parseTrace parses the output of the '(s)|(f)trace' command. +// +// It returns true if command must be run with sudo, otherwise false. +func parseTrace(output string, data map[string]string) bool { + + var re = regexp.MustCompile(`([a-zA-Z_0-9@/-]+?)\((.*)`) + for _, match := range re.FindAllStringSubmatch(output, -1) { + if len(match) > 1 { + // Detect if Permission denied is thrown + detected := detectPermissionDenied(match[2]) + if detected { + // Command must be run with sudo + return true + } + // Add symbol to map + data[match[1]] = "" + } + } + return false +} + +// parseLsof parses the output of the 'lsof' command. +// +// It returns an error if any, otherwise it returns nil. +func parseLsof(output string, data *u.DynamicData, fullDeps bool) error { + + lddMap := make(map[string][]string) + for _, line := range strings.Split(output, "\n") { + if strings.Contains(line, ".so") { + words := strings.Split(line, "/") + data.SharedLibs[words[len(words)-1]] = nil + if fullDeps { + // Execute ldd only if fullDeps mode is set + if out, err := u.ExecutePipeCommand("ldd " + line + + " | awk '/ => / { print $1,$3 }'"); err != nil { + return err + } else { + data.SharedLibs[words[len(words)-1]] = + parseLDD(out, data.SharedLibs, lddMap, fullDeps) + } + } + } + } + + return nil +} diff --git a/contrib/gotools/srcs/dependtool/run_deptool.go b/contrib/gotools/srcs/dependtool/run_deptool.go new file mode 100644 index 00000000..ef607e01 --- /dev/null +++ b/contrib/gotools/srcs/dependtool/run_deptool.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package dependtool + +import ( + "fmt" + "runtime" + "strings" + u "tools/srcs/common" + + "github.com/fatih/color" +) + +// RunAnalyserTool allows to run the dependency analyser tool. +func RunAnalyserTool(homeDir string, data *u.Data) { + + // Support only Unix + if strings.ToLower(runtime.GOOS) != "linux" { + u.PrintErr("Only UNIX/Linux platforms are supported") + } + + // Init and parse local arguments + args := new(u.Arguments) + p, err := args.InitArguments() + if err != nil { + u.PrintErr(err) + } + if err := parseLocalArguments(p, args); err != nil { + u.PrintErr(err) + } + + // Get program path + programPath, err := u.GetProgramPath(&*args.StringArg[programArg]) + if err != nil { + u.PrintErr("Could not determine program path", err) + } + + // Get program Name + programName := *args.StringArg[programArg] + + // Create the folder 'output' if it does not exist + outFolder := homeDir + u.SEP + programName + "_" + u.OUTFOLDER + if _, err := u.CreateFolder(outFolder); err != nil { + u.PrintErr(err) + } + + // Display Minor Details + displayProgramDetails(programName, programPath, args) + + // Check if the program is an ELF + checkElf(&programPath) + + // Run static analyser + u.PrintHeader1("(1.1) RUN STATIC ANALYSIS") + runStaticAnalyser(args, programName, programPath, outFolder, data) + + // Run dynamic analyser + u.PrintHeader1("(1.2) RUN DYNAMIC ANALYSIS") + runDynamicAnalyser(args, programName, programPath, outFolder, data) + + // Save Data to JSON + if err = u.RecordDataJson(outFolder+programName, data); err != nil { + u.PrintErr(err) + } else { + u.PrintOk("JSON Data saved into " + outFolder + programName + + ".json") + } + + // Save graph if full dependencies option is set + if *args.BoolArg[fullDepsArg] { + saveGraph(programName, outFolder, data) + } +} + +// displayProgramDetails display various information such path, background, ... +func displayProgramDetails(programName, programPath string, args *u.Arguments) { + fmt.Println("----------------------------------------------") + fmt.Println("Analyze Program: ", color.GreenString(programName)) + fmt.Println("Full Path: ", color.GreenString(programPath)) + if len(*args.StringArg[optionsArg]) > 0 { + fmt.Println("Options: ", color.GreenString(*args.StringArg[optionsArg])) + } + + if len(*args.StringArg[configFileArg]) > 0 { + fmt.Println("Config file: ", color.GreenString(*args.StringArg[configFileArg])) + } + + if len(*args.StringArg[testFileArg]) > 0 { + fmt.Println("Test file: ", color.GreenString(*args.StringArg[testFileArg])) + } + + fmt.Println("----------------------------------------------") +} + +// checkElf checks if the program (from its path) is an ELF file +func checkElf(programPath *string) { + elfFile, err := getElf(*programPath) + if err != nil { + u.PrintErr(err) + } else if elfFile == nil { + *programPath = "" + u.PrintWarning("Only ELF binaries are supported! Some analysis" + + " procedures will be skipped") + } else { + // Get ELF architecture + architecture, machine := GetElfArchitecture(elfFile) + fmt.Println("ELF Class: ", architecture) + fmt.Println("Machine: ", machine) + fmt.Println("Entry Point: ", elfFile.Entry) + fmt.Println("----------------------------------------------") + } +} + +// runStaticAnalyser runs the static analyser +func runStaticAnalyser(args *u.Arguments, programName, programPath, + outFolder string, data *u.Data) { + + staticAnalyser(*args, data, programPath) + + // Save static Data into text file if display mode is set + if *args.BoolArg[saveOutputArg] { + + // Create the folder 'output/static' if it does not exist + outFolderStatic := outFolder + "static" + u.SEP + if _, err := u.CreateFolder(outFolderStatic); err != nil { + u.PrintErr(err) + } + + fn := outFolderStatic + programName + ".txt" + headersStr := []string{"Dependencies (from apt-cache show) list:", + "Shared libraries list:", "System calls list:", "Symbols list:"} + + if err := u.RecordDataTxt(fn, headersStr, data.StaticData); err != nil { + u.PrintWarning(err) + } else { + u.PrintOk("Data saved into " + fn) + } + } +} + +// runDynamicAnalyser runs the dynamic analyser. +func runDynamicAnalyser(args *u.Arguments, programName, programPath, + outFolder string, data *u.Data) { + + dynamicAnalyser(args, data, programPath) + + // Save dynamic Data into text file if display mode is set + if *args.BoolArg[saveOutputArg] { + + // Create the folder 'output/dynamic' if it does not exist + outFolderDynamic := outFolder + "dynamic" + u.SEP + if _, err := u.CreateFolder(outFolderDynamic); err != nil { + u.PrintErr(err) + } + + fn := outFolderDynamic + programName + ".txt" + headersStr := []string{"Shared libraries list:", "System calls list:", + "Symbols list:"} + + if err := u.RecordDataTxt(fn, headersStr, data.DynamicData); err != nil { + u.PrintWarning(err) + } else { + u.PrintOk("Data saved into " + fn) + } + } +} + +// saveGraph saves dependency graphs of a given app into the output folder. +func saveGraph(programName, outFolder string, data *u.Data) { + + if len(data.StaticData.SharedLibs) > 0 { + u.GenerateGraph(programName, outFolder+"static"+u.SEP+ + programName+"_shared_libs", data.StaticData.SharedLibs, nil) + } + + if len(data.StaticData.Dependencies) > 0 { + u.GenerateGraph(programName, outFolder+"static"+u.SEP+ + programName+"_dependencies", data.StaticData.Dependencies, nil) + } + + if len(data.StaticData.SharedLibs) > 0 { + u.GenerateGraph(programName, outFolder+"dynamic"+u.SEP+ + programName+"_shared_libs", data.DynamicData.SharedLibs, nil) + } +} diff --git a/contrib/gotools/srcs/dependtool/static_analyser.go b/contrib/gotools/srcs/dependtool/static_analyser.go new file mode 100644 index 00000000..9f1b730d --- /dev/null +++ b/contrib/gotools/srcs/dependtool/static_analyser.go @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package dependtool + +import ( + "bufio" + "fmt" + "os" + u "tools/srcs/common" +) + +// ---------------------------------Gather Data--------------------------------- + +// gatherStaticSymbols gathers symbols of a given application. +// +// It returns an error if any, otherwise it returns nil. +func gatherStaticSymbols(programPath string, data *u.StaticData) error { + + // Use 'readelf' to get symbols + if output, err := u.ExecuteCommand("readelf", []string{"-s", + programPath}); err != nil { + return err + } else { + // Init symbols members + data.Symbols = make(map[string]string) + parseReadELF(output, data) + } + return nil +} + +// gatherStaticSymbols gathers system calls of a given application. +// +// It returns an error if any, otherwise it returns nil. +func gatherStaticSystemCalls(programPath string, data *u.StaticData) error { + + // Use 'nm' to get symbols and system calls + if output, err := u.ExecuteCommand("nm", []string{"-D", + programPath}); err != nil { + return err + } else { + // Init system calls members + data.SystemCalls = make(map[string]string) + parseNM(output, data) + } + return nil +} + +// gatherStaticSymbols gathers shared libs of a given application. +// +// It returns an error if any, otherwise it returns nil. +func gatherStaticSharedLibs(programPath string, data *u.StaticData, + v bool) error { + + // Use 'ldd' to get shared libraries + if output, err := u.ExecutePipeCommand("ldd " + programPath + + " | awk '/ => / { print $1,$3 }'"); err != nil { + return err + } else { + // Init SharedLibs + data.SharedLibs = make(map[string][]string) + lddGlMap := make(map[string][]string) + _ = parseLDD(output, data.SharedLibs, lddGlMap, v) + } + return nil +} + +// gatherDependencies gathers dependencies of a given application. +// +// It returns an error if any, otherwise it returns nil. +func gatherDependencies(programName string, data *u.StaticData, v bool) error { + + // Use 'apt-cache pkgnames' to get the name of the package + output, err := u.ExecuteCommand("apt-cache", + []string{"pkgnames", programName}) + if err != nil { + return err + } + + // If the name of the package is know, execute apt-cache depends + if len(output) > 0 { + // Parse package name + packageName := parsePackagesName(output) + + if len(packageName) > 0 { + return executeDependAptCache(packageName, data, v) + } + } else { + // Enter manually the name of the package + u.PrintWarning(programName + " not found in apt-cache") + var output string + for len(output) == 0 { + fmt.Print("Please enter manually the name of the package " + + "(empty string to exit): ") + scanner := bufio.NewScanner(os.Stdin) + if err := scanner.Err(); err != nil { + return err + } + + if scanner.Scan() { + + // Get the new package name + input := scanner.Text() + if input == "" { + break + } + + output, err = u.ExecuteCommand("apt-cache", + []string{"pkgnames", input}) + if err != nil { + return err + } + } + } + + if len(output) == 0 { + u.PrintWarning("Skip dependencies analysis from apt-cache depends") + } else { + packageName := parsePackagesName(output) + return executeDependAptCache(packageName, data, v) + } + } + + return nil +} + +// executeDependAptCache gathers dependencies by executing 'apt-cache depends'. +// +// It returns an error if any, otherwise it returns nil. +func executeDependAptCache(programName string, data *u.StaticData, + fullDeps bool) error { + + // Use 'apt-cache depends' to get dependencies + if output, err := u.ExecutePipeCommand("apt-cache depends " + + programName + " | awk '/Depends/ { print $2 }'"); err != nil { + return err + } else { + // Init Dependencies (from apt cache depends) + data.Dependencies = make(map[string][]string) + dependenciesMap := make(map[string][]string) + printDep := make(map[string][]string) + _ = parseDependencies(output, data.Dependencies, dependenciesMap, + printDep, fullDeps, 0) + } + + fmt.Println("----------------------------------------------") + return nil +} + +// -------------------------------------Run------------------------------------- + +// staticAnalyser runs the static analysis to get shared libraries, +// system calls and library calls of a given application. +// +func staticAnalyser(args u.Arguments, data *u.Data, programPath string) { + + programName := *args.StringArg[programArg] + fullDeps := *args.BoolArg[fullDepsArg] + + staticData := &data.StaticData + + // If the program is a binary, runs static analysis tools + if len(programPath) > 0 { + // Gather Data from binary file + u.PrintHeader2("(*) Gathering symbols from ELF file") + if err := gatherStaticSymbols(programPath, staticData); err != nil { + u.PrintWarning(err) + } + + u.PrintHeader2("(*) Gathering symbols & system calls from ELF file") + if err := gatherStaticSystemCalls(programPath, staticData); err != nil { + u.PrintWarning(err) + } + + u.PrintHeader2("(*) Gathering shared libraries from ELF file") + if err := gatherStaticSharedLibs(programPath, staticData, + fullDeps); err != nil { + u.PrintWarning(err) + } + } + + // Gather Data from apt-cache + u.PrintHeader2("(*) Gathering dependencies from apt-cache depends") + if err := gatherDependencies(programName, staticData, fullDeps); err != nil { + u.PrintWarning(err) + } +} diff --git a/contrib/gotools/srcs/dependtool/system_calls.go b/contrib/gotools/srcs/dependtool/system_calls.go new file mode 100644 index 00000000..c23c98e0 --- /dev/null +++ b/contrib/gotools/srcs/dependtool/system_calls.go @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package dependtool + +// InitSystemCalls initialises all Linux system calls. +// +// It returns a map of all system calls. +func initSystemCalls() map[string]*string { + return map[string]*string{"_llseek": nil, "_newselect": nil, "_sysctl": nil, + "accept": nil, "accept4": nil, "access": nil, "acct": nil, + "add_key": nil, "adjtimex": nil, "alarm": nil, "alloc_hugepages": nil, + "arc_gettls": nil, "arc_settls": nil, "arc_usr_cmpxchg": nil, + "arch_prctl": nil, "atomic_barrier": nil, "atomic_cmpxchg_32": nil, + "bdflush": nil, "bfin_spinlock": nil, "bind": nil, "bpf": nil, + "brk": nil, "breakpoint": nil, "cacheflush": nil, "capget": nil, + "capset": nil, "chdir": nil, "chmod": nil, "chown": nil, "chown32": nil, + "chroot": nil, "clock_adjtime": nil, "clock_getres": nil, + "clock_gettime": nil, "clock_nanosleep": nil, "connect": nil, + "copy_file_range": nil, "creat": nil, "create_module": nil, + "delete_module": nil, "dma_memcpy": nil, "dup": nil, "dup2": nil, + "dup3": nil, "epoll_create": nil, "epoll_create1": nil, + "epoll_ctl": nil, "epoll_pwait": nil, "epoll_wait": nil, "eventfd": nil, + "eventfd2": nil, "execv": nil, "execve": nil, "execveat": nil, + "exit": nil, "exit_group": nil, "faccessat": nil, "fadvise64": nil, + "fadvise64_64": nil, "fallocate": nil, "fanotify_init": nil, + "fanotify_mark": nil, "fchdir": nil, "fchmod": nil, + "fchmodat": nil, "fchown": nil, "fchown32": nil, "fchownat": nil, + "fcntl": nil, "fcntl64": nil, "fdatasync": nil, "fgetxattr": nil, + "finit_module": nil, "flistxattr": nil, "flock": nil, "fork": nil, + "free_hugepages": nil, "fremovexattr": nil, "fsetxattr": nil, + "fstat": nil, "fstat64": nil, "fstatat64": nil, "fstatfs": nil, + "fstatfs64": nil, "fsync": nil, "ftruncate": nil, "ftruncate64": nil, + "futex": nil, "futimesat": nil, "get_kernel_syms": nil, + "get_mempolicy": nil, "get_robust_list": nil, "get_thread_area": nil, + "get_tls": nil, "getcpu": nil, "getcwd": nil, "getdents": nil, + "getdents64": nil, "getdomainname": nil, "getdtablesize": nil, + "getegid": nil, "getegid32": nil, "geteuid": nil, "geteuid32": nil, + "getgid": nil, "getgid32": nil, "getgroups": nil, "getgroups32": nil, + "gethostname": nil, "getitimer": nil, "getpeername": nil, + "getpagesize": nil, "getpgid": nil, "getpgrp": nil, "getpid": nil, + "getppid": nil, "getpriority": nil, "getrandom": nil, "getresgid": nil, + "getresgid32": nil, "getresuid": nil, "getresuid32": nil, + "getrlimit": nil, "getrusage": nil, "getsid": nil, "getsockname": nil, + "getsockopt": nil, "gettid": nil, "gettimeofday": nil, "getuid": nil, + "getuid32": nil, "getunwind": nil, "getxattr": nil, "getxgid": nil, + "getxpid": nil, "getxuid": nil, "init_module": nil, + "inotify_add_watch": nil, "inotify_init": nil, "inotify_init1": nil, + "inotify_rm_watch": nil, "io_cancel": nil, "io_destroy": nil, + "io_getevents": nil, "io_pgetevents": nil, "io_setup": nil, + "io_submit": nil, "ioctl": nil, "ioperm": nil, "iopl": nil, + "ioprio_get": nil, "ioprio_set": nil, "ipc": nil, "kcmp": nil, + "kern_features": nil, "kexec_file_load": nil, "kexec_load": nil, + "keyctl": nil, "kill": nil, "lchown": nil, "lchown32": nil, + "lgetxattr": nil, "link": nil, "linkat": nil, "listen": nil, + "listxattr": nil, "llistxattr": nil, "lookup_dcookie": nil, + "lremovexattr": nil, "lseek": nil, "lsetxattr": nil, "lstat": nil, + "lstat64": nil, "madvise": nil, "mbind": nil, "memory_ordering": nil, + "metag_get_tls": nil, "metag_set_fpu_flags": nil, "metag_set_tls": nil, + "metag_setglobalbit": nil, "membarrier": nil, "memfd_create": nil, + "migrate_pages": nil, "mincore": nil, "mkdir": nil, + "mkdirat": nil, "mknod": nil, "mknodat": nil, "mlock": nil, + "mlock2": nil, "mlockall": nil, "mmap": nil, "mmap2": nil, + "modify_ldt": nil, "mount": nil, "move_pages": nil, "mprotect": nil, + "mq_getsetattr": nil, "mq_notify": nil, "mq_open": nil, + "mq_timedreceive": nil, "mq_timedsend": nil, "mq_unlink": nil, + "mremap": nil, "msgctl": nil, "msgget": nil, "msgrcv": nil, + "msgsnd": nil, "msync": nil, "munlock": nil, "munlockall": nil, + "munmap": nil, "name_to_handle_at": nil, "nanosleep": nil, + "newfstatat": nil, "nfsservctl": nil, "nice": nil, "old_adjtimex": nil, + "old_getrlimit": nil, "oldfstat": nil, "oldlstat": nil, + "oldolduname": nil, "oldstat": nil, "oldumount": nil, "olduname": nil, + "open": nil, "open_by_handle_at": nil, "openat": nil, + "or1k_atomic": nil, "pause": nil, "pciconfig_iobase": nil, + "pciconfig_read": nil, "pciconfig_write": nil, "perf_event_open": nil, + "personality": nil, "perfctr": nil, "perfmonctl": nil, "pipe": nil, + "pipe2": nil, "pivot_root": nil, "pkey_alloc": nil, "pkey_free": nil, + "pkey_mprotect": nil, "poll": nil, "ppoll": nil, "prctl": nil, + "pread": nil, "pread64": nil, "preadv": nil, "preadv2": nil, + "prlimit64": nil, "process_vm_readv": nil, "process_vm_writev": nil, + "pselect6": nil, "ptrace": nil, "pwrite": nil, "pwrite64": nil, + "pwritev": nil, "pwritev2": nil, "query_module": nil, "quotactl": nil, + "read": nil, "readahead": nil, "readdir": nil, "readlink": nil, + "readlinkat": nil, "readv": nil, "reboot": nil, "recv": nil, + "recvfrom": nil, "recvmsg": nil, "recvmmsg": nil, + "remap_file_pages": nil, "removexattr": nil, "rename": nil, + "renameat": nil, "renameat2": nil, "request_key": nil, + "restart_syscall": nil, "riscv_flush_icache": nil, "rmdir": nil, + "rseq": nil, "rt_sigaction": nil, "rt_sigpending": nil, + "rt_sigprocmask": nil, "rt_sigqueueinfo": nil, "rt_sigreturn": nil, + "rt_sigsuspend": nil, "rt_sigtimedwait": nil, "rt_tgsigqueueinfo": nil, + "rtas": nil, "s390_runtime_instr": nil, "s390_pci_mmio_read": nil, + "s390_pci_mmio_write": nil, "s390_sthyi": nil, + "s390_guarded_storage": nil, "sched_get_affinity": nil, + "sched_get_priority_max": nil, "sched_get_priority_min": nil, + "sched_getaffinity": nil, "sched_getattr": nil, "sched_getparam": nil, + "sched_getscheduler": nil, "sched_rr_get_interval": nil, + "sched_set_affinity": nil, "sched_setaffinity": nil, + "sched_setattr": nil, "sched_setparam": nil, "sched_setscheduler": nil, + "sched_yield": nil, "seccomp": nil, "select": nil, "semctl": nil, + "semget": nil, "semop": nil, "semtimedop": nil, "send": nil, + "sendfile": nil, "sendfile64": nil, "sendmmsg": nil, "sendmsg": nil, + "sendto": nil, "set_mempolicy": nil, "set_robust_list": nil, + "set_thread_area": nil, "set_tid_address": nil, "set_tls": nil, + "setdomainname": nil, "setfsgid": nil, "setfsgid32": nil, + "setfsuid": nil, "setfsuid32": nil, "setgid": nil, "setgid32": nil, + "setgroups": nil, "setgroups32": nil, "sethae": nil, "sethostname": nil, + "setitimer": nil, "setns": nil, "setpgid": nil, "setpgrp": nil, + "setpriority": nil, "setregid": nil, "setregid32": nil, + "setresgid": nil, "setresgid32": nil, "setresuid": nil, + "setresuid32": nil, "setreuid": nil, "setreuid32": nil, + "setrlimit": nil, "setsid": nil, "setsockopt": nil, "settimeofday": nil, + "setuid": nil, "setuid32": nil, "setup": nil, "setxattr": nil, + "sgetmask": nil, "shmat": nil, "shmctl": nil, "shmdt": nil, + "shmget": nil, "shutdown": nil, "sigaction": nil, "sigaltstack": nil, + "signal": nil, "signalfd": nil, "signalfd4": nil, "sigpending": nil, + "sigprocmask": nil, "sigreturn": nil, "sigsuspend": nil, "socket": nil, + "socketcall": nil, "socketpair": nil, "spill": nil, "splice": nil, + "spu_create": nil, "spu_run": nil, "sram_alloc": nil, "sram_free": nil, + "ssetmask": nil, "stat": nil, "stat64": nil, "statfs": nil, + "statfs64": nil, "statx": nil, "stime": nil, "subpage_prot": nil, + "switch_endian": nil, "swapcontext": nil, "swapoff": nil, "swapon": nil, + "symlink": nil, "symlinkat": nil, "sync": nil, "sync_file_range": nil, + "sync_file_range2": nil, "syncfs": nil, "sys_debug_setcontext": nil, + "syscall": nil, "sysfs": nil, "sysinfo": nil, "syslog": nil, + "sysmips": nil, "tee": nil, "tgkill": nil, "time": nil, + "timer_create": nil, "timer_delete": nil, "timer_getoverrun": nil, + "timer_gettime": nil, "timer_settime": nil, + "timerfd_create": nil, "timerfd_gettime": nil, "timerfd_settime": nil, + "times": nil, "tkill": nil, "truncate": nil, "truncate64": nil, + "ugetrlimit": nil, "umask": nil, "umount": nil, "umount2": nil, + "uname": nil, "unlink": nil, "unlinkat": nil, "unshare": nil, + "uselib": nil, "ustat": nil, "userfaultfd": nil, "usr26": nil, + "usr32": nil, "utime": nil, "utimensat": nil, "utimes": nil, + "utrap_install": nil, "vfork": nil, "vhangup": nil, "vm86old": nil, + "vm86": nil, "vmsplice": nil, "wait4": nil, "waitid": nil, + "waitpid": nil, "write": nil, "writev": nil, "xtensa": nil} +} diff --git a/contrib/gotools/srcs/dependtool/tester.go b/contrib/gotools/srcs/dependtool/tester.go new file mode 100644 index 00000000..71d95eac --- /dev/null +++ b/contrib/gotools/srcs/dependtool/tester.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +package dependtool + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "net" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + "time" + u "tools/srcs/common" +) + +const ( + stdinTest = iota + execTest + telnetTest + externalTesting +) + +const ( + stdinTestString = "stdin" + execTestString = "exec" + telnetTestString = "telnet" +) + +const ( + timeOutMs = 3000 + startupSec = 3 +) + +type Testing struct { + TypeTest string `json:"typeTest"` + TimeOutTest int `json:"timeOutMsTest"` + + // Only for telnet test + AddressTelnet string `json:"addressTelnet"` + PortTelnet int `json:"portTelnet"` + + TimeCommand int32 `json:"timeMsCommand"` + ListCommands []string `json:"listCommands"` +} + +// checkTypeTest checks the type of testing: exec, stdin and telnet tests. +// +// It returns an integer which represents the type of tests +func checkTypeTest(testStruct *Testing) int { + + if testStruct == nil { + return externalTesting + } + + if strings.Compare(testStruct.TypeTest, stdinTestString) == 0 { + return stdinTest + } else if strings.Compare(testStruct.TypeTest, execTestString) == 0 { + return execTest + } else if strings.Compare(testStruct.TypeTest, telnetTestString) == 0 { + return telnetTest + } + + return externalTesting +} + +// setDurationTimeOut sets the duration of the timeout by computing a reference +// value. In addition an extra margin value is added (3sec). +// +// It returns a duration either in milliseconds or in seconds. +func setDurationTimeOut(t *Testing, dArgs DynamicArgs) time.Duration { + + if checkTypeTest(t) != externalTesting { + // Compute the number of commands + execution time (+ 3 seconds safe margin) + totalMs := t.TimeCommand*int32(len(t.ListCommands)) + timeOutMs + return time.Duration(totalMs) * time.Millisecond + } + + return time.Duration(dArgs.waitTime+startupSec) * time.Second +} + +// runCommandTester run commands and captures stdout and stderr of a the +// executed command. It will also run the Tester to explore several execution +// paths of the given app. +// +// It returns to string which are respectively stdout and stderr. +func runCommandTester(programPath, programName, command, option string, + testStruct *Testing, dArgs DynamicArgs, data *u.DynamicData) (string, string) { + + timeOut := setDurationTimeOut(testStruct, dArgs) + u.PrintInfo("Duration of " + programName + " : " + timeOut.String()) + ctx, cancel := context.WithTimeout(context.Background(), timeOut) + defer cancel() + + args := strings.Fields("-f " + programPath + " " + option) + cmd := exec.CommandContext(ctx, command, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + bufOut, bufErr, bufIn := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + cmd.Stdout = bufOut // Add io.MultiWriter(os.Stdout) to record on stdout + cmd.Stderr = bufErr // Add io.MultiWriter(os.Stderr) to record on stderr + + if checkTypeTest(testStruct) == stdinTest { + cmd.Stdin = os.Stdin + for _, cmd := range testStruct.ListCommands { + bufIn.Write([]byte(cmd)) + } + } + + // Run the process (traced by strace/ltrace) + if err := cmd.Start(); err != nil { + u.PrintErr(err) + } + + // Run a go routine to handle the tests + go func() { + if checkTypeTest(testStruct) != stdinTest { + Tester(programName, cmd, data, testStruct, dArgs) + + // Kill the program after the tester has finished the job + if err := u.PKill(programName, syscall.SIGINT); err != nil { + u.PrintErr(err) + } + } + }() + + // Ignore the error because the program is killed (waitTime) + _ = cmd.Wait() + + if ctx.Err() == context.DeadlineExceeded { + u.PrintInfo("Time out during executing: " + cmd.String()) + return bufOut.String(), bufErr.String() + } + + return bufOut.String(), bufErr.String() +} + +// Tester runs the executable file of a given application to perform tests to +// get program dependencies. +// +func Tester(programName string, cmd *exec.Cmd, data *u.DynamicData, + testStruct *Testing, dArgs DynamicArgs) { + + if len(dArgs.testFile) > 0 { + // Wait until the program has started + time.Sleep(time.Second * startupSec) + u.PrintInfo("Run internal tests from file " + dArgs.testFile) + + // Launch execution tests + if checkTypeTest(testStruct) == execTest { + launchTestsExternal(testStruct) + } else if checkTypeTest(testStruct) == telnetTest { + if len(testStruct.AddressTelnet) == 0 || testStruct.PortTelnet == 0 { + u.PrintWarning("Cannot find Address and port for telnet " + + "within json file. Skip tests") + } else { + launchTelnetTest(testStruct) + } + } + } else { + u.PrintInfo("Waiting for external tests for " + strconv.Itoa( + dArgs.waitTime) + " sec") + ticker := time.Tick(time.Second) + for i := 1; i <= dArgs.waitTime; i++ { + <-ticker + fmt.Printf("-") + } + fmt.Printf("\n") + } + + // Gather shared libs + u.PrintHeader2("(*) Gathering shared libs") + if err := gatherDynamicSharedLibs(programName, cmd.Process.Pid, data, + dArgs.fullDeps); err != nil { + u.PrintWarning(err) + } +} + +//----------------------------------Tests--------------------------------------- + +// launchTestsExternal runs external tests written in the 'test.json' file. +// +func launchTestsExternal(testStruct *Testing) { + + for _, cmd := range testStruct.ListCommands { + if len(cmd) > 0 { + + // Perform a sleep between command if specified + if testStruct.TimeCommand > 0 { + timeMs := rand.Int31n(testStruct.TimeCommand) + time.Sleep(time.Duration(timeMs) * time.Millisecond) + } + + // Execute each line as a command + if _, err := u.ExecutePipeCommand(cmd); err != nil { + u.PrintWarning("Impossible to execute test: " + cmd) + } else { + u.PrintInfo("Test executed: " + cmd) + } + } + } +} + +// launchTelnetTest runs telnet tests written in the 'test.json' file. +// +func launchTelnetTest(testStruct *Testing) { + + addr := testStruct.AddressTelnet + ":" + strconv.Itoa(testStruct.PortTelnet) + conn, _ := net.Dial("tcp", addr) + + for _, cmd := range testStruct.ListCommands { + if len(cmd) > 0 { + + // Perform a sleep between command if specified + if testStruct.TimeCommand > 0 { + timeMs := rand.Int31n(testStruct.TimeCommand) + time.Sleep(time.Duration(timeMs) * time.Millisecond) + } + + // Set a timeout to avoid blocking + if err := conn.SetReadDeadline( + time.Now().Add(time.Duration(timeOutMs) * time.Millisecond)); err != nil { + u.PrintWarning("Impossible to set a timeout to TCP command") + } + + // Send commands (test) + if _, err := fmt.Fprintf(conn, cmd+"\n"); err != nil { + u.PrintWarning("Impossible to execute test: " + cmd) + } else { + u.PrintInfo("Test executed: " + cmd) + } + + // Read response + message := readerTelnet(conn) + fmt.Println("----->Message from server: " + message) + } + } +} + +// readerTelnet reads data from the telnet connection. +// +func readerTelnet(conn net.Conn) (out string) { + var buffer [1]byte + recvData := buffer[:] + var n int + var err error + + for { + n, err = conn.Read(recvData) + if n <= 0 || err != nil { + break + } else { + out += string(recvData) + } + } + return out +} diff --git a/contrib/gotools/srcs/main.go b/contrib/gotools/srcs/main.go new file mode 100644 index 00000000..9b806efe --- /dev/null +++ b/contrib/gotools/srcs/main.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package main + +import ( + "os/user" + "tools/srcs/buildtool" + u "tools/srcs/common" + "tools/srcs/crawlertool" + "tools/srcs/dependtool" + "tools/srcs/veriftool" +) + +func main() { + + // Init global arguments + args := new(u.Arguments) + parser, err := args.InitArguments() + if err != nil { + u.PrintErr(err) + } + + // Parse arguments + if err := args.ParseMainArguments(parser, args); err != nil { + u.PrintErr(err) + } + + // Checks if the toolchain must be completely executed + all := false + if !*args.BoolArg[u.DEP] && !*args.BoolArg[u.BUILD] && + !*args.BoolArg[u.VERIF] && !*args.BoolArg[u.PERF] { + all = true + } + + // Get user home folder + usr, err := user.Current() + if err != nil { + u.PrintErr(err) + } + + var data *u.Data + + if *args.BoolArg[u.CRAWLER] { + u.PrintHeader1("(*) RUN CRAWLER UNIKRAFT ANALYSER") + crawlertool.RunCrawler() + return + } + + if all || *args.BoolArg[u.DEP] { + + // Initialize data + data = new(u.Data) + + u.PrintHeader1("(1) RUN DEPENDENCIES ANALYSER") + dependtool.RunAnalyserTool(usr.HomeDir, data) + } + + if all || *args.BoolArg[u.BUILD] { + u.PrintHeader1("(2) AUTOMATIC BUILD TOOL") + buildtool.RunBuildTool(usr.HomeDir, data) + } + + if all || *args.BoolArg[u.VERIF] { + u.PrintHeader1("(3) VERIFICATION TOOL") + veriftool.RunVerificationTool() + } + + if all || *args.BoolArg[u.PERF] { + u.PrintHeader1("(4) PERFORMANCE OPTIMIZATION TOOL") + } +} diff --git a/contrib/gotools/srcs/veriftool/args.go b/contrib/gotools/srcs/veriftool/args.go new file mode 100644 index 00000000..4bbe73ce --- /dev/null +++ b/contrib/gotools/srcs/veriftool/args.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package veriftool + +import ( + "os" + u "tools/srcs/common" + + "github.com/akamensky/argparse" +) + +const ( + programArg = "program" + testFileArg = "testFile" + configFileArg = "configFile" + optionsArg = "options" + unikraftArg = "unikraft" +) + +// ParseArguments parses arguments of the application. +// +// It returns an error if any, otherwise it returns nil. +func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error { + + args.InitArgParse(p, args, u.STRING, "p", programArg, + &argparse.Options{Required: true, Help: "Program name"}) + args.InitArgParse(p, args, u.STRING, "t", testFileArg, + &argparse.Options{Required: false, Help: "Path of the test file"}) + args.InitArgParse(p, args, u.STRING, "c", configFileArg, + &argparse.Options{Required: false, Help: "Path of the config file"}) + args.InitArgParse(p, args, u.STRING, "o", optionsArg, + &argparse.Options{Required: false, Default: "", Help: "Extra options for " + + "launching program"}) + + args.InitArgParse(p, args, u.STRING, "u", unikraftArg, + &argparse.Options{Required: false, Help: "Unikraft Path"}) + + return u.ParserWrapper(p, os.Args) +} diff --git a/contrib/gotools/srcs/veriftool/run_veriftool.go b/contrib/gotools/srcs/veriftool/run_veriftool.go new file mode 100644 index 00000000..9fefc9a2 --- /dev/null +++ b/contrib/gotools/srcs/veriftool/run_veriftool.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Authors: Gaulthier Gain +// +// Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package veriftool + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + u "tools/srcs/common" + + "github.com/sergi/go-diff/diffmatchpatch" +) + +const stdinCmd = "[STDIN]" +const testCmd = "[TEST]" + +func RunVerificationTool() { + + // Init and parse local arguments + args := new(u.Arguments) + p, err := args.InitArguments() + if err != nil { + u.PrintErr(err) + } + if err := parseLocalArguments(p, args); err != nil { + u.PrintErr(err) + } + + // Get program Name + programName := *args.StringArg[programArg] + + // Take base path if absolute path is used + if filepath.IsAbs(programName) { + programName = filepath.Base(programName) + } + + unikraftPath := *args.StringArg[unikraftArg] + if len(unikraftPath) == 0 { + u.PrintErr("Unikraft folder must exist! Run the build tool before " + + "using the verification tool") + } + + // Get the app folder + var appFolder string + if unikraftPath[len(unikraftPath)-1] != os.PathSeparator { + appFolder = unikraftPath + u.SEP + u.APPSFOLDER + programName + u.SEP + } else { + appFolder = unikraftPath + u.APPSFOLDER + programName + u.SEP + } + + // Get the build folder + buildAppFolder := appFolder + u.BUILDFOLDER + + // Get KVM image + var kvmUnikernel string + if file, err := u.OSReadDir(buildAppFolder); err != nil { + u.PrintWarning(err) + } else { + for _, f := range file { + if !f.IsDir() && strings.Contains(f.Name(), u.KVM_IMAGE) && + len(filepath.Ext(f.Name())) == 0 { + kvmUnikernel = f.Name() + } + } + } + + // Kvm unikernel image + if len(kvmUnikernel) == 0 { + u.PrintWarning(errors.New("no KVM image found")) + } + + // Read test + argStdin := "" + if len(*args.StringArg[testFileArg]) > 0 { + + var err error + var cmdTests []string + cmdTests, err = u.ReadLinesFile(*args.StringArg[testFileArg]) + if err != nil { + u.PrintWarning("Cannot find test files" + err.Error()) + } + if strings.Contains(cmdTests[0], stdinCmd) { + argStdin = strings.Join(cmdTests[1:], "") + argStdin += "\n" + } else if strings.Contains(cmdTests[0], testCmd) { + //todo add for other tests + } + } + + // Test KVM app unikernel + unikernelFilename := appFolder + "output_" + kvmUnikernel + ".txt" + if err := testUnikernel(buildAppFolder+kvmUnikernel, unikernelFilename, + []byte(argStdin)); err != nil { + u.PrintWarning("Impossible to write the output of verification to " + + unikernelFilename) + } + + // Test general app + appFilename := appFolder + "output_" + programName + ".txt" + if err := testApp(programName, appFilename, []byte(argStdin)); err != nil { + u.PrintWarning("Impossible to write the output of verification to " + + unikernelFilename) + } + + u.PrintInfo("Comparison output:") + + // Compare both output + fmt.Println(compareOutput(unikernelFilename, appFilename)) + +} + +func compareOutput(unikernelFilename, appFilename string) string { + f1, err := ioutil.ReadFile(unikernelFilename) + if err != nil { + u.PrintErr(err) + } + + f2, err := ioutil.ReadFile(appFilename) + if err != nil { + u.PrintErr(err) + } + + dmp := diffmatchpatch.New() + + diffs := dmp.DiffMain(string(f2), string(f1), false) + + return dmp.DiffPrettyText(diffs) +} + +func testApp(programName, outputFile string, argsStdin []byte) error { + bOut, _ := u.ExecuteRunCmdStdin(programName, argsStdin) + + return u.WriteToFile(outputFile, bOut) +} + +func testUnikernel(kvmUnikernel, outputFile string, argsStdin []byte) error { + argsQemu := []string{"-nographic", "-vga", "none", "-device", + "isa-debug-exit", "-kernel", kvmUnikernel} + + bOut, _ := u.ExecuteRunCmdStdin("qemu-system-x86_64", argsStdin, argsQemu...) + + return u.WriteToFile(outputFile, bOut) +} diff --git a/contrib/gotools/testfiles/queries_firebird.txt b/contrib/gotools/testfiles/queries_firebird.txt new file mode 100644 index 00000000..bd135bd6 --- /dev/null +++ b/contrib/gotools/testfiles/queries_firebird.txt @@ -0,0 +1,18 @@ +display +z +mapping set +add test +delete test +add test +display test +modify test -pw newpassword +modify test -uid 1892 +modify test -admin yes +add newuser -pw newuser -fname New -lname User +display newuser +add newadmin -pw secret -fname New -mname admin -lname User -admin yes +display newadmin +delete newuser +modify test -pw test +modify test -mname MiddleName -fname Fred +create database 'test' user 'test' password 'test'; \ No newline at end of file diff --git a/contrib/gotools/testfiles/queries_influx.txt b/contrib/gotools/testfiles/queries_influx.txt new file mode 100755 index 00000000..605667f9 --- /dev/null +++ b/contrib/gotools/testfiles/queries_influx.txt @@ -0,0 +1,85 @@ +# DDL +CREATE DATABASE pirates +CREATE RETENTION POLICY oneday ON pirates DURATION 1d REPLICATION 1 + +# DML +# CONTEXT-DATABASE: pirates +# CONTEXT-RETENTION-POLICY: oneday + +treasures,captain_id=dread_pirate_roberts value=801 1439856000 +treasures,captain_id=flint value=29 1439856000 +treasures,captain_id=sparrow value=38 1439856000 +treasures,captain_id=tetra value=47 1439856000 +treasures,captain_id=crunch value=109 1439858880 + +SHOW DATABASES +USE NOAA_water_database +SHOW measurements +SHOW RETENTION POLICIES ON NOAA_water_database +SHOW SERIES ON NOAA_water_database +SHOW SERIES ON NOAA_water_database FROM "h2o_quality" WHERE "location" = 'coyote_creek' LIMIT 2 +SHOW MEASUREMENTS ON NOAA_water_database WITH MEASUREMENT =~ /h2o.*/ WHERE "randtag" =~ /\d/ +SHOW TAG KEYS ON "NOAA_water_database" +SHOW FIELD KEYS ON "NOAA_water_database" + +SELECT * FROM "h2o_feet" +SELECT "level description","location","water_level" FROM "h2o_feet" +SELECT "level description"::field,"location"::tag,"water_level"::field FROM "h2o_feet" +SELECT *::field FROM "h2o_feet" +SELECT * FROM "NOAA_water_database"."autogen"."h2o_feet" +SELECT * FROM "NOAA_water_database".."h2o_feet" +SELECT "location" FROM "h2o_feet" +SELECT "water_level","location" FROM "h2o_feet" +SELECT * FROM "h2o_feet" WHERE "water_level" > 8 +SELECT * FROM "h2o_feet" WHERE "level description" = 'below 3 feet' +SELECT * FROM "h2o_feet" WHERE "water_level" + 2 > 11.9 +SELECT "water_level" FROM "h2o_feet" WHERE "location" = 'santa_monica' +SELECT "water_level" FROM "h2o_feet" WHERE "location" <> 'santa_monica' AND (water_level < -0.59 OR water_level > 9.95) +SELECT * FROM "h2o_feet" WHERE time > now() - 7d +SELECT "water_level" FROM "h2o_feet" WHERE "location" = 'santa_monica +SELECT "level description" FROM "h2o_feet" WHERE "level description" = "at or greater than 9 feet" +SELECT MEAN("water_level") FROM "h2o_feet" GROUP BY "location" +SELECT MEAN("index") FROM "h2o_quality" GROUP BY location,randtag +SELECT MEAN("index") FROM "h2o_quality" GROUP BY * +SELECT "water_level","location" FROM "h2o_feet" WHERE time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:30:00Z' +SELECT COUNT("water_level") FROM "h2o_feet" WHERE "location"='coyote_creek' AND time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:30:00Z' GROUP BY time(12m) +SELECT COUNT("water_level") FROM "h2o_feet" WHERE time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:30:00Z' GROUP BY time(12m),"location" +SELECT "water_level" FROM "h2o_feet" WHERE "location"='coyote_creek' AND time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:18:00Z' +SELECT MEAN("water_level") FROM "h2o_feet" WHERE "location"='coyote_creek' AND time >= '2015-08-18T00:06:00Z' AND time <= '2015-08-18T00:54:00Z' GROUP BY time(18m) +SELECT COUNT("water_level") FROM "h2o_feet" WHERE "location"='coyote_creek' AND time >= '2015-08-18T00:06:00Z' AND time < '2015-08-18T00:18:00Z' GROUP BY time(12m,6m) +SELECT COUNT("water_level") FROM "h2o_feet" WHERE "location"='coyote_creek' AND time >= '2015-08-18T00:06:00Z' AND time < '2015-08-18T00:18:00Z' GROUP BY time(12m) +SELECT MAX("water_level") FROM "h2o_feet" WHERE "location"='coyote_creek' AND time >= '2015-09-18T16:00:00Z' AND time <= '2015-09-18T16:42:00Z' GROUP BY time(12m) +SELECT MAX("water_level") FROM "h2o_feet" WHERE location = 'coyote_creek' AND time >= '2015-09-18T16:36:00Z' AND time <= '2015-09-18T16:54:00Z' GROUP BY time(12m) fill(previous) +SELECT MEAN("tadpoles") FROM "pond" WHERE time > '2016-11-11T21:24:00Z' AND time <= '2016-11-11T22:06:00Z' GROUP BY time(12m) fill(linear) +SELECT "water_level" FROM "h2o_feet" WHERE "location"='coyote_creek' AND time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:54:00Z' +SELECT * FROM "h2o_feet","h2o_pH" +SELECT "water_level" INTO "h2o_feet_copy_1" FROM "h2o_feet" WHERE "location" = 'coyote_creek' +SELECT "water_level" INTO "where_else"."autogen"."h2o_feet_copy_2" FROM "h2o_feet" WHERE "location" = 'coyote_creek' +SELECT * FROM "all_my_averages" +SELECT * FROM "where_else"."autogen"./.*/ +SELECT "water_level" FROM "h2o_feet" WHERE "location" = 'santa_monica' ORDER BY time DESC +SELECT MEAN("water_level") FROM "h2o_feet" WHERE time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:42:00Z' GROUP BY *,time(12m) LIMIT 2 +SELECT "water_level" FROM "h2o_feet" GROUP BY * SLIMIT 1 +SELECT "water_level","location" FROM "h2o_feet" LIMIT 3 OFFSET 3 +SELECT "water_level" FROM "h2o_feet" GROUP BY * SLIMIT 1 SOFFSET 1 +SELECT "water_level" FROM "h2o_feet" WHERE "location" = 'santa_monica' AND time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:18:00Z' tz('America/Chicago') +SELECT "water_level" FROM "h2o_feet" WHERE "location" = 'santa_monica' AND time >= '2015-08-18T00:00:00.000000000Z' AND time <= '2015-08-18T00:12:00Z' +SELECT "water_level" FROM "h2o_feet" WHERE "location" = 'santa_monica' AND time >= 1439856000s AND time <= 1439856720s +SELECT "level description" FROM "h2o_feet" WHERE time > '2015-09-18T21:18:00Z' AND time < now() + 1000d +SELECT MEAN("water_level") FROM "h2o_feet" WHERE "location"='santa_monica' AND time >= '2015-09-18T21:30:00Z' GROUP BY time(12m) fill(none) +SELECT /l/ FROM "h2o_feet" LIMIT 1 +SELECT MEAN("water_level") FROM "h2o_feet" WHERE "location" =~ /./ +SELECT MEAN("water_level") FROM "h2o_feet" +SELECT SUM("max") FROM (SELECT MAX("water_level") FROM "h2o_feet" GROUP BY "location") +SELECT DERIVATIVE(MEAN("water_level")) AS "water_level_derivative" FROM "h2o_feet" WHERE time >= '2015-08-18T00:00:00Z' AND time <= '2015-08-18T00:30:00Z' GROUP BY time(12m),"location" + +SELECT * FROM "add" WHERE "A" + 5 > 10 +SELECT "A" ^ 4294967295 FROM "data" + +CREATE RETENTION POLICY "one_day_only" ON "NOAA_water_database" DURATION 23h60m REPLICATION 1 DEFAULT +ALTER RETENTION POLICY "what_is_time" ON "NOAA_water_database" DURATION 3w SHARD DURATION 2h DEFAULT +DROP RETENTION POLICY "what_is_time" ON "NOAA_water_database" +DROP SERIES FROM "h2o_feet" WHERE "location" = 'santa_monica' +DROP SERIES FROM "h2o_feet" + + diff --git a/contrib/gotools/testfiles/queries_mongo1.js b/contrib/gotools/testfiles/queries_mongo1.js new file mode 100755 index 00000000..9a5b0de4 --- /dev/null +++ b/contrib/gotools/testfiles/queries_mongo1.js @@ -0,0 +1,10 @@ +use admin +db.dropUser("myUserAdmin") +db.createUser( + { + user: "myUserAdmin", + pwd: "abc123", + roles: [ { role: "userAdminAnyDatabase", db: "admin" }, "readWriteAnyDatabase" ] + } +) +exit \ No newline at end of file diff --git a/contrib/gotools/testfiles/queries_mongo2.js b/contrib/gotools/testfiles/queries_mongo2.js new file mode 100644 index 00000000..979c0681 --- /dev/null +++ b/contrib/gotools/testfiles/queries_mongo2.js @@ -0,0 +1,99 @@ +db +use test +db.createUser( + { + user: "myTester", + pwd: "xyz123", + roles: [ { role: "readWrite", db: "test" }, + { role: "read", db: "reporting" } ] + } +) +db.foo.insert( { x: 1, y: 1 } ) +show dbs +show collections +db.getCollectionNames(); +db.printCollectionStats() +show users +show roles +show profile +show databases +use myNewDatabase +db.hostInfo() +db.myCollection.insertOne( { x: 1 } ); +db.getCollection("3 test").find() +db.getCollection("3-test").find() +db.getCollection("stats").find() +db.inventory.insertMany([ + // MongoDB adds the _id field with an ObjectId if _id is not present + { item: "journal", qty: 25, status: "A", + size: { h: 14, w: 21, uom: "cm" }, tags: [ "blank", "red" ] }, + { item: "notebook", qty: 50, status: "A", + size: { h: 8.5, w: 11, uom: "in" }, tags: [ "red", "blank" ] }, + { item: "paper", qty: 100, status: "D", + size: { h: 8.5, w: 11, uom: "in" }, tags: [ "red", "blank", "plain" ] }, + { item: "planner", qty: 75, status: "D", + size: { h: 22.85, w: 30, uom: "cm" }, tags: [ "blank", "red" ] }, + { item: "postcard", qty: 45, status: "A", + size: { h: 10, w: 15.25, uom: "cm" }, tags: [ "blue" ] } +]); +db.inventory.find( {} ) +db.inventory.find( { status: "D" } ) +db.inventory.find( { size: { h: 14, w: 21, uom: "cm" } } ) +db.inventory.find( { "size.uom": "in" } ) +db.inventory.find( { tags: "red" } ) +db.inventory.find( { tags: ["red", "blank"] } ) +use myNewDB +db.myNewCollection1.insertOne( { x: 1 } ) +db.myNewCollection2.insertOne( { x: 1 } ) +db.myNewCollection3.createIndex( { y: 1 } ) +db.runCommand( { create: , viewOn: , pipeline: } ) +db.runCommand( { create: , viewOn: , pipeline: , collation: } ) +db.createView(, , , ) +db.view.find().sort({$natural: 1}) +db.createCollection( "log", { capped: true, size: 100000 } ) +db.createCollection("log", { capped : true, size : 5242880, max : 5000 } ) +db.cappedCollection.find().sort( { $natural: -1 } ) +db.collection.isCapped() +db.runCommand({"convertToCapped": "mycoll", size: 100000}); +var mydoc = { + _id: ObjectId("5099803df3f4948bd2f98391"), + name: { first: "Alan", last: "Turing" }, + birth: new Date('Jun 23, 1912'), + death: new Date('Jun 07, 1954'), + contribs: [ "Turing machine", "Turing test", "Turingery" ], + views : NumberLong(1250000) +} +var a = new Timestamp(); +db.test.insertOne( { ts: a } ); +{ "_id" : ObjectId("542c2b97bac0595474108b48"), "ts" : Timestamp(1412180887, 1) } +var mydate1 = new Date() +var mydate2 = ISODate() +mydate1.toString() +mydate1.getMonth() +{ + locale: , + caseLevel: , + caseFirst: , + strength: , + numericOrdering: , + alternate: , + maxVariable: , + backwards: +} +{ "$binary": "", "$type": "" } +db.json.insert( { longQuoted : NumberLong("9223372036854775807") } ) +db.json.insert( { longUnQuoted : NumberLong(9223372036854775807) } ) +db.json.find() +db.json.insert( { decimalQuoted : NumberDecimal("123.40") } ) +db.json.insert( { decimalUnQuoted : NumberDecimal(123.40) } ) +db.json.find() +db.students.drop( { writeConcern: { w: "majority" } } ) +db.students.drop() +db.printCollectionStats() +db.printReplicationInfo() +db.printShardingStatus() +db.printSlaveReplicationInfo() +db.repairDatabase() +db.resetError() +db.getMongo() +exit \ No newline at end of file diff --git a/contrib/gotools/testfiles/queries_mysql.txt b/contrib/gotools/testfiles/queries_mysql.txt new file mode 100755 index 00000000..894148b5 --- /dev/null +++ b/contrib/gotools/testfiles/queries_mysql.txt @@ -0,0 +1,141 @@ +SELECT '' AS ' '; +SHOW VARIABLES LIKE '%ssl%'; +SHOW SESSION STATUS LIKE 'Ssl_cipher'; +SET @plaintextpassword = 'hello'; +SET @USER = 'hello'; +SELECT UPPER(SHA1(UNHEX(SHA1(@plaintextpassword)))) PWD_CREATION; +SELECT PASSWORD(@plaintextpassword) PWD_FUNCTION; +SELECT User, Host FROM mysql.user; +SELECT authentication_string FROM mysql.user; +SHOW VARIABLES LIKE 'validate_password%'; +#UNINSTALL PLUGIN validate_password; +DROP USER IF EXISTS 'test'@'localhost'; +FLUSH privileges; +CREATE USER 'test'@'localhost' IDENTIFIED BY 'test'; +GRANT ALL PRIVILEGES ON *.* TO 'test'@'localhost'; +SHOW DATABASES; +DROP DATABASE IF EXISTS menagerie; +CREATE DATABASE menagerie; +USE menagerie; +SHOW TABLES; +DROP TABLE IF EXISTS pet; +SHOW TABLES; +CREATE TABLE pet (name VARCHAR(20), owner VARCHAR(20), species VARCHAR(20), sex CHAR(1), birth DATE, death DATE); +DESCRIBE pet; +INSERT INTO pet VALUES ('Fluffy','Harold','cat','f','1993-02-04', NULL); +INSERT INTO pet VALUES ('Claws','Gwen','cat','m','1994-03-17', NULL); +INSERT INTO pet VALUES ('Buffy','Harold','dog','f','1989-05-13',NULL); +INSERT INTO pet VALUES ('Fang',' Benny','dog','m','1990-08-27', NULL); +INSERT INTO pet VALUES ('Whistler','Gwen','bird',NULL,'1997-12-09', NULL); +INSERT INTO pet VALUES ('Slim','Benny','snake','m','1996-04-29', NULL); +SELECT * FROM pet; +DELETE FROM pet; +INSERT INTO pet VALUES ('Puffball','Diane','hamster','f','1999-03-30',NULL); +UPDATE pet SET birth = '1989-08-31' WHERE name = 'Bowser'; +SELECT * FROM pet; +SELECT * FROM pet WHERE name = 'Bowser'; +SELECT * FROM pet WHERE birth >= '1998-1-1'; +SELECT * FROM pet WHERE species = 'dog' AND sex = 'f'; +SELECT * FROM pet WHERE species = 'snake' OR species = 'bird'; +SELECT * FROM pet WHERE (species = 'cat' AND sex = 'm') OR (species = 'dog' AND sex = 'f'); +SELECT name, birth FROM pet; +SELECT owner FROM pet; +SELECT DISTINCT owner FROM pet; +SELECT name, species, birth FROM pet WHERE species = 'dog' OR species = 'cat'; +SELECT name, birth FROM pet ORDER BY birth; +SELECT name, birth FROM pet ORDER BY birth DESC; +SELECT name, species, birth FROM pet ORDER BY species, birth DESC; +SELECT name, birth, CURDATE(), TIMESTAMPDIFF(YEAR,birth,CURDATE()) AS age FROM pet; +SELECT name, birth, CURDATE(), TIMESTAMPDIFF(YEAR,birth,CURDATE()) AS age FROM pet ORDER BY name; +SELECT name, birth, death, TIMESTAMPDIFF(YEAR,birth,death) AS age FROM pet WHERE death IS NOT NULL ORDER BY age; +SELECT name, birth, MONTH(birth) FROM pet; +SELECT name, birth FROM pet WHERE MONTH(birth) = 5; +SELECT name, birth FROM pet WHERE MONTH(birth) = MOD(MONTH(CURDATE()), 12) + 1; +SELECT '2018-10-31' + INTERVAL 1 DAY; +SELECT '2018-10-32' + INTERVAL 1 DAY; +SHOW WARNINGS; +SELECT 1 IS NULL, 1 IS NOT NULL; +SELECT 1 = NULL, 1 <> NULL, 1 < NULL, 1 > NULL; +SELECT 0 IS NULL, 0 IS NOT NULL, '' IS NULL, '' IS NOT NULL; +SELECT * FROM pet WHERE name LIKE 'b%'; +SELECT * FROM pet WHERE name LIKE '%fy'; +SELECT * FROM pet WHERE name LIKE '%w%'; +SELECT * FROM pet WHERE name LIKE '_____'; +SELECT * FROM pet WHERE name REGEXP '^b'; +SELECT * FROM pet WHERE name REGEXP BINARY '^b'; +SELECT * FROM pet WHERE name REGEXP 'fy$'; +SELECT * FROM pet WHERE name REGEXP 'w'; +SELECT * FROM pet WHERE name REGEXP '^.....$'; +SELECT * FROM pet WHERE name REGEXP '^.{5}$'; +SELECT COUNT(*) FROM pet; +SELECT owner, COUNT(*) FROM pet GROUP BY owner; +SELECT species, COUNT(*) FROM pet GROUP BY species; +SELECT sex, COUNT(*) FROM pet GROUP BY sex; +SELECT species, sex, COUNT(*) FROM pet GROUP BY species, sex; +SELECT species, sex, COUNT(*) FROM pet WHERE species = 'dog' OR species = 'cat' GROUP BY species, sex; +SELECT species, sex, COUNT(*) FROM pet WHERE sex IS NOT NULL GROUP BY species, sex; +SET sql_mode = 'ONLY_FULL_GROUP_BY'; +SET sql_mode = ''; +SELECT owner, COUNT(*) FROM pet; +DROP TABLE IF EXISTS event; +SHOW TABLES; +CREATE TABLE event (name VARCHAR(20), date DATE, type VARCHAR(15), remark VARCHAR(255)); +INSERT INTO event VALUES ('Fluffy','1995-05-15','litter','4 kittens, 3 female, 1 male'); +INSERT INTO event VALUES ('Buffy','1993-06-23','litter','5 puppies, 2 female, 3 male'); +INSERT INTO event VALUES ('Buffy','1994-06-19','litter','3 puppies, 3 female'); +INSERT INTO event VALUES ('Chirpy','1999-03-21','vet','needed beak straightened'); +INSERT INTO event VALUES ('Slim','1997-08-03','vet','broken rib'); +INSERT INTO event VALUES ('Bowser','1991-10-12','kennel', NULL); +INSERT INTO event VALUES ('Fang','1991-10-12','kennel',NULL); +INSERT INTO event VALUES ('Fang','1998-08-28','birthday','Gave him a new chew toy'); +INSERT INTO event VALUES ('Claws','1998-03-17','birthday','Gave him a new flea collar'); +INSERT INTO event VALUES ('Whistler','1998-12-09','birthday','First birthday'); +SELECT pet.name, TIMESTAMPDIFF(YEAR,birth,date) AS age, remark FROM pet INNER JOIN event ON pet.name = event.name WHERE event.type = 'litter'; +SELECT p1.name, p1.sex, p2.name, p2.sex, p1.species FROM pet AS p1 INNER JOIN pet AS p2 ON p1.species = p2.species AND p1.sex = 'f' AND p1.death IS NULL AND p2.sex = 'm' AND p2.death IS NULL; +SELECT DATABASE(); +DESCRIBE pet; +DROP TABLE IF EXISTS shop; +CREATE TABLE shop (article INT(4) UNSIGNED ZEROFILL DEFAULT '0000' NOT NULL, dealer CHAR(20) DEFAULT '' NOT NULL, price DOUBLE(16,2) DEFAULT '0.00' NOT NULL, PRIMARY KEY(article, dealer)); +INSERT INTO shop VALUES (1,'A',3.45),(1,'B',3.99),(2,'A',10.99),(3,'B',1.45), (3,'C',1.69),(3,'D',1.25),(4,'D',19.95); +SELECT * FROM shop ORDER BY article; +SELECT MAX(article) AS article FROM shop; +SELECT article, dealer, price FROM shop WHERE price=(SELECT MAX(price) FROM shop); +SELECT s1.article, s1.dealer, s1.price FROM shop s1 LEFT JOIN shop s2 ON s1.price < s2.price WHERE s2.article IS NULL; +SELECT article, dealer, price FROM shop ORDER BY price DESC LIMIT 1; +SELECT article, MAX(price) AS price FROM shop GROUP BY article ORDER BY article; +SELECT article, dealer, price FROM shop s1 WHERE price=(SELECT MAX(s2.price) FROM shop s2 WHERE s1.article = s2.article) ORDER BY article; +SELECT s1.article, dealer, s1.price +FROM shop s1 +JOIN (SELECT article, MAX(price) AS price FROM shop GROUP BY article) AS s2 ON s1.article = s2.article AND s1.price = s2.price ORDER BY article; +SELECT s1.article, s1.dealer, s1.price FROM shop s1 LEFT JOIN shop s2 ON s1.article = s2.article AND s1.price < s2.price WHERE s2.article IS NULL ORDER BY s1.article; +SELECT @min_price:=MIN(price),@max_price:=MAX(price) FROM shop; +SELECT * FROM shop WHERE price=@min_price OR price=@max_price; +CREATE TABLE person (id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, name CHAR(60) NOT NULL, PRIMARY KEY (id)); +CREATE TABLE shirt (id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, style ENUM('t-shirt', 'polo', 'dress') NOT NULL, color ENUM('red', 'blue', 'orange', 'white', 'black') NOT NULL, owner SMALLINT UNSIGNED NOT NULL REFERENCES person(id), PRIMARY KEY (id)); +INSERT INTO person VALUES (NULL, 'Antonio Paz'); +SELECT @last := LAST_INSERT_ID(); +INSERT INTO shirt VALUES(NULL, 'polo', 'blue', @last), (NULL, 'dress', 'white', @last), (NULL, 't-shirt', 'blue', @last); +INSERT INTO person VALUES (NULL, 'Lilliana Angelovska'); +SELECT @last := LAST_INSERT_ID(); +INSERT INTO shirt VALUES(NULL, 'dress', 'orange', @last),(NULL, 'polo', 'red', @last),(NULL, 'dress', 'blue', @last),(NULL, 't-shirt', 'white', @last); +SELECT * FROM person; +SELECT * FROM shirt; +SELECT s.* FROM person p INNER JOIN shirt s ON s.owner = p.id WHERE p.name LIKE 'Lilliana%' AND s.color <> 'white'; +SHOW CREATE TABLE shirt\G +CREATE TABLE t1 (year YEAR(4), month INT(2) UNSIGNED ZEROFILL, day INT(2) UNSIGNED ZEROFILL); +INSERT INTO t1 VALUES(2000,1,1),(2000,1,20),(2000,1,30),(2000,2,2), (2000,2,23),(2000,2,23); +SELECT year,month,BIT_COUNT(BIT_OR(1<\", | sort | uniq -c", + "sudo exim -bpr | grep -Eo \"^\s*[^ ]*@[^ ]*$\", | sort | uniq -c", + "sudo exiqgrep -o 43000 -i | xargs exim -Mrm", + "sudo exiqgrep -z -i | xargs exim -Mrm", + "sudo exiqgrep -i -f [user]@domaine | xargs exim -Mrm", + "sudo exiqgrep -o 43000 -i -f [user]@domaine | xargs exim -Mrm", + "sudo grep -lr 'bla bla bla' /var/spool/exim/input/ | sed -e 's/^.*\/\([a-zA-Z0-9-]*\)-[DH]$/\1/g' | xargs exim -Mrm" + ] +} \ No newline at end of file diff --git a/kraft/commands/__init__.py b/kraft/commands/__init__.py index 2546a4f1..775a4a13 100644 --- a/kraft/commands/__init__.py +++ b/kraft/commands/__init__.py @@ -38,6 +38,7 @@ from .init import init from .lib.bump import bump as libbump from .lib.init import init as libinit +from .devel.dependency import dependency as develdependency from .list import list from .run import run from .up import up diff --git a/kraft/commands/devel/dependency.py b/kraft/commands/devel/dependency.py new file mode 100644 index 00000000..74919f37 --- /dev/null +++ b/kraft/commands/devel/dependency.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Authors: Gaulthier Gain +# +# Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import sys + +import click + +from kraft.commands.list import update +from kraft.components.library import Library +from kraft.components.types import RepositoryType +from kraft.context import kraft_context +from kraft.errors import KraftError +from kraft.logger import logger +from kraft.utils import op, dir + +TOOLS = "./tools" + +@click.command('dependency', short_help='Gather the dependencies of a specific application.', context_settings=dict( + ignore_unknown_options=True, + allow_extra_args=True)) # noqa: C901 +@click.pass_context +def dependency(ctx): + """ + Gather the dependencies of a specific application. + """ + # check if bin/tools is PATH + path = op.which(TOOLS) + if path is not None: + # Execute the toolchain with unparsed arguments + op.execute_command(path, ctx.args) + else: + logger.error('Program is not found in PATH') + sys.exit(1) \ No newline at end of file diff --git a/kraft/kraft.py b/kraft/kraft.py index c4623847..f9e06138 100644 --- a/kraft/kraft.py +++ b/kraft/kraft.py @@ -36,6 +36,7 @@ from kraft.commands import build from kraft.commands import clean from kraft.commands import configure +from kraft.commands import develdependency from kraft.commands import init from kraft.commands import libbump from kraft.commands import libinit @@ -87,9 +88,18 @@ def lib(ctx): """ pass +@click.group(name='devel', short_help='Unikraft devel commands.') +@kraft_context +def devel(ctx): + """ + Unikraft developer sub-commands useful for maintaing and working + directly with Unikraft source code. + """ + pass lib.add_command(libinit) lib.add_command(libbump) +devel.add_command(develdependency) kraft.add_command(up) kraft.add_command(run) @@ -99,3 +109,4 @@ def lib(ctx): kraft.add_command(clean) kraft.add_command(configure) kraft.add_command(lib) +kraft.add_command(devel) \ No newline at end of file diff --git a/kraft/utils/op.py b/kraft/utils/op.py index cecf69d3..b43a469d 100644 --- a/kraft/utils/op.py +++ b/kraft/utils/op.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # # Authors: Alexander Jung +# Gaulthier Gain # # Copyright (c) 2020, NEC Europe Ltd., NEC Corporation. All rights reserved. # @@ -32,10 +33,33 @@ from __future__ import unicode_literals import os +import stat +import shlex import subprocess from kraft.logger import logger +def which(program): + """Get the path of a program""" + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None + +def execute_command(command, parameters=' '): + """Run a specific command on the host.""" + subprocess.call(shlex.split(command + ' ' + ' '.join(parameters))) + def merge_dicts(x, y): z = x.copy() diff --git a/package/debian/unikraft-tools.links b/package/debian/unikraft-tools.links index 221f47e2..200cab17 100644 --- a/package/debian/unikraft-tools.links +++ b/package/debian/unikraft-tools.links @@ -2,3 +2,5 @@ usr/share/unikraft-tools/bin/kraft usr/bin/kraft usr/share/unikraft-tools/bin/kraft-net usr/bin/kraft-net usr/share/unikraft-tools/bin/qemu-guest usr/bin/qemu-guest usr/share/unikraft-tools/bin/kvm-guest usr/bin/kvm-guest +usr/share/unikraft-tools/bin/kvm-guest usr/bin/kvm-guest +usr/share/unikraft-tools/usr/local/bin/ usr/bin/gotools \ No newline at end of file diff --git a/package/docker/Dockerfile.gotools b/package/docker/Dockerfile.gotools new file mode 100644 index 00000000..7269a8b0 --- /dev/null +++ b/package/docker/Dockerfile.gotools @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Authors: Gaulthier Gain +# +# Copyright (c) 2020, Université de Liège., ULiege. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +ARG GOLANG_VERSION=1.14.4 +ARG SHARED_DIR=/usr/shared + +FROM golang:${GOLANG_VERSION} AS go-build + +LABEL maintainer="Gaulthier Gain " + +RUN mkdir /usr/shared +VOLUME [${SHARED_DIR}] + +RUN mkdir /go/src/tools +COPY gotools/ /go/src/tools +WORKDIR /go/src/tools/srcs +RUN make + +FROM scratch AS gotools + +COPY --from=go-build /go/src/tools/srcs/gotools /usr/local/bin/ \ No newline at end of file diff --git a/package/docker/Dockerfile.kraft b/package/docker/Dockerfile.kraft index f8809f32..b0897ee1 100644 --- a/package/docker/Dockerfile.kraft +++ b/package/docker/Dockerfile.kraft @@ -31,8 +31,10 @@ ARG UK_ARCH=x86_64 ARG GCC_VERSION=9.2.0 +ARG GO_VERSION=1.14.4 FROM unikraft/gcc:${GCC_VERSION}-${UK_ARCH} AS gcc +FROM unikraft/gotools:${GO_VERSION} AS gotools LABEL MAINTAINER="Alexander Jung " @@ -43,6 +45,8 @@ COPY --from=gcc /lib/ /lib COPY --from=gcc /include/ /include COPY --from=gcc /libexec/ /libexec +COPY --from=gotools /usr/local/bin/ /usr/local/bin/ + COPY . /usr/src/kraft WORKDIR /usr/src/unikraft/apps/app diff --git a/package/docker/Makefile b/package/docker/Makefile index 54e8e4c1..e6740718 100644 --- a/package/docker/Makefile +++ b/package/docker/Makefile @@ -101,6 +101,32 @@ else $(DOCKER_BUILD_EXTRA) $(DOCKERDIR) endif +# GO build targets +GO_VERSION ?= 1.14.4 +#CONTRIBDIR ?= $(KRAFTDIR)/contrib +#GOTOOLS ?= $(CONTRIBDIR)/gotools/ + +.PHONY: docker-gotools +docker-gotools: IMAGE_NAME ?= $(ORG)/gotools:$(GO_VERSION) +docker-gotools: + # Copy contrib to avoid forbidden path outside the build context + cp -r $(GOTOOLS) $(DOCKERDIR) + +ifneq (,$(findstring help,$(MAKECMDGOALS))) + @echo "Usage: [IMAGE_NAME=...] $(MAKE) $@ " + @echo "This command build the devel tools which allows developers to build unikernels" + @echo "from existing applications" + @echo +else + $(DOCKER) build \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --tag $(IMAGE_NAME) \ + --cache-from $(ORG)/gotools:$(GO_VERSION) \ + --cache-from $(IMAGE_NAME) \ + --file $(DOCKERDIR)/Dockerfile.gotools \ + $(DOCKER_BUILD_EXTRA) $(DOCKERDIR) +endif + # GCC build targets GCC_VERSION ?= 9.2.0 BINUTILS_VERSION ?= 2.31.1