From 6e05f661f931eed4adc86805f78c8286db5528f2 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 12:14:47 -0400 Subject: [PATCH 01/56] Initialize habitat plan Used command: hab plan init --- habitat/default.toml | 3 + habitat/plan.sh | 264 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 habitat/default.toml create mode 100644 habitat/plan.sh diff --git a/habitat/default.toml b/habitat/default.toml new file mode 100644 index 0000000..03a0530 --- /dev/null +++ b/habitat/default.toml @@ -0,0 +1,3 @@ +# Use this file to templatize your application's native configuration files. +# See the docs at https://www.habitat.sh/docs/create-packages-configure/. +# You can safely delete this file if you don't need it. diff --git a/habitat/plan.sh b/habitat/plan.sh new file mode 100644 index 0000000..c38ed01 --- /dev/null +++ b/habitat/plan.sh @@ -0,0 +1,264 @@ +# This file is the heart of your application's habitat. +# See full docs at https://www.habitat.sh/docs/reference/plan-syntax/ + +# Required. +# Sets the name of the package. This will be used in along with `pkg_origin`, +# and `pkg_version` to define the fully-qualified package name, which determines +# where the package is installed to on disk, how it is referred to in package +# metadata, and so on. +pkg_name=emergence + +# Required unless overridden by the `HAB_ORIGIN` environment variable. +# The origin is used to denote a particular upstream of a package. +pkg_origin=jarvus + +# Required. +# Sets the version of the package. +pkg_version="0.1.0" + +# Optional. +# The name and email address of the package maintainer. +# pkg_maintainer="The Habitat Maintainers " + +# Optional. +# An array of valid software licenses that relate to this package. +# Please choose a license from http://spdx.org/licenses/ +# pkg_license=('Apache-2.0') + +# Required. +# A URL that specifies where to download the source from. Any valid wget url +# will work. Typically, the relative path for the URL is partially constructed +# from the pkg_name and pkg_version values; however, this convention is not +# required. +pkg_source="http://some_source_url/releases/${pkg_name}-${pkg_version}.tar.gz" + +# Optional. +# The resulting filename for the download, typically constructed from the +# pkg_name and pkg_version values. +# pkg_filename="${pkg_name}-${pkg_version}.tar.gz" + +# Required if a valid URL is provided for pkg_source or unless do_verify() is overridden. +# The value for pkg_shasum is a sha-256 sum of the downloaded pkg_source. If you +# do not have the checksum, you can easily generate it by downloading the source +# and using the sha256sum or gsha256sum tools. Also, if you do not have +# do_verify() overridden, and you do not have the correct sha-256 sum, then the +# expected value will be shown in the build output of your package. +pkg_shasum="TODO" + +# Optional. +# An array of package dependencies needed at runtime. You can refer to packages +# at three levels of specificity: `origin/package`, `origin/package/version`, or +# `origin/package/version/release`. +# pkg_deps=(core/glibc) + +# Optional. +# An array of the package dependencies needed only at build time. +# pkg_build_deps=(core/make core/gcc) + +# Optional. +# An array of paths, relative to the final install of the software, where +# libraries can be found. Used to populate LD_FLAGS and LD_RUN_PATH for +# software that depends on your package. +# pkg_lib_dirs=(lib) + +# Optional. +# An array of paths, relative to the final install of the software, where +# headers can be found. Used to populate CFLAGS for software that depends on +# your package. +# pkg_include_dirs=(include) + +# Optional. +# An array of paths, relative to the final install of the software, where +# binaries can be found. Used to populate PATH for software that depends on +# your package. +# pkg_bin_dirs=(bin) + +# Optional. +# An array of paths, relative to the final install of the software, where +# pkg-config metadata (.pc files) can be found. Used to populate +# PKG_CONFIG_PATH for software that depends on your package. +# pkg_pconfig_dirs=(lib/pconfig) + +# Optional. +# The command for the supervisor to execute when starting a service. You can +# omit this setting if your package is not intended to be run directly by a +# supervisor of if your plan contains a run hook in hooks/run. +# pkg_svc_run="bin/haproxy -f $pkg_svc_config_path/haproxy.conf" + +# Optional. +# An associative array representing configuration data which should be gossiped to peers. The keys +# in this array represent the name the value will be assigned and the values represent the toml path +# to read the value. +# pkg_exports=( +# [host]=srv.address +# [port]=srv.port +# [ssl-port]=srv.ssl.port +# ) + +# Optional. +# An array of `pkg_exports` keys containing default values for which ports that this package +# exposes. These values are used as sensible defaults for other tools. For example, when exporting +# a package to a container format. +# pkg_exposes=(port ssl-port) + +# Optional. +# An associative array representing services which you depend on and the configuration keys that +# you expect the service to export (by their `pkg_exports`). These binds *must* be set for the +# supervisor to load the service. The loaded service will wait to run until it's bind becomes +# available. If the bind does not contain the expected keys, the service will not start +# successfully. +# pkg_binds=( +# [database]="port host" +# ) + +# Optional. +# Same as `pkg_binds` but these represent optional services to connect to. +# pkg_binds_optional=( +# [storage]="port host" +# ) + +# Optional. +# An array of interpreters used in shebang lines for scripts. Specify the +# subdirectory where the binary is relative to the package, for example, +# bin/bash or libexec/neverland, since binaries can be located in directories +# besides bin. This list of interpreters will be written to the metadata +# INTERPRETERS file, located inside a package, with their fully-qualified path. +# Then these can be used with the fix_interpreter function. +# pkg_interpreters=(bin/bash) + +# Optional. +# The user to run the service as. The default is hab. +# pkg_svc_user="hab" + +# Optional. +# The group to run the service as. The default is hab. +# pkg_svc_group="$pkg_svc_user" + +# Required for core plans, optional otherwise. +# A short description of the package. It can be a simple string, or you can +# create a multi-line description using markdown to provide a rich description +# of your package. +# pkg_description="Some description." + +# Required for core plans, optional otherwise. +# The project home page for the package. +# pkg_upstream_url="http://example.com/project-name" + + +# Callback Functions +# +# When defining your plan, you have the flexibility to override the default +# behavior of Habitat in each part of the package building stage through a +# series of callbacks. To define a callback, simply create a shell function +# of the same name in your plan.sh file and then write your script. If you do +# not want to use the default callback behavior, you must override the callback +# and return 0 in the function definition. +# +# Callbacks are defined here with either their "do_default_x", if they have a +# default implementation, or empty with "return 0" if they have no default +# implementation (Bash does not allow empty function bodies.) If callbacks do +# nothing or do the same as the default implementation, they can be removed from +# this template. +# +# The default implementations (the do_default_* functions) are defined in the +# plan build script: +# https://github.com/habitat-sh/habitat/tree/master/components/plan-build/bin/hab-plan-build.sh + +# There is no default implementation of this callback. You can use it to execute +# any arbitrary commands before anything else happens. +do_begin() { + return 0 +} + +# The default implementation is that the software specified in $pkg_source is +# downloaded, checksum-verified, and placed in $HAB_CACHE_SRC_PATH/$pkgfilename, +# which resolves to a path like /hab/cache/src/filename.tar.gz. You should +# override this behavior if you need to change how your binary source is +# downloaded, if you are not downloading any source code at all, or if your are +# cloning from git. If you do clone a repo from git, you must override +# do_verify() to return 0. +do_download() { + do_default_download +} + +# The default implementation tries to verify the checksum specified in the plan +# against the computed checksum after downloading the source tarball to disk. +# If the specified checksum doesn't match the computed checksum, then an error +# and a message specifying the mismatch will be printed to stderr. You should +# not need to override this behavior unless your package does not download +# any files. +do_verify() { + do_default_verify +} + +# The default implementation removes the HAB_CACHE_SRC_PATH/$pkg_dirname folder +# in case there was a previously-built version of your package installed on +# disk. This ensures you start with a clean build environment. +do_clean() { + do_default_clean +} + +# The default implementation extracts your tarball source file into +# HAB_CACHE_SRC_PATH. The supported archives are: .tar, .tar.bz2, .tar.gz, +# .tar.xz, .rar, .zip, .Z, .7z. If the file archive could not be found or was +# not supported, then a message will be printed to stderr with additional +# information. +do_unpack() { + do_default_unpack +} + +# There is no default implementation of this callback. At this point in the +# build process, the tarball source has been downloaded, unpacked, and the build +# environment variables have been set, so you can use this callback to perform +# any actions before the package starts building, such as exporting variables, +# adding symlinks, and so on. +do_prepare() { + return 0 +} + +# The default implementation is to update the prefix path for the configure +# script to use $pkg_prefix and then run make to compile the downloaded source. +# This means the script in the default implementation does +# ./configure --prefix=$pkg_prefix && make. You should override this behavior +# if you have additional configuration changes to make or other software to +# build and install as part of building your package. +do_build() { + do_default_build +} + +# The default implementation runs nothing during post-compile. An example of a +# command you might use in this callback is make test. To use this callback, two +# conditions must be true. A) do_check() function has been declared, B) DO_CHECK +# environment variable exists and set to true, env DO_CHECK=true. +do_check() { + return 0 +} + +# The default implementation is to run make install on the source files and +# place the compiled binaries or libraries in HAB_CACHE_SRC_PATH/$pkg_dirname, +# which resolves to a path like /hab/cache/src/packagename-version/. It uses +# this location because of do_build() using the --prefix option when calling the +# configure script. You should override this behavior if you need to perform +# custom installation steps, such as copying files from HAB_CACHE_SRC_PATH to +# specific directories in your package, or installing pre-built binaries into +# your package. +do_install() { + do_default_install +} + +# The default implementation is to strip any binaries in $pkg_prefix of their +# debugging symbols. You should override this behavior if you want to change +# how the binaries are stripped, which additional binaries located in +# subdirectories might also need to be stripped, or whether you do not want the +# binaries stripped at all. +do_strip() { + do_default_strip +} + +# There is no default implementation of this callback. This is called after the +# package has been built and installed. You can use this callback to remove any +# temporary files or perform other post-install clean-up actions. +do_end() { + return 0 +} + From a54c680c1045cccb00d3a2d23fca84eb3d49f02f Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 12:17:55 -0400 Subject: [PATCH 02/56] Enable node scaffolding --- habitat/plan.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/habitat/plan.sh b/habitat/plan.sh index c38ed01..23b3a74 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -144,6 +144,8 @@ pkg_shasum="TODO" # The project home page for the package. # pkg_upstream_url="http://example.com/project-name" +pkg_scaffolding=core/scaffolding-node + # Callback Functions # From 1931095deca5380e9c8ee3a34e973ed19f46d017 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 12:19:41 -0400 Subject: [PATCH 03/56] Remove pkg_source for local build --- habitat/plan.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/habitat/plan.sh b/habitat/plan.sh index 23b3a74..32b0986 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -25,13 +25,6 @@ pkg_version="0.1.0" # Please choose a license from http://spdx.org/licenses/ # pkg_license=('Apache-2.0') -# Required. -# A URL that specifies where to download the source from. Any valid wget url -# will work. Typically, the relative path for the URL is partially constructed -# from the pkg_name and pkg_version values; however, this convention is not -# required. -pkg_source="http://some_source_url/releases/${pkg_name}-${pkg_version}.tar.gz" - # Optional. # The resulting filename for the download, typically constructed from the # pkg_name and pkg_version values. From 90e5a36aa23ca7aaa9aded6952c2a4a8c2d2e37d Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 12:19:56 -0400 Subject: [PATCH 04/56] Set pkg_upstream_url to github --- habitat/plan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/habitat/plan.sh b/habitat/plan.sh index 32b0986..a187805 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -135,7 +135,7 @@ pkg_shasum="TODO" # Required for core plans, optional otherwise. # The project home page for the package. -# pkg_upstream_url="http://example.com/project-name" +pkg_upstream_url="https://github.com/JarvusInnovations/emergence" pkg_scaffolding=core/scaffolding-node From 305eae081689798a43d8c82e60ec5cd50f935cb0 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:29:24 -0400 Subject: [PATCH 05/56] Set package name and origin --- habitat/plan.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/habitat/plan.sh b/habitat/plan.sh index a187805..7046647 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -6,11 +6,11 @@ # and `pkg_version` to define the fully-qualified package name, which determines # where the package is installed to on disk, how it is referred to in package # metadata, and so on. -pkg_name=emergence +pkg_name=emergence-kernel # Required unless overridden by the `HAB_ORIGIN` environment variable. # The origin is used to denote a particular upstream of a package. -pkg_origin=jarvus +pkg_origin=emergence # Required. # Sets the version of the package. From 9d22b78ec742fdc4074dd04d616af4e78f09e016 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:29:50 -0400 Subject: [PATCH 06/56] Add build dependencies --- habitat/plan.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/habitat/plan.sh b/habitat/plan.sh index 7046647..9595ebe 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -46,7 +46,11 @@ pkg_shasum="TODO" # Optional. # An array of the package dependencies needed only at build time. -# pkg_build_deps=(core/make core/gcc) +pkg_build_deps=( + core/make + core/gcc + core/python2 +) # Optional. # An array of paths, relative to the final install of the software, where From 32f81cb0f05968a7dcf38c7d7495d35e5b061a7f Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:30:07 -0400 Subject: [PATCH 07/56] Add core services and runtime dependencies --- habitat/plan.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/habitat/plan.sh b/habitat/plan.sh index 9595ebe..e8be028 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -42,7 +42,11 @@ pkg_shasum="TODO" # An array of package dependencies needed at runtime. You can refer to packages # at three levels of specificity: `origin/package`, `origin/package/version`, or # `origin/package/version/release`. -# pkg_deps=(core/glibc) +pkg_deps=( + core/mariadb + core/php + core/nginx +) # Optional. # An array of the package dependencies needed only at build time. From b5e283e8ffe01fb7a73d5598fe1a0a50d1c85cd0 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:30:24 -0400 Subject: [PATCH 08/56] Add habitat results/ directory to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a4c24ac..0a1877f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /node_modules/ .DS_Store -/typings/ \ No newline at end of file +/typings/ +/results/ From 178877972bb086c064214d02c65899bcec56c524 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:30:44 -0400 Subject: [PATCH 09/56] Configure node engine version --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index 01ebf42..77d1335 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,9 @@ "preferGlobal": true, "version": "0.0.3", "license": "MIT", + "engines": { + "node": "6.10.3" + }, "dependencies": { "underscore": "1.3.x", "mariasql": "^0.2.6", From 992cc010ec71912db471ab6d684cc861cf0496ed Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:31:07 -0400 Subject: [PATCH 10/56] Bump kernel package version to 1.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 77d1335..b52a34e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "emergence", "preferGlobal": true, - "version": "0.0.3", + "version": "1.0.0", "license": "MIT", "engines": { "node": "6.10.3" From 1f88be9b544d4d08349e7476a1fa6485f0329b75 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:31:22 -0400 Subject: [PATCH 11/56] Configure npm start script --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index b52a34e..cebbe24 100644 --- a/package.json +++ b/package.json @@ -28,5 +28,8 @@ "repository": { "type": "git", "url": "https://github.com/JarvusInnovations/Emergence.git" + }, + "scripts": { + "start": "node bin/kernel" } } From 550d6978989e814b1ab9702afbd3fdaccf4d1e2c Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 13:54:40 -0400 Subject: [PATCH 12/56] Add basic config for kernel --- habitat/config/config.json | 23 +++++++++++++++++++++++ habitat/default.toml | 9 +++++++++ 2 files changed, 32 insertions(+) create mode 100644 habitat/config/config.json diff --git a/habitat/config/config.json b/habitat/config/config.json new file mode 100644 index 0000000..c3bb842 --- /dev/null +++ b/habitat/config/config.json @@ -0,0 +1,23 @@ +{ + "user": "{{pkg.svc_user}}", + "group": "{{pkg.svc_group}}", + "server": { + "host": "{{cfg.kernel.host}}" + }, + "services": { + "web": { + "type": "nginx", + "execPath": "{{pkgPathFor "core/nginx"}}/bin/nginx", + "bindHost": "{{cfg.web.host}}" + }, + "sql": { + "type": "mariadb", + "execPath": "{{pkgPathFor "core/mariadb"}}/bin/mysqld", + "bindHost": "{{cfg.sql.host}}" + }, + "php": { + "type": "php-fpm", + "execPath": "{{pkgPathFor "core/php"}}/sbin/php-fpm" + } + } +} \ No newline at end of file diff --git a/habitat/default.toml b/habitat/default.toml index 03a0530..435aa58 100644 --- a/habitat/default.toml +++ b/habitat/default.toml @@ -1,3 +1,12 @@ # Use this file to templatize your application's native configuration files. # See the docs at https://www.habitat.sh/docs/create-packages-configure/. # You can safely delete this file if you don't need it. + +[kernel] +host="0.0.0.0" + +[web] +host="0.0.0.0" + +[sql] +host="127.0.0.1" \ No newline at end of file From 99b2ee44053f6a246c93a5053323d3bb50be864d Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 18:36:16 -0400 Subject: [PATCH 13/56] Copy php plan from habitat core-plans --- service-plans/php/plan.sh | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 service-plans/php/plan.sh diff --git a/service-plans/php/plan.sh b/service-plans/php/plan.sh new file mode 100644 index 0000000..edac8f5 --- /dev/null +++ b/service-plans/php/plan.sh @@ -0,0 +1,67 @@ +pkg_name=php +pkg_distname=php +pkg_origin=core +pkg_version=7.1.4 +pkg_maintainer="The Habitat Maintainers " +pkg_license=('PHP-3.01') +pkg_upstream_url=http://php.net/ +pkg_description="PHP is a popular general-purpose scripting language that is especially suited to web development." +pkg_source=https://php.net/get/${pkg_distname}-${pkg_version}.tar.bz2/from/this/mirror +pkg_filename=${pkg_distname}-${pkg_version}.tar.bz2 +pkg_dirname=${pkg_distname}-${pkg_version} +pkg_shasum=39bf697836e2760b3a44ea322e9e5f1f5b1f07abeb0111f6495eff7538e25805 +pkg_deps=( + core/coreutils + core/curl + core/glibc + core/libxml2 + core/libjpeg-turbo + core/libpng + core/openssl + core/zlib +) +pkg_build_deps=( + core/bison2 + core/gcc + core/make + core/re2c +) +pkg_bin_dirs=(bin sbin) +pkg_lib_dirs=(lib) +pkg_include_dirs=(include) +pkg_interpreters=(bin/php) + +do_build() { + ./configure --prefix="$pkg_prefix" \ + --enable-exif \ + --enable-fpm \ + --with-fpm-user=hab \ + --with-fpm-group=hab \ + --enable-mbstring \ + --enable-opcache \ + --with-mysql=mysqlnd \ + --with-mysqli=mysqlnd \ + --with-pdo-mysql=mysqlnd \ + --with-curl="$(pkg_path_for curl)" \ + --with-gd \ + --with-jpeg-dir="$(pkg_path_for libjpeg-turbo)" \ + --with-libxml-dir="$(pkg_path_for libxml2)" \ + --with-openssl="$(pkg_path_for openssl)" \ + --with-png-dir="$(pkg_path_for libpng)" \ + --with-xmlrpc \ + --with-zlib="$(pkg_path_for zlib)" + make +} + +do_install() { + do_default_install + + # Modify PHP-FPM config so it will be able to run out of the box. To run a real + # PHP-FPM application you would want to supply your own config with + # --fpm-config . + mv "$pkg_prefix/etc/php-fpm.conf.default" "$pkg_prefix/etc/php-fpm.conf" +} + +do_check() { + make test +} From d3a44730f33161807e4d1dc1917560b0ff66b303 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 18:36:50 -0400 Subject: [PATCH 14/56] Compile apcu statically into php --- service-plans/php/plan.sh | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/service-plans/php/plan.sh b/service-plans/php/plan.sh index edac8f5..8357536 100644 --- a/service-plans/php/plan.sh +++ b/service-plans/php/plan.sh @@ -21,6 +21,7 @@ pkg_deps=( core/zlib ) pkg_build_deps=( + core/autoconf core/bison2 core/gcc core/make @@ -31,15 +32,43 @@ pkg_lib_dirs=(lib) pkg_include_dirs=(include) pkg_interpreters=(bin/php) +apcu_version=5.1.8 +apcu_source=https://github.com/krakjoe/apcu/archive/v${apcu_version}.tar.gz +apcu_filename=apcu-${apcu_version}.tar.gz +apcu_shasum=09848619674a0871053cabba3907d2aade395772d54464d3aee45f519e217128 +apcu_dirname=apcu-${apcu_version} + +do_download() { + do_default_download + + download_file $apcu_source $apcu_filename $apcu_shasum +} + +do_verify() { + do_default_verify + + verify_file $apcu_filename $apcu_shasum +} + +do_unpack() { + do_default_unpack + + unpack_file $apcu_filename + mv "$HAB_CACHE_SRC_PATH/$apcu_dirname" "$HAB_CACHE_SRC_PATH/$pkg_dirname/ext/apcu" +} + do_build() { + rm aclocal.m4 + ./buildconf --force + ./configure --prefix="$pkg_prefix" \ --enable-exif \ --enable-fpm \ --with-fpm-user=hab \ --with-fpm-group=hab \ + --enable-apcu \ --enable-mbstring \ --enable-opcache \ - --with-mysql=mysqlnd \ --with-mysqli=mysqlnd \ --with-pdo-mysql=mysqlnd \ --with-curl="$(pkg_path_for curl)" \ From 5b325934cd9d63f2b0ecff8252a7b544bb099259 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 18:47:07 -0400 Subject: [PATCH 15/56] Add service-plans results to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a1877f..32b9048 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .DS_Store /typings/ /results/ +/service-plans/*/results/ \ No newline at end of file From 1285533537f0a1e0b935f7877c72f3f50990b2a9 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 18:48:23 -0400 Subject: [PATCH 16/56] Change php origin to emergence --- service-plans/php/plan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service-plans/php/plan.sh b/service-plans/php/plan.sh index 8357536..9eb0167 100644 --- a/service-plans/php/plan.sh +++ b/service-plans/php/plan.sh @@ -1,6 +1,6 @@ pkg_name=php pkg_distname=php -pkg_origin=core +pkg_origin=emergence pkg_version=7.1.4 pkg_maintainer="The Habitat Maintainers " pkg_license=('PHP-3.01') From 843141643132d040e3c4701f2dd21290e5dfe098 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 19:51:57 -0400 Subject: [PATCH 17/56] Copy mariadb plan from habitat core-plans --- kernel-lib/services/mysql.js | 421 -------------------------- service-plans/mariadb/config/init.sql | 6 + service-plans/mariadb/config/my.cnf | 23 ++ service-plans/mariadb/default.toml | 4 + service-plans/mariadb/hooks/init | 14 + service-plans/mariadb/hooks/run | 7 + service-plans/mariadb/plan.sh | 43 +++ 7 files changed, 97 insertions(+), 421 deletions(-) delete mode 100644 kernel-lib/services/mysql.js create mode 100644 service-plans/mariadb/config/init.sql create mode 100644 service-plans/mariadb/config/my.cnf create mode 100644 service-plans/mariadb/default.toml create mode 100644 service-plans/mariadb/hooks/init create mode 100644 service-plans/mariadb/hooks/run create mode 100644 service-plans/mariadb/plan.sh diff --git a/kernel-lib/services/mysql.js b/kernel-lib/services/mysql.js deleted file mode 100644 index ae462e2..0000000 --- a/kernel-lib/services/mysql.js +++ /dev/null @@ -1,421 +0,0 @@ -var _ = require('underscore'), - fs = require('fs'), - path = require('path'), - util = require('util'), - spawn = require('child_process').spawn, - exec = require('child_process').exec, - shell = require('shelljs'), - semver = require('semver'), - mariasql = require('mariasql'); - -exports.createService = function(name, controller, options) { - return new exports.MysqlService(name, controller, options); -}; - -exports.MysqlService = function(name, controller, options) { - var me = this, - versionMatch; - - // call parent constructor - exports.MysqlService.super_.apply(me, arguments); - - // default options - me.options.configPath = me.options.configPath || controller.options.configDir + '/my.cnf'; - me.options.execPath = me.options.execPath || '/usr/sbin/mysqld'; - me.options.bindHost = me.options.bindHost || false; - me.options.runDir = me.options.runDir || controller.options.runDir + '/mysqld'; - me.options.pidPath = me.options.pidPath || me.options.runDir + '/mysqld.pid'; - me.options.socketPath = me.options.socketPath || me.options.runDir + '/mysqld.sock'; - me.options.dataDir = me.options.dataDir || controller.options.dataDir + '/mysql'; - me.options.logsDir = me.options.logsDir || controller.options.logsDir + '/mysql'; - me.options.errorLogPath = me.options.errorLogPath || me.options.logsDir + '/mysqld.err'; - me.options.managerUser = me.options.managerUser || 'emergence'; - me.options.managerPassword = me.options.managerPassword || ''; - - - // verify binary - if (!fs.existsSync(me.options.execPath)) { - throw 'execPath not found: ' + me.options.execPath; - } - - // check binary version - console.log(me.name+': detecting mysqld version...'); - versionMatch = shell.exec(me.options.execPath+' --version').output.match(/mysqld\s+Ver\s+(\d+(\.\d+)*)(-MariaDB)?/); - - if (!versionMatch) { - throw 'Failed to detect mysql version'; - } - - me.mysqldVersion = versionMatch[1]; - me.mysqldIsMaria = versionMatch[3] == '-MariaDB'; - console.log('%s: determined mysqld version: %s', me.name, me.mysqldVersion + (me.mysqldIsMaria ? ' (MariaDB)' : '')); - - // check for existing mysqld process - if (fs.existsSync(me.options.pidPath)) { - me.pid = parseInt(fs.readFileSync(me.options.pidPath, 'ascii')); - console.log(me.name+': found existing PID: '+me.pid+', checking /proc/'+me.pid); - - if (fs.existsSync('/proc/'+me.pid)) { - me.status = 'online'; - - // instantiate MySQL client - me.client = new mariasql({ - unixSocket: me.options.socketPath, - user: me.options.managerUser, - password: me.options.managerPassword, - multiStatements: true - }); - } else { - console.log(me.name+': process '+me.pid + ' not found, deleting .pid file'); - fs.unlinkSync(me.options.pidPath); - } - } - - // listen for site creation - controller.sites.on('siteCreated', _.bind(me.onSiteCreated, me)); -}; - -util.inherits(exports.MysqlService, require('./abstract.js').AbstractService); - - - -exports.MysqlService.prototype.start = function(firstRun) { - var me = this; - - if (me.pid) { - console.log(me.name+': mysql already runnig with PID '+me.pid); - return false; - } - - // write configuration file - this.writeConfig(); - - // init logs directory if needed - if (!fs.existsSync(me.options.logsDir)) { - console.log(me.name+': initializing new log directory'); - fs.mkdirSync(me.options.logsDir, '775'); - exec('chown -R mysql:mysql '+me.options.logsDir); - } - - - // init run directory if needed - if (!fs.existsSync(me.options.runDir)) { - console.log(me.name+': initializing new run directory'); - fs.mkdirSync(me.options.runDir, '775'); - exec('chown -R mysql:mysql '+me.options.runDir); - } - - // init data directory if needed - if (!fs.existsSync(me.options.dataDir)) { - console.log(me.name+': initializing new data directory...'); - fs.mkdirSync(me.options.dataDir, '775'); - exec('chown -R mysql:mysql '+me.options.dataDir); - - if (semver.lt(me.mysqldVersion, '5.7.6') || me.mysqldIsMaria) { - exec('mysql_install_db --defaults-file='+me.options.configPath, function(error, stdout, stderr) { - me.start(true); - }); - } else { - exec('mysqld --initialize-insecure --user=mysql --datadir='+me.options.dataDir, function(error, stdout, stderr) { - me.start(true); - }); - } - - me.status = 'configuring'; - return true; // not really started, we have to try again after mysql_install_db is done - } - - // instantiate MySQL client - me.client = new mariasql({ - unixSocket: me.options.socketPath, - user: me.options.managerUser, - password: me.options.managerPassword, - multiStatements: true - }); - - // spawn process - console.log(me.name+': spawning mysql: '+me.options.execPath); - me.proc = spawn(me.options.execPath, ['--defaults-file='+me.options.configPath, '--console'], {detached: true}); - me.pid = me.proc.pid; - me.status = 'online'; - - console.log(me.name+': spawned mysqld with pid '+me.pid); - - // add listeners to process - me.proc.on('exit', function (code) { - - if (code !== 0) { - me.status = 'offline'; - me.exitCode = code; - console.log(me.name+': exited with code: '+code); - } - }); - - me.proc.stdout.on('data', function (data) { - console.log(me.name+': stdout:\n\t' + data.toString().replace(/\n/g,'\n\t')); - }); - - me.proc.stderr.on('data', function (data) { - console.log(me.name+': stderr:\n\t' + data.toString().replace(/\n/g,'\n\t')); - - if (/^execvp\(\)/.test(data)) { - console.log('Failed to start child process.'); - me.status = 'offline'; - } - - if (/ready for connections/.test(data) && firstRun) { - me.secureInstallation(); - } - }); - - return true; -}; - - -exports.MysqlService.prototype.stop = function() { - var me = this; - - if (!me.pid) { - return false; - } - - // disconnect client - if (me.client && me.client.connected) { - me.client.end(); - console.log(me.name+': mysql client disconnected'); - } - - try { - console.log(me.name+': sending sigterm to '+me.pid); - process.kill(me.pid, 'SIGTERM'); - } catch (error) { - console.log(me.name+': failed to stop process: '+error); - return false; - } - - me.status = 'offline'; - me.pid = null; - return true; -}; - -exports.MysqlService.prototype.restart = function() { - var me = this, - now; - - if (!me.stop()) { - return false; - } - - // wait for pid to disappear before attempting start - process.stdout.write(me.name+': waiting for shutdown'); - while (fs.existsSync(me.options.pidPath)) { - process.stdout.write('.'); - now = new Date().getTime(); - - while (new Date().getTime() < now + 500) { - // do nothing - } - } - - process.stdout.write('\n'); - - return me.start(); -}; - -exports.MysqlService.prototype.writeConfig = function() { - fs.writeFileSync(this.options.configPath, this.makeConfig()); -}; - -exports.MysqlService.prototype.makeConfig = function() { - var me = this, - config = []; - - config.push( - '[mysqld]', - 'character-set-server = utf8', - 'user = mysql', - 'port = 3306', - 'socket = '+me.options.socketPath, - 'pid-file = '+me.options.pidPath, -// 'log-error = '+me.options.errorLogPath, // disabled due to http://bugs.mysql.com/bug.php?id=65592 -- errors output to STDIN will usually go into emergence-kernel's log - 'basedir = /usr', - 'datadir = '+me.options.dataDir, - 'skip-external-locking', - 'key_buffer_size = 16M', - 'max_allowed_packet = 1M', - 'sort_buffer_size = 512K', - 'net_buffer_length = 8K', - 'read_buffer_size = 256K', - 'read_rnd_buffer_size = 512K', - 'myisam_sort_buffer_size = 8M', -// 'lc-messages-dir = /usr/local/share/mysql', - - 'log-bin = mysqld-bin', - 'expire_logs_days = 2', - 'server-id = 1', - - 'tmpdir = /tmp/', - - 'innodb_buffer_pool_size = 16M', - 'innodb_data_file_path = ibdata1:10M:autoextend:max:128M', - 'innodb_log_file_size = 5M', - 'innodb_log_buffer_size = 8M', - 'innodb_log_files_in_group = 2', - 'innodb_flush_log_at_trx_commit = 1', - 'innodb_lock_wait_timeout = 50', - 'innodb_file_per_table', - 'max_binlog_size = 100M', - 'binlog_format = row' - ); - - if (semver.gt(me.mysqldVersion, '5.6.0')) { - config.push('table_open_cache = 64'); - } else { - config.push('table_cache = 64'); - } - - if (semver.lt(me.mysqldVersion, '5.7.4')) { - config.push('innodb_additional_mem_pool_size = 2M'); - } - - if (me.options.bindHost) { - config.push('bind-address = '+me.options.bindHost); - } else { - config.push('skip-networking'); - } - - return config.join('\n'); -}; - -exports.MysqlService.prototype.secureInstallation = function() { - var me = this, - sql = ''; - - console.log(me.name+': securing installation...'); - - // set root password - if (semver.lt(me.mysqldVersion, '5.7.0') || me.mysqldIsMaria) { - sql += 'UPDATE mysql.user SET Password=PASSWORD("'+me.options.managerPassword+'") WHERE User="root";'; - } else { - sql += 'UPDATE mysql.user SET authentication_string=PASSWORD("'+me.options.managerPassword+'") WHERE User="root";'; - } - - // remove anonymous users - sql += 'DELETE FROM mysql.user WHERE User="";'; - - // delete remote roots - sql += 'DELETE FROM mysql.user WHERE User="root" AND Host NOT IN ("localhost", "127.0.0.1", "::1");'; - - // remove test database - sql += 'DROP DATABASE IF EXISTS test;'; - sql += 'DELETE FROM mysql.db WHERE Db="test" OR Db="test\\_%";'; - - // reload privs - sql += 'FLUSH PRIVILEGES;'; - - // open a temporary connection to the new non-secured installation - (new mariasql({ - unixSocket: me.options.socketPath, - user: 'root', - password: '', - multiStatements: true - })).query(sql, function(error) { - if (error) { - console.log(me.name+': failed to secure installation: ' + error); - } else { - console.log(me.name+': securing complete, mysql ready.'); - } - }); - -}; - - -exports.MysqlService.prototype.onSiteCreated = function(siteData, requestData, callbacks) { - var me = this, - sql = '', - dbConfig = { - socket: me.options.socketPath, - database: siteData.handle, - username: siteData.handle, - password: me.controller.sites.generatePassword() - }; - - console.log(me.name+': creating database `'+siteData.handle+'`'); - - sql += 'CREATE DATABASE IF NOT EXISTS `'+siteData.handle+'`;'; - sql += 'CREATE USER \''+siteData.handle+'\'@\'localhost\' IDENTIFIED BY \''+dbConfig.password+'\';'; - sql += 'GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER, LOCK TABLES ON `'+siteData.handle+'`.* TO \''+siteData.handle+'\'@\'localhost\';'; - sql += 'FLUSH PRIVILEGES;'; - - me.client.query(sql, function(error, results) { - if (error) { - console.log(me.name+': failed to setup database `'+siteData.handle+'`: '+error); - return; - } - - console.log(me.name+': database setup complete'); - me.controller.sites.updateSiteConfig(siteData.handle, { - mysql: dbConfig - }); - - // populate tables - me.createSkeletonTables(siteData, function() { - if (callbacks.databaseReady) { - callbacks.databaseReady(dbConfig, siteData, requestData); - } - }); - }); -}; - - - -exports.MysqlService.prototype.createSkeletonTables = function(siteData, callback) { - var me = this, - sql = ''; - - sql += 'USE `'+siteData.handle+'`;'; - - // Table: _e_file_collections - sql += 'CREATE TABLE `_e_file_collections` ('; - sql += '`ID` int(10) unsigned NOT NULL AUTO_INCREMENT'; - sql += ',`Site` ENUM(\'Local\',\'Remote\') NOT NULL'; - sql += ',`Handle` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL'; - sql += ',`Status` enum(\'Normal\',\'Deleted\') NOT NULL DEFAULT \'Normal\''; - sql += ',`Created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP'; - sql += ',`CreatorID` int(10) unsigned DEFAULT NULL'; - sql += ',`ParentID` int(10) unsigned DEFAULT NULL'; - sql += ',`PosLeft` int(10) unsigned DEFAULT NULL'; - sql += ',`PosRight` int(10) unsigned DEFAULT NULL'; - sql += ',PRIMARY KEY (`ID`)'; - sql += ',UNIQUE KEY `PosLeft` (`PosLeft`)'; - sql += ',UNIQUE KEY `SiteCollection` (`Site`,`ParentID`,`Handle`,`Status`)'; - sql += ') ENGINE=MyISAM DEFAULT CHARSET=utf8;'; - - // Table: _e_files - sql += 'CREATE TABLE `_e_files` ('; - sql += '`ID` int(10) unsigned NOT NULL AUTO_INCREMENT'; - sql += ',`CollectionID` int(10) unsigned NOT NULL'; - sql += ',`Handle` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL'; - sql += ',`Status` enum(\'Phantom\',\'Normal\',\'Deleted\') NOT NULL DEFAULT \'Phantom\''; - sql += ',`SHA1` char(40) DEFAULT NULL'; - sql += ',`Size` int(10) unsigned DEFAULT NULL'; - sql += ',`Type` varchar(255) DEFAULT NULL'; - sql += ',`Timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP'; - sql += ',`AuthorID` int(10) unsigned DEFAULT NULL'; - sql += ',`AncestorID` int(10) unsigned DEFAULT NULL'; - sql += ',PRIMARY KEY (`ID`)'; - sql += ',KEY `CollectionID` (`CollectionID`)'; - sql += ') ENGINE=MyISAM DEFAULT CHARSET=utf8;'; - - // run tables - me.client.query(sql, function(error, results) { - if (error) { - console.log(me.name+': failed to setup skeleton tables on `'+siteData.handle+'`: '+error); - return; - } - - console.log(me.name+': skeleton table schema setup'); - - callback(); - }); -}; diff --git a/service-plans/mariadb/config/init.sql b/service-plans/mariadb/config/init.sql new file mode 100644 index 0000000..8c34743 --- /dev/null +++ b/service-plans/mariadb/config/init.sql @@ -0,0 +1,6 @@ +UPDATE mysql.user SET Password=PASSWORD('{{cfg.root_password}}') WHERE User='root'; +DELETE FROM mysql.user WHERE User=''; +DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1'); +DROP DATABASE IF EXISTS test; +DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%'; +FLUSH PRIVILEGES; \ No newline at end of file diff --git a/service-plans/mariadb/config/my.cnf b/service-plans/mariadb/config/my.cnf new file mode 100644 index 0000000..b03580e --- /dev/null +++ b/service-plans/mariadb/config/my.cnf @@ -0,0 +1,23 @@ +[mysqld_safe] +pid-file = {{pkg.svc_var_path}}/mysqld.pid +socket = {{pkg.svc_var_path}}/mysqld.sock +nice = 0 + +[mysqld] +port = {{cfg.port}} +socket = {{pkg.svc_var_path}}/mysqld.sock +datadir = {{pkg.svc_data_path}} +pid-file = {{pkg.svc_var_path}}/mysqld.pid +log-error = mysqld-err.log +log-isam = mysqld-isam.log +log-tc = mysqld-tc.log +aria-log-dir-path = {{pkg.svc_data_path}}/mysqld-aria +master-info-file = mysqld-master.info +{{~#if cfg.general_log}} +general-log = TRUE +general-log-file = {{pkg.svc_var_path}}/mysqld.log +{{~/if}} +{{~#if cfg.bind_address}} +bind-address = {{cfg.bind_address}} +{{~/if}} +init-file = {{pkg.svc_config_path}}/init.sql \ No newline at end of file diff --git a/service-plans/mariadb/default.toml b/service-plans/mariadb/default.toml new file mode 100644 index 0000000..52a153a --- /dev/null +++ b/service-plans/mariadb/default.toml @@ -0,0 +1,4 @@ +root_password = "" +port = 3306 +bind_address = "127.0.0.1" +general_log = true \ No newline at end of file diff --git a/service-plans/mariadb/hooks/init b/service-plans/mariadb/hooks/init new file mode 100644 index 0000000..e5a3270 --- /dev/null +++ b/service-plans/mariadb/hooks/init @@ -0,0 +1,14 @@ +#!/bin/sh + +exec 2>&1 + +#mariadb aria files folder +mkdir -p {{pkg.svc_data_path}}/mysqld-aria + +#mariadb needs to create 'mysql' database +if [ ! -f {{pkg.svc_files_path}}/db_installed ]; then + cd {{pkg.path}} && \ + scripts/mysql_install_db --defaults-file={{pkg.svc_config_path}}/my.cnf \ + --user={{pkg.svc_user}} > \ + {{pkg.svc_files_path}}/db_installed +fi \ No newline at end of file diff --git a/service-plans/mariadb/hooks/run b/service-plans/mariadb/hooks/run new file mode 100644 index 0000000..fffcd13 --- /dev/null +++ b/service-plans/mariadb/hooks/run @@ -0,0 +1,7 @@ +#!/bin/sh + +exec 2>&1 + +#start the mysql server +mysqld_safe --defaults-file={{pkg.svc_config_path}}/my.cnf \ + --user={{pkg.svc_user}} \ No newline at end of file diff --git a/service-plans/mariadb/plan.sh b/service-plans/mariadb/plan.sh new file mode 100644 index 0000000..18d0d23 --- /dev/null +++ b/service-plans/mariadb/plan.sh @@ -0,0 +1,43 @@ +pkg_name=mariadb +pkg_origin=core +pkg_version=10.1.18 +pkg_description="An open source monitoring software for networks and applications" +pkg_maintainer="The Habitat Maintainers " +pkg_license=('GPL-2.0') +pkg_source=http://ftp.hosteurope.de/mirror/archive.mariadb.org//${pkg_name}-${pkg_version}/source/${pkg_name}-${pkg_version}.tar.gz +pkg_shasum=d7336907e9ff44496d6453f92526b25bd253638a64a051ca879f953499873b73 +pkg_deps=(core/ncurses core/gcc-libs core/zlib) +pkg_build_deps=(core/gcc core/make core/coreutils core/cmake) +pkg_bin_dirs=(bin) +pkg_include_dirs=(include) +pkg_lib_dirs=(lib) +pkg_exports=( + [port]=port +) +pkg_exposes=(port) +pkg_svc_user="hab" + +do_prepare() { + if [ -f CMakeCache.txt ]; then + rm CMakeCache.txt + fi + + sed -i 's/^.*abi_check.*$/#/' CMakeLists.txt + sed -i "s@data/test@\${INSTALL_MYSQLTESTDIR}@g" sql/CMakeLists.txt + export CXXFLAGS="$CFLAGS" +} + +do_build() { + cmake . -DCMAKE_INSTALL_PREFIX="${pkg_prefix}" \ + -DCMAKE_PREFIX_PATH="$(pkg_path_for core/ncurses)" \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_READLINE=OFF + make +} + +do_install() { + make install + rm -rf "${pkg_prefix}/mysql-test" + rm -rf "${pkg_prefix}/bin/mysql_client_test" + rm -rf "${pkg_prefix}/bin/mysql_test" +} From c211143972fdf5b10040f6fadbf574ac330a536a Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 19:52:34 -0400 Subject: [PATCH 18/56] Change mariadb origin to emergence --- service-plans/mariadb/plan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service-plans/mariadb/plan.sh b/service-plans/mariadb/plan.sh index 18d0d23..5b98328 100644 --- a/service-plans/mariadb/plan.sh +++ b/service-plans/mariadb/plan.sh @@ -1,5 +1,5 @@ pkg_name=mariadb -pkg_origin=core +pkg_origin=emergence pkg_version=10.1.18 pkg_description="An open source monitoring software for networks and applications" pkg_maintainer="The Habitat Maintainers " From 8e549ad3160ba34dcdbb1e0fc63920980da9c4b1 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 27 May 2017 19:52:50 -0400 Subject: [PATCH 19/56] Add sed to mariadb runtime requirements for mysql_install_db --- service-plans/mariadb/plan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service-plans/mariadb/plan.sh b/service-plans/mariadb/plan.sh index 5b98328..8b18464 100644 --- a/service-plans/mariadb/plan.sh +++ b/service-plans/mariadb/plan.sh @@ -6,7 +6,7 @@ pkg_maintainer="The Habitat Maintainers " pkg_license=('GPL-2.0') pkg_source=http://ftp.hosteurope.de/mirror/archive.mariadb.org//${pkg_name}-${pkg_version}/source/${pkg_name}-${pkg_version}.tar.gz pkg_shasum=d7336907e9ff44496d6453f92526b25bd253638a64a051ca879f953499873b73 -pkg_deps=(core/ncurses core/gcc-libs core/zlib) +pkg_deps=(core/ncurses core/gcc-libs core/zlib core/sed) pkg_build_deps=(core/gcc core/make core/coreutils core/cmake) pkg_bin_dirs=(bin) pkg_include_dirs=(include) From 1962e15babc5c1982044fff62095e56bc7a01b73 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 10:03:27 -0400 Subject: [PATCH 20/56] Copy nginx plan from habitat core-plans --- service-plans/nginx/config/fastcgi.conf | 25 +++++ service-plans/nginx/config/fastcgi_params | 24 +++++ service-plans/nginx/config/koi-utf | 109 +++++++++++++++++++ service-plans/nginx/config/koi-win | 103 ++++++++++++++++++ service-plans/nginx/config/mime.types | 89 +++++++++++++++ service-plans/nginx/config/nginx.conf | 34 ++++++ service-plans/nginx/config/scgi_params | 16 +++ service-plans/nginx/config/uwsgi_params | 16 +++ service-plans/nginx/config/win-utf | 125 ++++++++++++++++++++++ service-plans/nginx/default.toml | 38 +++++++ service-plans/nginx/plan.sh | 66 ++++++++++++ 11 files changed, 645 insertions(+) create mode 100644 service-plans/nginx/config/fastcgi.conf create mode 100644 service-plans/nginx/config/fastcgi_params create mode 100644 service-plans/nginx/config/koi-utf create mode 100644 service-plans/nginx/config/koi-win create mode 100644 service-plans/nginx/config/mime.types create mode 100644 service-plans/nginx/config/nginx.conf create mode 100644 service-plans/nginx/config/scgi_params create mode 100644 service-plans/nginx/config/uwsgi_params create mode 100644 service-plans/nginx/config/win-utf create mode 100644 service-plans/nginx/default.toml create mode 100644 service-plans/nginx/plan.sh diff --git a/service-plans/nginx/config/fastcgi.conf b/service-plans/nginx/config/fastcgi.conf new file mode 100644 index 0000000..ac9ff92 --- /dev/null +++ b/service-plans/nginx/config/fastcgi.conf @@ -0,0 +1,25 @@ + +fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; +fastcgi_param QUERY_STRING $query_string; +fastcgi_param REQUEST_METHOD $request_method; +fastcgi_param CONTENT_TYPE $content_type; +fastcgi_param CONTENT_LENGTH $content_length; + +fastcgi_param SCRIPT_NAME $fastcgi_script_name; +fastcgi_param REQUEST_URI $request_uri; +fastcgi_param DOCUMENT_URI $document_uri; +fastcgi_param DOCUMENT_ROOT $document_root; +fastcgi_param SERVER_PROTOCOL $server_protocol; +fastcgi_param HTTPS $https if_not_empty; + +fastcgi_param GATEWAY_INTERFACE CGI/1.1; +fastcgi_param SERVER_SOFTWARE nginx/$nginx_version; + +fastcgi_param REMOTE_ADDR $remote_addr; +fastcgi_param REMOTE_PORT $remote_port; +fastcgi_param SERVER_ADDR $server_addr; +fastcgi_param SERVER_PORT $server_port; +fastcgi_param SERVER_NAME $server_name; + +# PHP only, required if PHP was built with --enable-force-cgi-redirect +fastcgi_param REDIRECT_STATUS 200; diff --git a/service-plans/nginx/config/fastcgi_params b/service-plans/nginx/config/fastcgi_params new file mode 100644 index 0000000..71e2c2e --- /dev/null +++ b/service-plans/nginx/config/fastcgi_params @@ -0,0 +1,24 @@ + +fastcgi_param QUERY_STRING $query_string; +fastcgi_param REQUEST_METHOD $request_method; +fastcgi_param CONTENT_TYPE $content_type; +fastcgi_param CONTENT_LENGTH $content_length; + +fastcgi_param SCRIPT_NAME $fastcgi_script_name; +fastcgi_param REQUEST_URI $request_uri; +fastcgi_param DOCUMENT_URI $document_uri; +fastcgi_param DOCUMENT_ROOT $document_root; +fastcgi_param SERVER_PROTOCOL $server_protocol; +fastcgi_param HTTPS $https if_not_empty; + +fastcgi_param GATEWAY_INTERFACE CGI/1.1; +fastcgi_param SERVER_SOFTWARE nginx/$nginx_version; + +fastcgi_param REMOTE_ADDR $remote_addr; +fastcgi_param REMOTE_PORT $remote_port; +fastcgi_param SERVER_ADDR $server_addr; +fastcgi_param SERVER_PORT $server_port; +fastcgi_param SERVER_NAME $server_name; + +# PHP only, required if PHP was built with --enable-force-cgi-redirect +fastcgi_param REDIRECT_STATUS 200; diff --git a/service-plans/nginx/config/koi-utf b/service-plans/nginx/config/koi-utf new file mode 100644 index 0000000..e7974ff --- /dev/null +++ b/service-plans/nginx/config/koi-utf @@ -0,0 +1,109 @@ + +# This map is not a full koi8-r <> utf8 map: it does not contain +# box-drawing and some other characters. Besides this map contains +# several koi8-u and Byelorussian letters which are not in koi8-r. +# If you need a full and standard map, use contrib/unicode2nginx/koi-utf +# map instead. + +charset_map koi8-r utf-8 { + + 80 E282AC ; # euro + + 95 E280A2 ; # bullet + + 9A C2A0 ; #   + + 9E C2B7 ; # · + + A3 D191 ; # small yo + A4 D194 ; # small Ukrainian ye + + A6 D196 ; # small Ukrainian i + A7 D197 ; # small Ukrainian yi + + AD D291 ; # small Ukrainian soft g + AE D19E ; # small Byelorussian short u + + B0 C2B0 ; # ° + + B3 D081 ; # capital YO + B4 D084 ; # capital Ukrainian YE + + B6 D086 ; # capital Ukrainian I + B7 D087 ; # capital Ukrainian YI + + B9 E28496 ; # numero sign + + BD D290 ; # capital Ukrainian soft G + BE D18E ; # capital Byelorussian short U + + BF C2A9 ; # (C) + + C0 D18E ; # small yu + C1 D0B0 ; # small a + C2 D0B1 ; # small b + C3 D186 ; # small ts + C4 D0B4 ; # small d + C5 D0B5 ; # small ye + C6 D184 ; # small f + C7 D0B3 ; # small g + C8 D185 ; # small kh + C9 D0B8 ; # small i + CA D0B9 ; # small j + CB D0BA ; # small k + CC D0BB ; # small l + CD D0BC ; # small m + CE D0BD ; # small n + CF D0BE ; # small o + + D0 D0BF ; # small p + D1 D18F ; # small ya + D2 D180 ; # small r + D3 D181 ; # small s + D4 D182 ; # small t + D5 D183 ; # small u + D6 D0B6 ; # small zh + D7 D0B2 ; # small v + D8 D18C ; # small soft sign + D9 D18B ; # small y + DA D0B7 ; # small z + DB D188 ; # small sh + DC D18D ; # small e + DD D189 ; # small shch + DE D187 ; # small ch + DF D18A ; # small hard sign + + E0 D0AE ; # capital YU + E1 D090 ; # capital A + E2 D091 ; # capital B + E3 D0A6 ; # capital TS + E4 D094 ; # capital D + E5 D095 ; # capital YE + E6 D0A4 ; # capital F + E7 D093 ; # capital G + E8 D0A5 ; # capital KH + E9 D098 ; # capital I + EA D099 ; # capital J + EB D09A ; # capital K + EC D09B ; # capital L + ED D09C ; # capital M + EE D09D ; # capital N + EF D09E ; # capital O + + F0 D09F ; # capital P + F1 D0AF ; # capital YA + F2 D0A0 ; # capital R + F3 D0A1 ; # capital S + F4 D0A2 ; # capital T + F5 D0A3 ; # capital U + F6 D096 ; # capital ZH + F7 D092 ; # capital V + F8 D0AC ; # capital soft sign + F9 D0AB ; # capital Y + FA D097 ; # capital Z + FB D0A8 ; # capital SH + FC D0AD ; # capital E + FD D0A9 ; # capital SHCH + FE D0A7 ; # capital CH + FF D0AA ; # capital hard sign +} diff --git a/service-plans/nginx/config/koi-win b/service-plans/nginx/config/koi-win new file mode 100644 index 0000000..72afabe --- /dev/null +++ b/service-plans/nginx/config/koi-win @@ -0,0 +1,103 @@ + +charset_map koi8-r windows-1251 { + + 80 88 ; # euro + + 95 95 ; # bullet + + 9A A0 ; #   + + 9E B7 ; # · + + A3 B8 ; # small yo + A4 BA ; # small Ukrainian ye + + A6 B3 ; # small Ukrainian i + A7 BF ; # small Ukrainian yi + + AD B4 ; # small Ukrainian soft g + AE A2 ; # small Byelorussian short u + + B0 B0 ; # ° + + B3 A8 ; # capital YO + B4 AA ; # capital Ukrainian YE + + B6 B2 ; # capital Ukrainian I + B7 AF ; # capital Ukrainian YI + + B9 B9 ; # numero sign + + BD A5 ; # capital Ukrainian soft G + BE A1 ; # capital Byelorussian short U + + BF A9 ; # (C) + + C0 FE ; # small yu + C1 E0 ; # small a + C2 E1 ; # small b + C3 F6 ; # small ts + C4 E4 ; # small d + C5 E5 ; # small ye + C6 F4 ; # small f + C7 E3 ; # small g + C8 F5 ; # small kh + C9 E8 ; # small i + CA E9 ; # small j + CB EA ; # small k + CC EB ; # small l + CD EC ; # small m + CE ED ; # small n + CF EE ; # small o + + D0 EF ; # small p + D1 FF ; # small ya + D2 F0 ; # small r + D3 F1 ; # small s + D4 F2 ; # small t + D5 F3 ; # small u + D6 E6 ; # small zh + D7 E2 ; # small v + D8 FC ; # small soft sign + D9 FB ; # small y + DA E7 ; # small z + DB F8 ; # small sh + DC FD ; # small e + DD F9 ; # small shch + DE F7 ; # small ch + DF FA ; # small hard sign + + E0 DE ; # capital YU + E1 C0 ; # capital A + E2 C1 ; # capital B + E3 D6 ; # capital TS + E4 C4 ; # capital D + E5 C5 ; # capital YE + E6 D4 ; # capital F + E7 C3 ; # capital G + E8 D5 ; # capital KH + E9 C8 ; # capital I + EA C9 ; # capital J + EB CA ; # capital K + EC CB ; # capital L + ED CC ; # capital M + EE CD ; # capital N + EF CE ; # capital O + + F0 CF ; # capital P + F1 DF ; # capital YA + F2 D0 ; # capital R + F3 D1 ; # capital S + F4 D2 ; # capital T + F5 D3 ; # capital U + F6 C6 ; # capital ZH + F7 C2 ; # capital V + F8 DC ; # capital soft sign + F9 DB ; # capital Y + FA C7 ; # capital Z + FB D8 ; # capital SH + FC DD ; # capital E + FD D9 ; # capital SHCH + FE D7 ; # capital CH + FF DA ; # capital hard sign +} diff --git a/service-plans/nginx/config/mime.types b/service-plans/nginx/config/mime.types new file mode 100644 index 0000000..89be9a4 --- /dev/null +++ b/service-plans/nginx/config/mime.types @@ -0,0 +1,89 @@ + +types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + image/svg+xml svg svgz; + image/webp webp; + + application/font-woff woff; + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.wap.wmlc wmlc; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; + application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} diff --git a/service-plans/nginx/config/nginx.conf b/service-plans/nginx/config/nginx.conf new file mode 100644 index 0000000..f96b97f --- /dev/null +++ b/service-plans/nginx/config/nginx.conf @@ -0,0 +1,34 @@ +worker_processes {{cfg.worker_processes}}; +daemon off; + +events { + worker_connections {{cfg.events.worker_connections}}; +} + +http { + include mime.types; + default_type application/octet-stream; + + sendfile {{cfg.http.sendfile}}; + tcp_nopush {{cfg.http.tcp_nopush}}; + tcp_nodelay {{cfg.http.tcp_nodelay}}; + + keepalive_timeout {{cfg.http.keepalive_timeout}}; + + gzip on; + gzip_vary on; + gzip_min_length 10240; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml; + gzip_disable "MSIE [1-6]\."; + + server { + listen {{cfg.http.listen.port}}; + server_name localhost; + + location / { + root {{pkg.svc_data_path}}; + index index.html index.htm; + } + } +} diff --git a/service-plans/nginx/config/scgi_params b/service-plans/nginx/config/scgi_params new file mode 100644 index 0000000..47348ca --- /dev/null +++ b/service-plans/nginx/config/scgi_params @@ -0,0 +1,16 @@ + +scgi_param REQUEST_METHOD $request_method; +scgi_param REQUEST_URI $request_uri; +scgi_param QUERY_STRING $query_string; +scgi_param CONTENT_TYPE $content_type; + +scgi_param DOCUMENT_URI $document_uri; +scgi_param DOCUMENT_ROOT $document_root; +scgi_param SCGI 1; +scgi_param SERVER_PROTOCOL $server_protocol; +scgi_param HTTPS $https if_not_empty; + +scgi_param REMOTE_ADDR $remote_addr; +scgi_param REMOTE_PORT $remote_port; +scgi_param SERVER_PORT $server_port; +scgi_param SERVER_NAME $server_name; diff --git a/service-plans/nginx/config/uwsgi_params b/service-plans/nginx/config/uwsgi_params new file mode 100644 index 0000000..f539451 --- /dev/null +++ b/service-plans/nginx/config/uwsgi_params @@ -0,0 +1,16 @@ + +uwsgi_param QUERY_STRING $query_string; +uwsgi_param REQUEST_METHOD $request_method; +uwsgi_param CONTENT_TYPE $content_type; +uwsgi_param CONTENT_LENGTH $content_length; + +uwsgi_param REQUEST_URI $request_uri; +uwsgi_param PATH_INFO $document_uri; +uwsgi_param DOCUMENT_ROOT $document_root; +uwsgi_param SERVER_PROTOCOL $server_protocol; +uwsgi_param HTTPS $https if_not_empty; + +uwsgi_param REMOTE_ADDR $remote_addr; +uwsgi_param REMOTE_PORT $remote_port; +uwsgi_param SERVER_PORT $server_port; +uwsgi_param SERVER_NAME $server_name; diff --git a/service-plans/nginx/config/win-utf b/service-plans/nginx/config/win-utf new file mode 100644 index 0000000..391b335 --- /dev/null +++ b/service-plans/nginx/config/win-utf @@ -0,0 +1,125 @@ +# This map is not a full windows-1251 <> utf8 map: it does not +# contain Serbian and Macedonian letters. If you need a full map, +# use contrib/unicode2nginx/win-utf map instead. + +charset_map windows-1251 utf-8 { + + 82 E2809A ; # single low-9 quotation mark + + 84 E2809E ; # double low-9 quotation mark + 85 E280A6 ; # ellipsis + 86 E280A0 ; # dagger + 87 E280A1 ; # double dagger + 88 E282AC ; # euro + 89 E280B0 ; # per mille + + 91 E28098 ; # left single quotation mark + 92 E28099 ; # right single quotation mark + 93 E2809C ; # left double quotation mark + 94 E2809D ; # right double quotation mark + 95 E280A2 ; # bullet + 96 E28093 ; # en dash + 97 E28094 ; # em dash + + 99 E284A2 ; # trade mark sign + + A0 C2A0 ; #   + A1 D18E ; # capital Byelorussian short U + A2 D19E ; # small Byelorussian short u + + A4 C2A4 ; # currency sign + A5 D290 ; # capital Ukrainian soft G + A6 C2A6 ; # borken bar + A7 C2A7 ; # section sign + A8 D081 ; # capital YO + A9 C2A9 ; # (C) + AA D084 ; # capital Ukrainian YE + AB C2AB ; # left-pointing double angle quotation mark + AC C2AC ; # not sign + AD C2AD ; # soft hypen + AE C2AE ; # (R) + AF D087 ; # capital Ukrainian YI + + B0 C2B0 ; # ° + B1 C2B1 ; # plus-minus sign + B2 D086 ; # capital Ukrainian I + B3 D196 ; # small Ukrainian i + B4 D291 ; # small Ukrainian soft g + B5 C2B5 ; # micro sign + B6 C2B6 ; # pilcrow sign + B7 C2B7 ; # · + B8 D191 ; # small yo + B9 E28496 ; # numero sign + BA D194 ; # small Ukrainian ye + BB C2BB ; # right-pointing double angle quotation mark + + BF D197 ; # small Ukrainian yi + + C0 D090 ; # capital A + C1 D091 ; # capital B + C2 D092 ; # capital V + C3 D093 ; # capital G + C4 D094 ; # capital D + C5 D095 ; # capital YE + C6 D096 ; # capital ZH + C7 D097 ; # capital Z + C8 D098 ; # capital I + C9 D099 ; # capital J + CA D09A ; # capital K + CB D09B ; # capital L + CC D09C ; # capital M + CD D09D ; # capital N + CE D09E ; # capital O + CF D09F ; # capital P + + D0 D0A0 ; # capital R + D1 D0A1 ; # capital S + D2 D0A2 ; # capital T + D3 D0A3 ; # capital U + D4 D0A4 ; # capital F + D5 D0A5 ; # capital KH + D6 D0A6 ; # capital TS + D7 D0A7 ; # capital CH + D8 D0A8 ; # capital SH + D9 D0A9 ; # capital SHCH + DA D0AA ; # capital hard sign + DB D0AB ; # capital Y + DC D0AC ; # capital soft sign + DD D0AD ; # capital E + DE D0AE ; # capital YU + DF D0AF ; # capital YA + + E0 D0B0 ; # small a + E1 D0B1 ; # small b + E2 D0B2 ; # small v + E3 D0B3 ; # small g + E4 D0B4 ; # small d + E5 D0B5 ; # small ye + E6 D0B6 ; # small zh + E7 D0B7 ; # small z + E8 D0B8 ; # small i + E9 D0B9 ; # small j + EA D0BA ; # small k + EB D0BB ; # small l + EC D0BC ; # small m + ED D0BD ; # small n + EE D0BE ; # small o + EF D0BF ; # small p + + F0 D180 ; # small r + F1 D181 ; # small s + F2 D182 ; # small t + F3 D183 ; # small u + F4 D184 ; # small f + F5 D185 ; # small kh + F6 D186 ; # small ts + F7 D187 ; # small ch + F8 D188 ; # small sh + F9 D189 ; # small shch + FA D18A ; # small hard sign + FB D18B ; # small y + FC D18C ; # small soft sign + FD D18D ; # small e + FE D18E ; # small yu + FF D18F ; # small ya +} diff --git a/service-plans/nginx/default.toml b/service-plans/nginx/default.toml new file mode 100644 index 0000000..e7ccdf9 --- /dev/null +++ b/service-plans/nginx/default.toml @@ -0,0 +1,38 @@ + # # #### # # # # # + ## # # # # ## # # # + # # # # # # # # ## + # # # # ### # # # # ## + # ## # # # # ## # # + # # #### # # # # # + # For help with NGINX Config Tuning, + # refer to: http://nginx.org/en/docs/http/ngx_http_core_module.html + + +#### General Configuration +# worker_processes: Number of NGINX processes. Default = 1 +worker_processes = 4 + + + +#### Events Context Configuration +[events] +# worker_connections: Connections per Worker Process. Default = 1024 +worker_connections = 1024 + + +#### HTTP Context Configuration +[http] +# http.sendfile: Enable (on) or disable (off) Sendfile Support. Default = on +sendfile = "on" + +# http.tcp_nopush: Enable (on) or disable (off) use of TCP_NOPUSH or TCP_CORK socket option. Default = on +tcp_nopush = "on" + +# http.tcp_nodelay: Enable (on) or disable (off) use of the TCP_NODELAY option. Default = on +tcp_nodelay = "on" + +# http.keepalive_timeout: Timeout on client connection keepalive, in seconds. Default = 75 +keepalive_timeout = 60 + +[http.listen] +port = 80 diff --git a/service-plans/nginx/plan.sh b/service-plans/nginx/plan.sh new file mode 100644 index 0000000..313fd1b --- /dev/null +++ b/service-plans/nginx/plan.sh @@ -0,0 +1,66 @@ +pkg_name=nginx +pkg_origin=core +pkg_version=1.11.10 +pkg_description="NGINX web server." +pkg_maintainer="The Habitat Maintainers " +pkg_license=('bsd') +pkg_source=https://nginx.org/download/nginx-${pkg_version}.tar.gz +pkg_upstream_url=https://nginx.org/ +pkg_shasum=778b3cabb07633f754cd9dee32fc8e22582bce22bfa407be76a806abd935533d +pkg_deps=(core/glibc core/libedit core/ncurses core/zlib core/bzip2 core/openssl core/pcre) +pkg_build_deps=(core/gcc core/make core/coreutils) +pkg_lib_dirs=(lib) +pkg_bin_dirs=(sbin) +pkg_include_dirs=(include) +pkg_svc_run="nginx" +pkg_svc_user="root" +pkg_exports=( + [port]=http.listen.port +) +pkg_exposes=(port) + +do_build() { + ./configure --prefix="$pkg_prefix" \ + --conf-path="$pkg_svc_config_path/nginx.conf" \ + --sbin-path="$pkg_prefix/bin/nginx" \ + --pid-path="$pkg_svc_var_path/nginx.pid" \ + --lock-path="$pkg_svc_var_path/nginx.lock" \ + --user=hab \ + --group=hab \ + --http-log-path=/dev/stdout \ + --error-log-path=stderr \ + --http-client-body-temp-path="$pkg_svc_var_path/client-body" \ + --http-proxy-temp-path="$pkg_svc_var_path/proxy" \ + --http-fastcgi-temp-path="$pkg_svc_var_path/fastcgi" \ + --http-scgi-temp-path="$pkg_svc_var_path/scgi" \ + --http-uwsgi-temp-path="$pkg_svc_var_path/uwsgi" \ + --with-ipv6 \ + --with-pcre \ + --with-pcre-jit \ + --with-file-aio \ + --with-stream=dynamic \ + --with-mail=dynamic \ + --with-http_gunzip_module \ + --with-http_gzip_static_module \ + --with-http_realip_module \ + --with-http_v2_module \ + --with-http_ssl_module \ + --with-http_stub_status_module \ + --with-http_addition_module \ + --with-http_degradation_module \ + --with-http_flv_module \ + --with-http_mp4_module \ + --with-http_secure_link_module \ + --with-http_sub_module \ + --with-http_slice_module \ + --with-cc-opt="$CFLAGS" \ + --with-ld-opt="$LDFLAGS" + + make +} + +do_install() { + make install + mkdir -p "$pkg_prefix/sbin" + cp "$HAB_CACHE_SRC_PATH/$pkg_dirname/objs/nginx" "$pkg_prefix/sbin" +} From 9814cfdf9aba1fefec9d5dd80b3cceb0e40b27da Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 12:45:23 -0400 Subject: [PATCH 21/56] Move nginx service config to runtime configuration file --- service-plans/nginx/config/nginx.conf | 9 +++++++++ service-plans/nginx/hooks/run | 5 +++++ service-plans/nginx/plan.sh | 9 --------- 3 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 service-plans/nginx/hooks/run diff --git a/service-plans/nginx/config/nginx.conf b/service-plans/nginx/config/nginx.conf index f96b97f..b181641 100644 --- a/service-plans/nginx/config/nginx.conf +++ b/service-plans/nginx/config/nginx.conf @@ -1,3 +1,6 @@ +pid {{pkg.svc_var_path}}/nginx.pid; +lock_file {{pkg.svc_var_path}}/nginx.lock; + worker_processes {{cfg.worker_processes}}; daemon off; @@ -6,6 +9,12 @@ events { } http { + client_body_temp_path {{pkg.svc_var_path}}/client-body; + proxy_temp_path {{pkg.svc_var_path}}/proxy; + fastcgi_temp_path {{pkg.svc_var_path}}/fastcgi; + scgi_temp_path {{pkg.svc_var_path}}/scgi; + uwsgi_temp_path {{pkg.svc_var_path}}/uwsgi; + include mime.types; default_type application/octet-stream; diff --git a/service-plans/nginx/hooks/run b/service-plans/nginx/hooks/run new file mode 100644 index 0000000..4b7d005 --- /dev/null +++ b/service-plans/nginx/hooks/run @@ -0,0 +1,5 @@ +#!/bin/bash + +exec 2>&1 + +exec nginx -c {{pkg.svc_config_path}}/nginx.conf \ No newline at end of file diff --git a/service-plans/nginx/plan.sh b/service-plans/nginx/plan.sh index 313fd1b..0c30feb 100644 --- a/service-plans/nginx/plan.sh +++ b/service-plans/nginx/plan.sh @@ -12,7 +12,6 @@ pkg_build_deps=(core/gcc core/make core/coreutils) pkg_lib_dirs=(lib) pkg_bin_dirs=(sbin) pkg_include_dirs=(include) -pkg_svc_run="nginx" pkg_svc_user="root" pkg_exports=( [port]=http.listen.port @@ -21,19 +20,11 @@ pkg_exposes=(port) do_build() { ./configure --prefix="$pkg_prefix" \ - --conf-path="$pkg_svc_config_path/nginx.conf" \ --sbin-path="$pkg_prefix/bin/nginx" \ - --pid-path="$pkg_svc_var_path/nginx.pid" \ - --lock-path="$pkg_svc_var_path/nginx.lock" \ --user=hab \ --group=hab \ --http-log-path=/dev/stdout \ --error-log-path=stderr \ - --http-client-body-temp-path="$pkg_svc_var_path/client-body" \ - --http-proxy-temp-path="$pkg_svc_var_path/proxy" \ - --http-fastcgi-temp-path="$pkg_svc_var_path/fastcgi" \ - --http-scgi-temp-path="$pkg_svc_var_path/scgi" \ - --http-uwsgi-temp-path="$pkg_svc_var_path/uwsgi" \ --with-ipv6 \ --with-pcre \ --with-pcre-jit \ From ea9a64f11f4d84c753f56f91aa1adc803d83efbf Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 12:46:16 -0400 Subject: [PATCH 22/56] Move useful config-driven sections of nginx.conf to reusable includes --- service-plans/nginx/config/http.include | 15 +++++++++++++++ service-plans/nginx/config/nginx.conf | 23 ++--------------------- service-plans/nginx/config/worker.include | 6 ++++++ 3 files changed, 23 insertions(+), 21 deletions(-) create mode 100644 service-plans/nginx/config/http.include create mode 100644 service-plans/nginx/config/worker.include diff --git a/service-plans/nginx/config/http.include b/service-plans/nginx/config/http.include new file mode 100644 index 0000000..f73384f --- /dev/null +++ b/service-plans/nginx/config/http.include @@ -0,0 +1,15 @@ +include mime.types; +default_type application/octet-stream; + +sendfile {{cfg.http.sendfile}}; +tcp_nopush {{cfg.http.tcp_nopush}}; +tcp_nodelay {{cfg.http.tcp_nodelay}}; + +keepalive_timeout {{cfg.http.keepalive_timeout}}; + +gzip on; +gzip_vary on; +gzip_min_length 10240; +gzip_proxied expired no-cache no-store private auth; +gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml; +gzip_disable "MSIE [1-6]\."; \ No newline at end of file diff --git a/service-plans/nginx/config/nginx.conf b/service-plans/nginx/config/nginx.conf index b181641..7410ac0 100644 --- a/service-plans/nginx/config/nginx.conf +++ b/service-plans/nginx/config/nginx.conf @@ -1,12 +1,7 @@ pid {{pkg.svc_var_path}}/nginx.pid; lock_file {{pkg.svc_var_path}}/nginx.lock; -worker_processes {{cfg.worker_processes}}; -daemon off; - -events { - worker_connections {{cfg.events.worker_connections}}; -} +include worker.include; http { client_body_temp_path {{pkg.svc_var_path}}/client-body; @@ -15,21 +10,7 @@ http { scgi_temp_path {{pkg.svc_var_path}}/scgi; uwsgi_temp_path {{pkg.svc_var_path}}/uwsgi; - include mime.types; - default_type application/octet-stream; - - sendfile {{cfg.http.sendfile}}; - tcp_nopush {{cfg.http.tcp_nopush}}; - tcp_nodelay {{cfg.http.tcp_nodelay}}; - - keepalive_timeout {{cfg.http.keepalive_timeout}}; - - gzip on; - gzip_vary on; - gzip_min_length 10240; - gzip_proxied expired no-cache no-store private auth; - gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml; - gzip_disable "MSIE [1-6]\."; + include http.include; server { listen {{cfg.http.listen.port}}; diff --git a/service-plans/nginx/config/worker.include b/service-plans/nginx/config/worker.include new file mode 100644 index 0000000..6b5fb7a --- /dev/null +++ b/service-plans/nginx/config/worker.include @@ -0,0 +1,6 @@ +worker_processes {{cfg.worker_processes}}; +daemon off; + +events { + worker_connections {{cfg.events.worker_connections}}; +} \ No newline at end of file From 9e96f347d90eb1f5a74d372d3207dcd403a315f3 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 13:58:12 -0400 Subject: [PATCH 23/56] Add config_path to nginx for passing custom configuration file --- service-plans/nginx/default.toml | 3 +++ service-plans/nginx/hooks/run | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/service-plans/nginx/default.toml b/service-plans/nginx/default.toml index e7ccdf9..40272d8 100644 --- a/service-plans/nginx/default.toml +++ b/service-plans/nginx/default.toml @@ -9,6 +9,9 @@ #### General Configuration +# a config file to entirely replace the built-in nginx.conf template +config_path = false + # worker_processes: Number of NGINX processes. Default = 1 worker_processes = 4 diff --git a/service-plans/nginx/hooks/run b/service-plans/nginx/hooks/run index 4b7d005..a1bf958 100644 --- a/service-plans/nginx/hooks/run +++ b/service-plans/nginx/hooks/run @@ -2,4 +2,8 @@ exec 2>&1 -exec nginx -c {{pkg.svc_config_path}}/nginx.conf \ No newline at end of file +{{#if cfg.config_path~}} + exec nginx -c {{cfg.config_path}} +{{~else~}} + exec nginx -c {{pkg.svc_config_path}}/nginx.conf +{{~/if}} From 211e38d21be3cf5585e3e324500bb20a4f9fcf7e Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 16:53:02 -0400 Subject: [PATCH 24/56] Convert kernel to use habitat environment --- bin/create-site.sh | 2 +- bin/git-shell | 4 +- bin/kernel | 151 +-------------- bin/shell | 14 +- bin/su | 2 +- habitat/config/config.json | 7 +- habitat/config/mariadb | 37 ++++ habitat/config/nginx | 57 ++++++ habitat/config/php-fpm | 35 ++++ habitat/default.toml | 19 +- habitat/hooks/init | 39 ++++ habitat/plan.sh | 267 +------------------------- kernel-lib/server.js | 48 ++--- kernel-lib/services.js | 48 +---- kernel-lib/services/mariadb.js | 339 +++++++++++++++++++++++++++++++++ kernel-lib/services/nginx.js | 157 +++------------ kernel-lib/services/php-fpm.js | 102 ++-------- kernel-lib/sites.js | 45 ++--- package.json | 1 - php-bootstrap/mail.php | 16 +- 20 files changed, 664 insertions(+), 726 deletions(-) create mode 100644 habitat/config/mariadb create mode 100644 habitat/config/nginx create mode 100644 habitat/config/php-fpm create mode 100755 habitat/hooks/init create mode 100644 kernel-lib/services/mariadb.js diff --git a/bin/create-site.sh b/bin/create-site.sh index 0a098e6..1375463 100755 --- a/bin/create-site.sh +++ b/bin/create-site.sh @@ -82,7 +82,7 @@ if [ "$GENERATE_TABLES" ] || [ "$PRECACHE_TREES" ] || [ "$ENTER_SHELL" ]; then fi if [ -z "$KERNEL_SOCKET" ]; then - KERNEL_SOCKET="/emergence/kernel.sock" + KERNEL_SOCKET="/hab/svc/emergence-kernel/var/run/kernel.sock" fi if [ ! -S "$KERNEL_SOCKET" ]; then diff --git a/bin/git-shell b/bin/git-shell index c508f57..64d26c6 100755 --- a/bin/git-shell +++ b/bin/git-shell @@ -21,7 +21,7 @@ SITE_HANDLE=$1 SITE_LAYER=$2 UNDERSCORE=$DIR/../node_modules/.bin/underscore -EMERGENCE_USER=`cat /emergence/config.json | $UNDERSCORE extract user --outfmt text` +EMERGENCE_USER=`cat /hab/svc/emergence-kernel/config/config.json | $UNDERSCORE extract user --outfmt text` BLACK="\033[01;30m" MAGENTA="\033[1;31m" @@ -49,7 +49,7 @@ test -n "$SITE_HANDLE" -a -n "$SITE_LAYER"|| die "usage: emergence-git-shell sit # get site directory -SITE_DIR="/emergence/sites/$SITE_HANDLE" +SITE_DIR="/hab/svc/emergence-kernel/data/sites/$SITE_HANDLE" test -d "$SITE_DIR" || die "site $SITE_HANDLE not found at $SITE_DIR" diff --git a/bin/kernel b/bin/kernel index d231123..47eeacd 100755 --- a/bin/kernel +++ b/bin/kernel @@ -6,160 +6,19 @@ var _ = require('underscore'), url = require('url'), path = require('path'), fs = require('fs'), - sitesLib = require('../kernel-lib/sites.js'); - - -var CONFIG; - -if (fs.existsSync('/emergence/config.json')) { - // try to load existing kernel config - - CONFIG = JSON.parse(fs.readFileSync('/emergence/config.json', 'ascii')); -} else { - // try to smart-init a new kernel config - var issueFile = "", - platform = require('os').platform(), - distro = platform; - - // detect ubuntu - // TODO: find better ways to determine default options than platform detection - if (platform == 'linux') { - try { - issueFile = fs.readFileSync('/etc/issue', 'ascii'); - if (issueFile.match(/Ubuntu/)) { - distro = 'ubuntu'; - } - } catch (err) { - // ignore any failure to real /etc/issue and leave platform set to linux - } - } - - console.log('Initializing new emergence environment for distro="'+distro+'"...'); - - if (!fs.existsSync('/emergence')) { - fs.mkdirSync('/emergence', '775'); - } - - CONFIG = { - user: distro=='ubuntu' ? 'www-data' : 'nobody', - group: distro=='ubuntu' ? 'www-data' : 'nobody', - - server: { - host: "0.0.0.0" - }, - services: { - plugins: { } - } - }; - - - - - // detect nginx - if (fs.existsSync('/usr/sbin/nginx')) { - CONFIG.services.plugins.web = { - type: 'nginx', - autoStart: true, - execPath: '/usr/sbin/nginx', - bindHost: '0.0.0.0' - }; - } else if (fs.existsSync('/usr/local/sbin/nginx')) { - CONFIG.services.plugins.web = { - type: 'nginx', - autoStart: true, - execPath: '/usr/local/sbin/nginx', - bindHost: '0.0.0.0' - }; - } - - // detect mysql - if (fs.existsSync('/usr/sbin/mysqld')) { - CONFIG.services.plugins.sql = { - type: 'mysql', - autoStart: true, - execPath: '/usr/sbin/mysqld', - managerUser: 'root', - managerPassword: sitesLib.generatePassword() - }; - } else if (fs.existsSync('/usr/local/bin/mysqld')) { - CONFIG.services.plugins.sql = { - type: 'mysql', - autoStart: true, - execPath: '/usr/local/bin/mysqld', - managerUser: 'root', - managerPassword: sitesLib.generatePassword() - }; - } else if (fs.existsSync('/usr/bin/mysqld_safe')) { - CONFIG.services.plugins.sql = { - type: 'mysql', - autoStart: true, - execPath: '/usr/bin/mysqld_safe', - managerUser: 'root', - managerPassword: sitesLib.generatePassword() - }; - } - - // detect php-fpm - if (fs.existsSync('/usr/sbin/php-fpm5.6')) { - CONFIG.services.plugins.php = { - type: 'php-fpm', - autoStart: true, - execPath: '/usr/sbin/php-fpm5.6' - }; - } else if (fs.existsSync('/usr/sbin/php-fpm7.0')) { - CONFIG.services.plugins.php = { - type: 'php-fpm', - autoStart: true, - execPath: '/usr/sbin/php-fpm7.0' - }; - } else if (fs.existsSync('/usr/bin/php-fpm')) { - CONFIG.services.plugins.php = { - type: 'php-fpm', - autoStart: true, - execPath: '/usr/bin/php-fpm' - }; - } else if (fs.existsSync('/usr/sbin/php5-fpm')) { - CONFIG.services.plugins.php = { - type: 'php-fpm', - autoStart: true, - execPath: '/usr/sbin/php5-fpm' - }; - } else if (fs.existsSync('/usr/local/sbin/php-fpm')) { - CONFIG.services.plugins.php = { - type: 'php-fpm', - autoStart: true, - execPath: '/usr/local/sbin/php-fpm' - }; - } else if (fs.existsSync('/usr/sbin/php-fpm')) { - CONFIG.services.plugins.php = { - type: 'php-fpm', - autoStart: true, - execPath: '/usr/sbin/php-fpm' - }; - } - - fs.writeFileSync('/emergence/config.json', JSON.stringify(CONFIG, null, 4)); - fs.chmodSync('/emergence/config.json', '600'); - console.log('Generated and wrote initial config: /emergence/config.json'); -} - -// create default admin user -if (!fs.existsSync('/emergence/admins.htpasswd')) { - console.log('Creating default administrative user: admin/admin'); - fs.writeFileSync('/emergence/admins.htpasswd', 'admin:{SHA}0DPiKuNIrrVmD8IUCuw1hQxNqZc='); - fs.chmodSync('/emergence/admins.htpasswd', '600'); -} + sitesLib = require('../kernel-lib/sites.js'), + kernelConfig = require('/hab/svc/emergence-kernel/config/config.json'); // load core modules -var eSites = sitesLib.createSites(CONFIG); -var eServices = require('../kernel-lib/services.js').createServices(eSites, CONFIG); +var eSites = sitesLib.createSites(kernelConfig); +var eServices = require('../kernel-lib/services.js').createServices(eSites, kernelConfig); // instantiate management server var eManagementServer = require('../kernel-lib/server.js').createServer({ sites: eSites, services: eServices -}, CONFIG); +}, kernelConfig); // start server diff --git a/bin/shell b/bin/shell index 5f00f6e..9de4013 100755 --- a/bin/shell +++ b/bin/shell @@ -20,7 +20,7 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" [ -n "$1" ] || die "Usage: emergence-kernel site-handle" siteHandle=$1 -siteDir="/emergence/sites/$siteHandle" +siteDir="/hab/svc/emergence-kernel/data/sites/$siteHandle" [ -d "$siteDir" ] || die "Site '$siteHandle' not found at '$siteDir'" @@ -33,8 +33,8 @@ echo "require('`dirname $DIR`/php-bootstrap/bootstrap.inc.php');" >> $autoPrepen echo "Site::initialize('$siteDir');" >> $autoPrependScript -# give www-data access to script -chgrp www-data $autoPrependScript +# give hab access to script +chgrp hab $autoPrependScript chmod g+r $autoPrependScript @@ -55,7 +55,13 @@ fi # execute interactive shell -TERM=$TERM sudo -E -u www-data -g www-data php -d auto_prepend_file=$autoPrependScript -d apc.enable_cli=on $INPUT_ARGS +PHP_ARGS="-d auto_prepend_file=$autoPrependScript -d apc.enable_cli=on $INPUT_ARGS" + +if [ "$(whoami)" == "hab" ]; then + TERM=$TERM php $PHP_ARGS +else + TERM=$TERM sudo -E -u hab -g hab php $PHP_ARGS +fi # clean up diff --git a/bin/su b/bin/su index b44cb1e..2fbf742 100755 --- a/bin/su +++ b/bin/su @@ -1,2 +1,2 @@ #!/bin/bash -sudo su -l www-data -p \ No newline at end of file +sudo su -l hab -p \ No newline at end of file diff --git a/habitat/config/config.json b/habitat/config/config.json index c3bb842..988c768 100644 --- a/habitat/config/config.json +++ b/habitat/config/config.json @@ -7,17 +7,16 @@ "services": { "web": { "type": "nginx", - "execPath": "{{pkgPathFor "core/nginx"}}/bin/nginx", + "packagePath": "{{pkgPathFor "emergence/nginx"}}", "bindHost": "{{cfg.web.host}}" }, "sql": { "type": "mariadb", - "execPath": "{{pkgPathFor "core/mariadb"}}/bin/mysqld", - "bindHost": "{{cfg.sql.host}}" + "packagePath": "{{pkgPathFor "emergence/mariadb"}}" }, "php": { "type": "php-fpm", - "execPath": "{{pkgPathFor "core/php"}}/sbin/php-fpm" + "packagePath": "{{pkgPathFor "emergence/php"}}" } } } \ No newline at end of file diff --git a/habitat/config/mariadb b/habitat/config/mariadb new file mode 100644 index 0000000..30ff15f --- /dev/null +++ b/habitat/config/mariadb @@ -0,0 +1,37 @@ +[mysqld] +character-set-server = utf8 +user = hab +port = 3306 +socket = {{pkg.svc_var_path}}/run/mariadb.sock +pid-file = {{pkg.svc_var_path}}/run/mariadb.pid +datadir = {{pkg.svc_data_path}}/services/mariadb +basedir = {{pkgPathFor "emergence/mariadb"}} +skip-external-locking +key_buffer_size = 16M +max_allowed_packet = 1M +sort_buffer_size = 512K +net_buffer_length = 8K +read_buffer_size = 256K +read_rnd_buffer_size = 512K +myisam_sort_buffer_size = 8M +log-bin = mysqld-bin +expire_logs_days = 2 +server-id = 1 +tmpdir = /tmp/ +innodb_buffer_pool_size = 16M +innodb_data_file_path = ibdata1:10M:autoextend:max:128M +innodb_log_file_size = 5M +innodb_log_buffer_size = 8M +innodb_log_files_in_group = 2 +innodb_flush_log_at_trx_commit = 1 +innodb_lock_wait_timeout = 50 +innodb_file_per_table +max_binlog_size = 100M +binlog_format = row +table_open_cache = 64 + +{{#if cfg.sql.host~}} + bind-address = {{cfg.sql.host}} +{{~else~}} + skip-networking +{{~/if}} diff --git a/habitat/config/nginx b/habitat/config/nginx new file mode 100644 index 0000000..ad80380 --- /dev/null +++ b/habitat/config/nginx @@ -0,0 +1,57 @@ +user {{pkg.svc_user}}; +worker_processes auto; +pid {{pkg.svc_var_path}}/run/nginx.pid; +error_log {{pkg.svc_var_path}}/log/nginx.err info; + +events { + worker_connections 1024; + use epoll; +} + +http { + client_body_temp_path {{pkg.svc_var_path}}/tmp/nginx/client-body; + fastcgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/fastcgi; + proxy_temp_path {{pkg.svc_var_path}}/tmp/nginx/proxy; + scgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/scgi_temp_path; + uwsgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/uwsgi; + + include {{pkgPathFor "emergence/nginx"}}/config/mime.types; + default_type application/octet-stream; + + log_format main + '$host $remote_addr - $remote_user [$time_local] ' + '"$request" $status $bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '"$gzip_ratio"'; + + client_header_timeout 10m; + client_body_timeout 10m; + send_timeout 10m; + connection_pool_size 256; + client_max_body_size 200m; + client_body_buffer_size 128k; + client_header_buffer_size 1k; + large_client_header_buffers 8 512k; + request_pool_size 4k; + server_names_hash_bucket_size 1024; + types_hash_max_size 2048; + gzip on; + gzip_min_length 1100; + gzip_buffers 4 8k; + gzip_types text/plain text/css text/x-scss text/x-html-template text/x-component text/xml application/xml application/javascript application/json application/php application/atom+xml application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/xhtml+xml font/opentype image/svg+xml image/x-icon; + output_buffers 1 32k; + postpone_output 1460; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 75 20; + ignore_invalid_headers on; + fastcgi_read_timeout 6h; + fastcgi_buffers 32 64k; + server_tokens off; + + index index.php; + fastcgi_index index.php; + + include {{pkg.svc_var_path}}/config/nginx.sites; +} \ No newline at end of file diff --git a/habitat/config/php-fpm b/habitat/config/php-fpm new file mode 100644 index 0000000..bbdefa2 --- /dev/null +++ b/habitat/config/php-fpm @@ -0,0 +1,35 @@ +[global] +pid = {{pkg.svc_var_path}}/run/php-fpm.pid +error_log = {{pkg.svc_var_path}}/log/php-fpm.err + + +[www] +user = hab +group = hab +catch_workers_output = on + +listen = {{pkg.svc_var_path}}/run/php-fpm.sock +listen.owner = {{pkg.svc_user}} +listen.group = {{pkg.svc_group}} + +pm = dynamic +pm.max_children = {{cfg.php.max_children}} +pm.start_servers = {{cfg.php.start_servers}} +pm.min_spare_servers = {{cfg.php.min_spare_servers}} +pm.max_spare_servers = {{cfg.php.max_spare_servers}} + +{{~#if cfg.php.status_path}} +pm.status_path = {{cfg.php.status_path}} +{{~/if}} + +php_admin_flag[short_open_tag] = on +php_admin_value[apc.shm_size] = {{cfg.php.shm_size}} +php_admin_value[apc.shm_segments] = 1 +php_admin_value[apc.slam_defense] = 0 +php_admin_value[apc.stat] = {{#if cfg.php.stat_scripts}}1{{else}}0{{/if}} +php_admin_value[opcache.validate_timestamps] = {{#if cfg.php.stat_scripts}}1{{else}}0{{/if}} +php_admin_value[upload_max_filesize] = {{cfg.php.upload_max_filesize}} +php_admin_value[post_max_size] = {{cfg.php.post_max_size}} +php_admin_value[memory_limit] = {{cfg.php.memory_limit}} +php_admin_value[error_reporting] = {{cfg.php.error_reporting}} +php_admin_value[date.timezone] = {{cfg.php.timezone}} \ No newline at end of file diff --git a/habitat/default.toml b/habitat/default.toml index 435aa58..6cc0d34 100644 --- a/habitat/default.toml +++ b/habitat/default.toml @@ -7,6 +7,23 @@ host="0.0.0.0" [web] host="0.0.0.0" +port=80 [sql] -host="127.0.0.1" \ No newline at end of file +host="127.0.0.1" + +[php] +stat_scripts=false +status_path=false +error_reporting="E_ALL & ~E_NOTICE" +timezone="America/New_York" + +max_children=50 +start_servers=5 +min_spare_servers=5 +max_spare_servers=10 + +upload_max_filesize="200M" +post_max_size="200M" +memory_limit="200M" +shm_size="512M" \ No newline at end of file diff --git a/habitat/hooks/init b/habitat/hooks/init new file mode 100755 index 0000000..79d6c27 --- /dev/null +++ b/habitat/hooks/init @@ -0,0 +1,39 @@ +#!/bin/sh + +# initialize users database +AUTHFILE="{{pkg.svc_data_path}}/users.htpasswd" + +if [ ! -f "$AUTHFILE" ]; then + echo "Initializing $AUTHFILE with admin/admin user" + echo "admin:{SHA}0DPiKuNIrrVmD8IUCuw1hQxNqZc=" > "$AUTHFILE" + chown root:hab "$AUTHFILE" + chmod 640 "$AUTHFILE" +fi + +# initialize var directories +pushd "{{pkg.svc_var_path}}" > /dev/null +chmod g+rX -R . +for DIR in log run config tmp tmp/nginx; do + if [ ! -d "$DIR" ]; then + mkdir "$DIR" + chmod 770 "$DIR" + chown root:hab "$DIR" + fi +done +popd > /dev/null + +# initialize data directories +pushd "{{pkg.svc_data_path}}" > /dev/null +chmod g+rX -R . +for DIR in sites services; do + if [ ! -d "$DIR" ]; then + mkdir "$DIR" + chmod 770 "$DIR" + chown root:hab "$DIR" + fi +done +popd > /dev/null + +# initialize static asset links +ln -sf "{{pkg.path}}/app/php-bootstrap" "{{pkg.svc_static_path}}/" +ln -sf "{{pkg.path}}/app/kernel-www" "{{pkg.svc_static_path}}/" \ No newline at end of file diff --git a/habitat/plan.sh b/habitat/plan.sh index e8be028..366a194 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -1,267 +1,18 @@ -# This file is the heart of your application's habitat. -# See full docs at https://www.habitat.sh/docs/reference/plan-syntax/ - -# Required. -# Sets the name of the package. This will be used in along with `pkg_origin`, -# and `pkg_version` to define the fully-qualified package name, which determines -# where the package is installed to on disk, how it is referred to in package -# metadata, and so on. pkg_name=emergence-kernel - -# Required unless overridden by the `HAB_ORIGIN` environment variable. -# The origin is used to denote a particular upstream of a package. pkg_origin=emergence +pkg_version="1.0.0" +pkg_upstream_url="https://github.com/JarvusInnovations/emergence" +pkg_scaffolding=core/scaffolding-node +pkg_svc_user="root" -# Required. -# Sets the version of the package. -pkg_version="0.1.0" - -# Optional. -# The name and email address of the package maintainer. -# pkg_maintainer="The Habitat Maintainers " - -# Optional. -# An array of valid software licenses that relate to this package. -# Please choose a license from http://spdx.org/licenses/ -# pkg_license=('Apache-2.0') - -# Optional. -# The resulting filename for the download, typically constructed from the -# pkg_name and pkg_version values. -# pkg_filename="${pkg_name}-${pkg_version}.tar.gz" - -# Required if a valid URL is provided for pkg_source or unless do_verify() is overridden. -# The value for pkg_shasum is a sha-256 sum of the downloaded pkg_source. If you -# do not have the checksum, you can easily generate it by downloading the source -# and using the sha256sum or gsha256sum tools. Also, if you do not have -# do_verify() overridden, and you do not have the correct sha-256 sum, then the -# expected value will be shown in the build output of your package. -pkg_shasum="TODO" - -# Optional. -# An array of package dependencies needed at runtime. You can refer to packages -# at three levels of specificity: `origin/package`, `origin/package/version`, or -# `origin/package/version/release`. -pkg_deps=( - core/mariadb - core/php - core/nginx -) - -# Optional. -# An array of the package dependencies needed only at build time. pkg_build_deps=( core/make core/gcc core/python2 ) -# Optional. -# An array of paths, relative to the final install of the software, where -# libraries can be found. Used to populate LD_FLAGS and LD_RUN_PATH for -# software that depends on your package. -# pkg_lib_dirs=(lib) - -# Optional. -# An array of paths, relative to the final install of the software, where -# headers can be found. Used to populate CFLAGS for software that depends on -# your package. -# pkg_include_dirs=(include) - -# Optional. -# An array of paths, relative to the final install of the software, where -# binaries can be found. Used to populate PATH for software that depends on -# your package. -# pkg_bin_dirs=(bin) - -# Optional. -# An array of paths, relative to the final install of the software, where -# pkg-config metadata (.pc files) can be found. Used to populate -# PKG_CONFIG_PATH for software that depends on your package. -# pkg_pconfig_dirs=(lib/pconfig) - -# Optional. -# The command for the supervisor to execute when starting a service. You can -# omit this setting if your package is not intended to be run directly by a -# supervisor of if your plan contains a run hook in hooks/run. -# pkg_svc_run="bin/haproxy -f $pkg_svc_config_path/haproxy.conf" - -# Optional. -# An associative array representing configuration data which should be gossiped to peers. The keys -# in this array represent the name the value will be assigned and the values represent the toml path -# to read the value. -# pkg_exports=( -# [host]=srv.address -# [port]=srv.port -# [ssl-port]=srv.ssl.port -# ) - -# Optional. -# An array of `pkg_exports` keys containing default values for which ports that this package -# exposes. These values are used as sensible defaults for other tools. For example, when exporting -# a package to a container format. -# pkg_exposes=(port ssl-port) - -# Optional. -# An associative array representing services which you depend on and the configuration keys that -# you expect the service to export (by their `pkg_exports`). These binds *must* be set for the -# supervisor to load the service. The loaded service will wait to run until it's bind becomes -# available. If the bind does not contain the expected keys, the service will not start -# successfully. -# pkg_binds=( -# [database]="port host" -# ) - -# Optional. -# Same as `pkg_binds` but these represent optional services to connect to. -# pkg_binds_optional=( -# [storage]="port host" -# ) - -# Optional. -# An array of interpreters used in shebang lines for scripts. Specify the -# subdirectory where the binary is relative to the package, for example, -# bin/bash or libexec/neverland, since binaries can be located in directories -# besides bin. This list of interpreters will be written to the metadata -# INTERPRETERS file, located inside a package, with their fully-qualified path. -# Then these can be used with the fix_interpreter function. -# pkg_interpreters=(bin/bash) - -# Optional. -# The user to run the service as. The default is hab. -# pkg_svc_user="hab" - -# Optional. -# The group to run the service as. The default is hab. -# pkg_svc_group="$pkg_svc_user" - -# Required for core plans, optional otherwise. -# A short description of the package. It can be a simple string, or you can -# create a multi-line description using markdown to provide a rich description -# of your package. -# pkg_description="Some description." - -# Required for core plans, optional otherwise. -# The project home page for the package. -pkg_upstream_url="https://github.com/JarvusInnovations/emergence" - -pkg_scaffolding=core/scaffolding-node - - -# Callback Functions -# -# When defining your plan, you have the flexibility to override the default -# behavior of Habitat in each part of the package building stage through a -# series of callbacks. To define a callback, simply create a shell function -# of the same name in your plan.sh file and then write your script. If you do -# not want to use the default callback behavior, you must override the callback -# and return 0 in the function definition. -# -# Callbacks are defined here with either their "do_default_x", if they have a -# default implementation, or empty with "return 0" if they have no default -# implementation (Bash does not allow empty function bodies.) If callbacks do -# nothing or do the same as the default implementation, they can be removed from -# this template. -# -# The default implementations (the do_default_* functions) are defined in the -# plan build script: -# https://github.com/habitat-sh/habitat/tree/master/components/plan-build/bin/hab-plan-build.sh - -# There is no default implementation of this callback. You can use it to execute -# any arbitrary commands before anything else happens. -do_begin() { - return 0 -} - -# The default implementation is that the software specified in $pkg_source is -# downloaded, checksum-verified, and placed in $HAB_CACHE_SRC_PATH/$pkgfilename, -# which resolves to a path like /hab/cache/src/filename.tar.gz. You should -# override this behavior if you need to change how your binary source is -# downloaded, if you are not downloading any source code at all, or if your are -# cloning from git. If you do clone a repo from git, you must override -# do_verify() to return 0. -do_download() { - do_default_download -} - -# The default implementation tries to verify the checksum specified in the plan -# against the computed checksum after downloading the source tarball to disk. -# If the specified checksum doesn't match the computed checksum, then an error -# and a message specifying the mismatch will be printed to stderr. You should -# not need to override this behavior unless your package does not download -# any files. -do_verify() { - do_default_verify -} - -# The default implementation removes the HAB_CACHE_SRC_PATH/$pkg_dirname folder -# in case there was a previously-built version of your package installed on -# disk. This ensures you start with a clean build environment. -do_clean() { - do_default_clean -} - -# The default implementation extracts your tarball source file into -# HAB_CACHE_SRC_PATH. The supported archives are: .tar, .tar.bz2, .tar.gz, -# .tar.xz, .rar, .zip, .Z, .7z. If the file archive could not be found or was -# not supported, then a message will be printed to stderr with additional -# information. -do_unpack() { - do_default_unpack -} - -# There is no default implementation of this callback. At this point in the -# build process, the tarball source has been downloaded, unpacked, and the build -# environment variables have been set, so you can use this callback to perform -# any actions before the package starts building, such as exporting variables, -# adding symlinks, and so on. -do_prepare() { - return 0 -} - -# The default implementation is to update the prefix path for the configure -# script to use $pkg_prefix and then run make to compile the downloaded source. -# This means the script in the default implementation does -# ./configure --prefix=$pkg_prefix && make. You should override this behavior -# if you have additional configuration changes to make or other software to -# build and install as part of building your package. -do_build() { - do_default_build -} - -# The default implementation runs nothing during post-compile. An example of a -# command you might use in this callback is make test. To use this callback, two -# conditions must be true. A) do_check() function has been declared, B) DO_CHECK -# environment variable exists and set to true, env DO_CHECK=true. -do_check() { - return 0 -} - -# The default implementation is to run make install on the source files and -# place the compiled binaries or libraries in HAB_CACHE_SRC_PATH/$pkg_dirname, -# which resolves to a path like /hab/cache/src/packagename-version/. It uses -# this location because of do_build() using the --prefix option when calling the -# configure script. You should override this behavior if you need to perform -# custom installation steps, such as copying files from HAB_CACHE_SRC_PATH to -# specific directories in your package, or installing pre-built binaries into -# your package. -do_install() { - do_default_install -} - -# The default implementation is to strip any binaries in $pkg_prefix of their -# debugging symbols. You should override this behavior if you want to change -# how the binaries are stripped, which additional binaries located in -# subdirectories might also need to be stripped, or whether you do not want the -# binaries stripped at all. -do_strip() { - do_default_strip -} - -# There is no default implementation of this callback. This is called after the -# package has been built and installed. You can use this callback to remove any -# temporary files or perform other post-install clean-up actions. -do_end() { - return 0 -} - +pkg_deps=( + emergence/nginx + emergence/mariadb + emergence/php +) \ No newline at end of file diff --git a/kernel-lib/server.js b/kernel-lib/server.js index 84dd2c7..5a5b460 100644 --- a/kernel-lib/server.js +++ b/kernel-lib/server.js @@ -15,17 +15,17 @@ exports.Server = function(paths, config) { // call events constructor events.EventEmitter.call(me); - // initialize options and apply defaults + // initialize configuration + me.socketPath = '/hab/svc/emergence-kernel/var/run/kernel.sock'; + me.staticDir = '/hab/svc/emergence-kernel/static/kernel-www'; + me.usersPath = '/hab/svc/emergence-kernel/data/users.htpasswd'; + me.paths = paths || {}; - me.options = options || {}; - me.options.host = me.options.host || '0.0.0.0'; - me.options.port = me.options.port || 9083; - me.options.sslKey = me.options.sslKey || null; - me.options.sslCert = me.options.sslCert || null; - me.options.staticDir = me.options.staticDir || path.resolve(__dirname, '../kernel-www'); - me.options.socketPath = 'socketPath' in me.options ? me.options.socketPath : '/emergence/kernel.sock'; - - // initialize state + me.host = options.host || '0.0.0.0'; + me.port = options.port || 9083; + me.defaultSuffix = options.defaultSuffix || null; + me.sslKey = options.sslKey || null; + me.sslCert = options.sslCert || null; }; util.inherits(exports.Server, events.EventEmitter); @@ -35,36 +35,36 @@ exports.Server.prototype.start = function() { // create authenticator this.httpAuth = require('http-auth')({ authRealm: 'Emergence Node Management', - authFile: '/emergence/admins.htpasswd' + authFile: this.usersPath }); // create static fileserver - this.fileServer = new static.Server(this.options.staticDir); + this.fileServer = new static.Server(this.staticDir); // listen on web port - if (this.options.sslKey && this.options.sslCert) { + if (this.sslKey && this.sslCert) { this.webServer = require('https').createServer({ - key: fs.readFileSync(this.options.sslKey), - cert: fs.readFileSync(this.options.sslCert) - }, this.handleWebRequest.bind(this)).listen(this.options.port, this.options.host); + key: fs.readFileSync(this.sslKey), + cert: fs.readFileSync(this.sslCert) + }, this.handleWebRequest.bind(this)).listen(this.port, this.host); this.webProtocol = 'https'; } else { - this.webServer = require('http').createServer(this.handleWebRequest.bind(this)).listen(this.options.port, this.options.host); + this.webServer = require('http').createServer(this.handleWebRequest.bind(this)).listen(this.port, this.host); this.webProtocol = 'http'; } // listen on unix socket - if (this.options.socketPath) { - this.socketServer = require('http').createServer(this.handleRequest.bind(this)).listen(this.options.socketPath); - fs.chmodSync(this.options.socketPath, '400'); + if (this.socketPath) { + this.socketServer = require('http').createServer(this.handleRequest.bind(this)).listen(this.socketPath); + fs.chmodSync(this.socketPath, '400'); } // clean up on exit process.on('exit', this.close.bind(this)); - console.log('Management server listening on '+this.webProtocol+'://'+this.options.host+':'+this.options.port); + console.log('Management server listening on '+this.webProtocol+'://'+this.host+':'+this.port); }; exports.createServer = function(paths, options) { @@ -96,7 +96,11 @@ exports.Server.prototype.handleRequest = function(request, response) { if (request.path[0] == 'server-config') { response.writeHead(200, {'Content-Type':'application/json'}); - response.end(JSON.stringify(me.options)); + response.end(JSON.stringify({ + host: me.host, + port: me.port, + defaultSuffix: me.defaultSuffix + })); return; } diff --git a/kernel-lib/services.js b/kernel-lib/services.js index 0853748..ce10cdf 100644 --- a/kernel-lib/services.js +++ b/kernel-lib/services.js @@ -5,49 +5,17 @@ var _ = require('underscore'), events = require('events'); exports.ServicesController = function(sites, config) { - var me = this, - options = config.services; + var me = this; me.sites = sites; + me.services = config.services; // call events constructor events.EventEmitter.call(me); - // initialize options and apply defaults - me.options = options || {}; - me.options.plugins = me.options.plugins || {}; - me.options.servicesDir = me.options.servicesDir || '/emergence/services'; - me.options.logsDir = me.options.logsDir || me.options.servicesDir+'/logs'; - me.options.configDir = me.options.configDir || me.options.servicesDir+'/etc'; - me.options.runDir = me.options.runDir || me.options.servicesDir+'/run'; - me.options.dataDir = me.options.dataDir || me.options.servicesDir+'/data'; - me.options.user = me.options.user || config.user; - me.options.group = me.options.group || config.group; - - // create required directories - if (!fs.existsSync(me.options.servicesDir)) { - fs.mkdirSync(me.options.servicesDir, '775'); - } - - if (!fs.existsSync(me.options.logsDir)) { - fs.mkdirSync(me.options.logsDir, '775'); - } - - if (!fs.existsSync(me.options.configDir)) { - fs.mkdirSync(me.options.configDir, '775'); - } - - if (!fs.existsSync(me.options.runDir)) { - fs.mkdirSync(me.options.runDir, '775'); - } - - if (!fs.existsSync(me.options.dataDir)) { - fs.mkdirSync(me.options.dataDir, '775'); - } - - // load service plugins + // load services me.services = {}; - _.each(me.options.plugins, function(plugin, name) { + _.each(config.services, function(plugin, name) { console.log('Loading service: '+name); if (_.isString(plugin)) { @@ -59,12 +27,10 @@ exports.ServicesController = function(sites, config) { me.services[name] = plugin; }); - // auto-start service plugins + // start services _.each(me.services, function(service, name) { - if (service.options.autoStart) { - console.log('Autostarting service: '+name); - service.start(); - } + console.log('Starting service: '+name); + service.start(); }); }; diff --git a/kernel-lib/services/mariadb.js b/kernel-lib/services/mariadb.js new file mode 100644 index 0000000..6990d8e --- /dev/null +++ b/kernel-lib/services/mariadb.js @@ -0,0 +1,339 @@ +var _ = require('underscore'), + fs = require('fs'), + path = require('path'), + util = require('util'), + spawn = require('child_process').spawn, + exec = require('child_process').exec, + shell = require('shelljs'), + semver = require('semver'), + mariasql = require('mariasql'); + +exports.createService = function(name, controller, options) { + return new exports.MysqlService(name, controller, options); +}; + +exports.MysqlService = function(name, controller, options) { + var me = this, + versionMatch; + + // call parent constructor + exports.MysqlService.super_.apply(me, arguments); + + // initialize configuration + me.packagePath = options.packagePath; + me.execPath = me.packagePath + '/bin/mysqld'; + + me.configPath = '/hab/svc/emergence-kernel/config/mariadb'; + me.pidPath = '/hab/svc/emergence-kernel/var/run/mariadb.pid'; + me.socketPath = '/hab/svc/emergence-kernel/var/run/mariadb.sock'; + me.dataDir = '/hab/svc/emergence-kernel/data/services/mariadb'; + me.statePath = me.dataDir+'/state.json'; + + me.execOptions = ['--defaults-file='+me.configPath, '--basedir='+me.packagePath]; + + // load state + if (fs.existsSync(me.statePath)) { + me.state = JSON.parse(fs.readFileSync(me.statePath, 'ascii')); + } + + // check for existing mysqld process + if (fs.existsSync(me.pidPath)) { + me.pid = parseInt(fs.readFileSync(me.pidPath, 'ascii')); + console.log(me.name+': found existing PID: '+me.pid+', checking /proc/'+me.pid); + + if (fs.existsSync('/proc/'+me.pid)) { + me.status = 'online'; + + // instantiate MySQL client + if (me.state) { + me.client = new mariasql({ + unixSocket: me.socketPath, + user: 'root', + password: me.state.rootPassword, + multiStatements: true + }); + } + } else { + console.log(me.name+': process '+me.pid + ' not found, deleting .pid file'); + fs.unlinkSync(me.pidPath); + } + } + + // listen for site creation + controller.sites.on('siteCreated', _.bind(me.onSiteCreated, me)); +}; + +util.inherits(exports.MysqlService, require('./abstract.js').AbstractService); + + + +exports.MysqlService.prototype.start = function(firstRun) { + var me = this; + + if (me.pid) { + console.log(me.name+': mysql already runnig with PID '+me.pid); + return false; + } + + // init data directory if needed + if (!fs.existsSync(me.dataDir)) { + console.log('creating datadir as ', require("os").userInfo().username); + me.status = 'configuring'; + + console.log(me.name+': initializing new data directory...'); + fs.mkdirSync(me.dataDir); + fs.chownSync(me.dataDir, me.controller.sites.dataUid, me.controller.sites.dataGid); + fs.chmodSync(me.dataDir, '750'); + + // initialize state + me.state = { + rootPassword: me.controller.sites.generatePassword() + }; + + fs.writeFileSync(me.statePath, JSON.stringify(me.state, null, 4)); + fs.chmodSync(me.statePath, '600'); + + // use mysql_install_db to finish init + exec( + me.packagePath + '/scripts/mysql_install_db '+me.execOptions.join(' '), + function(error, stdout, stderr) { + if (error) { + console.log(me.name+': failed to initialize data directory', error); + return; + } + + me.start(true); + } + ); + + return true; // not really started, we have to try again after mysql_install_db is done + } + + // instantiate MySQL client + if (me.state) { + me.client = new mariasql({ + unixSocket: me.socketPath, + user: 'root', + password: me.state.rootPassword, + multiStatements: true + }); + } + + // spawn process + console.log(me.name+': spawning mysql: '+me.execPath); + me.proc = spawn(me.execPath, me.execOptions.concat(['--console']), {detached: true}); + me.pid = me.proc.pid; + me.status = 'online'; + + console.log(me.name+': spawned mysqld with pid '+me.pid); + + // add listeners to process + me.proc.on('exit', function (code) { + + if (code !== 0) { + me.status = 'offline'; + me.exitCode = code; + console.log(me.name+': exited with code: '+code); + } + }); + + me.proc.stdout.on('data', function (data) { + console.log(me.name+': stdout:\n\t' + data.toString().replace(/\n/g,'\n\t')); + }); + + me.proc.stderr.on('data', function (data) { + console.log(me.name+': stderr:\n\t' + data.toString().replace(/\n/g,'\n\t')); + + if (/^execvp\(\)/.test(data)) { + console.log('Failed to start child process.'); + me.status = 'offline'; + } + + if (/ready for connections/.test(data) && firstRun) { + me.secureInstallation(); + } + }); + + return true; +}; + + +exports.MysqlService.prototype.stop = function() { + var me = this; + + if (!me.pid) { + return false; + } + + // disconnect client + if (me.client && me.client.connected) { + me.client.end(); + console.log(me.name+': mysql client disconnected'); + } + + try { + console.log(me.name+': sending sigterm to '+me.pid); + process.kill(me.pid, 'SIGTERM'); + } catch (error) { + console.log(me.name+': failed to stop process: '+error); + return false; + } + + me.status = 'offline'; + me.pid = null; + return true; +}; + +exports.MysqlService.prototype.restart = function() { + var me = this, + now; + + if (!me.stop()) { + return false; + } + + // wait for pid to disappear before attempting start + process.stdout.write(me.name+': waiting for shutdown'); + while (fs.existsSync(me.pidPath)) { + process.stdout.write('.'); + now = new Date().getTime(); + + while (new Date().getTime() < now + 500) { + // do nothing + } + } + + process.stdout.write('\n'); + + return me.start(); +}; + + +exports.MysqlService.prototype.secureInstallation = function() { + var me = this, + sql = ''; + + console.log(me.name+': securing installation...'); + + // set root password + sql += 'UPDATE mysql.user SET Password=PASSWORD("'+me.state.rootPassword+'") WHERE User="root";'; + + // remove anonymous users + sql += 'DELETE FROM mysql.user WHERE User="";'; + + // delete remote roots + sql += 'DELETE FROM mysql.user WHERE User="root" AND Host NOT IN ("localhost", "127.0.0.1", "::1");'; + + // remove test database + sql += 'DROP DATABASE IF EXISTS test;'; + sql += 'DELETE FROM mysql.db WHERE Db="test" OR Db="test\\_%";'; + + // reload privs + sql += 'FLUSH PRIVILEGES;'; + + // open a temporary connection to the new non-secured installation + (new mariasql({ + unixSocket: me.socketPath, + user: 'root', + password: '', + multiStatements: true + })).query(sql, function(error) { + if (error) { + console.log(me.name+': failed to secure installation: ' + error); + } else { + console.log(me.name+': securing complete, mysql ready.'); + } + }); + +}; + + +exports.MysqlService.prototype.onSiteCreated = function(siteData, requestData, callbacks) { + var me = this, + sql = '', + dbConfig = { + socket: me.socketPath, + database: siteData.handle, + username: siteData.handle, + password: me.controller.sites.generatePassword() + }; + + console.log(me.name+': creating database `'+siteData.handle+'`'); + + sql += 'CREATE DATABASE IF NOT EXISTS `'+siteData.handle+'`;'; + sql += 'CREATE USER \''+siteData.handle+'\'@\'localhost\' IDENTIFIED BY \''+dbConfig.password+'\';'; + sql += 'GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER, LOCK TABLES ON `'+siteData.handle+'`.* TO \''+siteData.handle+'\'@\'localhost\';'; + sql += 'FLUSH PRIVILEGES;'; + + me.client.query(sql, function(error, results) { + if (error) { + console.log(me.name+': failed to setup database `'+siteData.handle+'`: '+error); + return; + } + + console.log(me.name+': database setup complete'); + me.controller.sites.updateSiteConfig(siteData.handle, { + mysql: dbConfig + }); + + // populate tables + me.createSkeletonTables(siteData, function() { + if (callbacks.databaseReady) { + callbacks.databaseReady(dbConfig, siteData, requestData); + } + }); + }); +}; + + + +exports.MysqlService.prototype.createSkeletonTables = function(siteData, callback) { + var me = this, + sql = ''; + + sql += 'USE `'+siteData.handle+'`;'; + + // Table: _e_file_collections + sql += 'CREATE TABLE `_e_file_collections` ('; + sql += '`ID` int(10) unsigned NOT NULL AUTO_INCREMENT'; + sql += ',`Site` ENUM(\'Local\',\'Remote\') NOT NULL'; + sql += ',`Handle` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL'; + sql += ',`Status` enum(\'Normal\',\'Deleted\') NOT NULL DEFAULT \'Normal\''; + sql += ',`Created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP'; + sql += ',`CreatorID` int(10) unsigned DEFAULT NULL'; + sql += ',`ParentID` int(10) unsigned DEFAULT NULL'; + sql += ',`PosLeft` int(10) unsigned DEFAULT NULL'; + sql += ',`PosRight` int(10) unsigned DEFAULT NULL'; + sql += ',PRIMARY KEY (`ID`)'; + sql += ',UNIQUE KEY `PosLeft` (`PosLeft`)'; + sql += ',UNIQUE KEY `SiteCollection` (`Site`,`ParentID`,`Handle`,`Status`)'; + sql += ') ENGINE=MyISAM DEFAULT CHARSET=utf8;'; + + // Table: _e_files + sql += 'CREATE TABLE `_e_files` ('; + sql += '`ID` int(10) unsigned NOT NULL AUTO_INCREMENT'; + sql += ',`CollectionID` int(10) unsigned NOT NULL'; + sql += ',`Handle` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL'; + sql += ',`Status` enum(\'Phantom\',\'Normal\',\'Deleted\') NOT NULL DEFAULT \'Phantom\''; + sql += ',`SHA1` char(40) DEFAULT NULL'; + sql += ',`Size` int(10) unsigned DEFAULT NULL'; + sql += ',`Type` varchar(255) DEFAULT NULL'; + sql += ',`Timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP'; + sql += ',`AuthorID` int(10) unsigned DEFAULT NULL'; + sql += ',`AncestorID` int(10) unsigned DEFAULT NULL'; + sql += ',PRIMARY KEY (`ID`)'; + sql += ',KEY `CollectionID` (`CollectionID`)'; + sql += ') ENGINE=MyISAM DEFAULT CHARSET=utf8;'; + + // run tables + me.client.query(sql, function(error, results) { + if (error) { + console.log(me.name+': failed to setup skeleton tables on `'+siteData.handle+'`: '+error); + return; + } + + console.log(me.name+': skeleton table schema setup'); + + callback(); + }); +}; diff --git a/kernel-lib/services/nginx.js b/kernel-lib/services/nginx.js index b42aba3..9c00ecd 100644 --- a/kernel-lib/services/nginx.js +++ b/kernel-lib/services/nginx.js @@ -14,31 +14,21 @@ exports.NginxService = function(name, controller, options) { // call parent constructor exports.NginxService.super_.apply(me, arguments); - // default options - me.options.configPath = me.options.configPath || controller.options.configDir + '/nginx.conf'; - me.options.execPath = me.options.execPath || '/usr/sbin/nginx'; - me.options.bindHost = me.options.bindHost || '127.0.0.1'; - me.options.bindPort = me.options.bindPort || 80; - me.options.runDir = me.options.runDir || controller.options.runDir + '/nginx'; - me.options.logsDir = me.options.logsDir || controller.options.logsDir + '/nginx'; - me.options.pidPath = me.options.pidPath || me.options.runDir + '/nginx.pid'; - me.options.errorLogPath = me.options.errorLogPath || me.options.logsDir + '/errors.log'; - me.options.miscConfigDir = me.options.miscConfigDir || (process.platform=='darwin'?'/usr/local/etc/nginx':'/etc/nginx'); - me.options.user = me.options.user || controller.options.user; - me.options.group = me.options.group || controller.options.group; - - // create required directories - if (!fs.existsSync(me.options.runDir)) { - fs.mkdirSync(me.options.runDir, '775'); - } + // initialize configuration + me.packagePath = options.packagePath; + me.execPath = me.packagePath + '/bin/nginx'; + + me.configPath = '/hab/svc/emergence-kernel/config/nginx'; + me.sitesConfigPath = '/hab/svc/emergence-kernel/var/config/nginx.sites'; + me.pidPath = '/hab/svc/emergence-kernel/var/run/nginx.pid'; + + me.bindHost = options.bindHost || '127.0.0.1'; + me.bindPort = options.bindPort || 80; - if (!fs.existsSync(me.options.logsDir)) { - fs.mkdirSync(me.options.logsDir, '775'); - } // check for existing master process - if (fs.existsSync(me.options.pidPath)) { - me.pid = parseInt(fs.readFileSync(me.options.pidPath, 'ascii')); + if (fs.existsSync(me.pidPath)) { + me.pid = parseInt(fs.readFileSync(me.pidPath, 'ascii')); console.log(me.name+': found existing PID: '+me.pid); me.status = 'online'; } @@ -57,16 +47,16 @@ util.inherits(exports.NginxService, require('./abstract.js').AbstractService); exports.NginxService.prototype.start = function() { var me = this; - console.log(me.name+': spawning daemon: '+me.options.execPath); + console.log(me.name+': spawning daemon: '+me.execPath); if (me.pid) { console.log(me.name+': already running with PID '+me.pid); return false; } - this.writeConfig(); + this.writeSitesConfig(); - me.proc = spawn(me.options.execPath, ['-c', me.options.configPath]); + me.proc = spawn(me.execPath, ['-c', me.configPath]); me.proc.on('exit', function (code) { @@ -77,16 +67,16 @@ exports.NginxService.prototype.start = function() { } // look for pid - if (fs.existsSync(me.options.pidPath)) { - me.pid = parseInt(fs.readFileSync(me.options.pidPath, 'ascii')); + if (fs.existsSync(me.pidPath)) { + me.pid = parseInt(fs.readFileSync(me.pidPath, 'ascii')); console.log(me.name+': found new PID: '+me.pid); me.status = 'online'; } else { console.log(me.name+': failed to find pid after launching, waiting 1000ms and trying again...'); setTimeout(function() { - if (fs.existsSync(me.options.pidPath)) { - me.pid = parseInt(fs.readFileSync(me.options.pidPath, 'ascii')); + if (fs.existsSync(me.pidPath)) { + me.pid = parseInt(fs.readFileSync(me.pidPath, 'ascii')); console.log(me.name+': found new PID: '+me.pid); me.status = 'online'; } else { @@ -143,7 +133,7 @@ exports.NginxService.prototype.restart = function() { return false; } - this.writeConfig(); + this.writeSitesConfig(); try { process.kill(me.pid, 'SIGHUP'); @@ -158,14 +148,14 @@ exports.NginxService.prototype.restart = function() { }; -exports.NginxService.prototype.writeConfig = function() { - fs.writeFileSync(this.options.configPath, this.makeConfig()); +exports.NginxService.prototype.writeSitesConfig = function() { + fs.writeFileSync(this.sitesConfigPath, this.makeSitesConfig()); }; -exports.NginxService.prototype.makeConfig = function() { +exports.NginxService.prototype.makeSitesConfig = function() { var me = this, - phpSocketPath = me.controller.services['php'].options.socketPath, - phpBootstrapDir = me.controller.services['php'].options.bootstrapDir, + phpSocketPath = me.controller.services['php'].socketPath, + phpBootstrapDir = me.controller.services['php'].bootstrapDir, config = []; // format socket path @@ -173,91 +163,10 @@ exports.NginxService.prototype.makeConfig = function() { phpSocketPath = 'unix:'+phpSocketPath; } - // configure top-level options - config.push( - 'user '+me.options.user+' '+me.options.group+';', - 'worker_processes auto;', - 'pid '+me.options.pidPath+';', - 'error_log '+me.options.errorLogPath+' info;' - ); - - - // configure connection processing - config.push( - 'events {', - ' worker_connections 1024;' - ); - - if (process.platform == 'linux') { - config.push(' use epoll;'); - } - - config.push( - '}' // end events block - ); - - - // configure http options - config.push( - 'http {', - ' include '+me.options.miscConfigDir+'/mime.types;', - ' default_type application/octet-stream;', - - ' log_format main', - ' \'$host $remote_addr - $remote_user [$time_local] \'', - ' \'"$request" $status $bytes_sent \'', - ' \'"$http_referer" "$http_user_agent" \'', - ' \'"$gzip_ratio"\';', - - ' client_header_timeout 10m;', - ' client_body_timeout 10m;', - ' send_timeout 10m;', - - ' connection_pool_size 256;', - ' client_max_body_size 200m;', - ' client_body_buffer_size 128k;', - ' client_header_buffer_size 1k;', - ' large_client_header_buffers 8 512k;', - ' request_pool_size 4k;', - ' server_names_hash_bucket_size 1024;', - ' types_hash_max_size 2048;', - - ' gzip on;', - ' gzip_min_length 1100;', - ' gzip_buffers 4 8k;', - ' gzip_types text/plain text/css text/x-scss text/x-html-template text/x-component text/xml application/xml application/javascript application/json application/php application/atom+xml application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/xhtml+xml font/opentype image/svg+xml image/x-icon;', - - ' output_buffers 1 32k;', - ' postpone_output 1460;', - - ' sendfile on;', - ' tcp_nopush on;', - ' tcp_nodelay on;', - - ' keepalive_timeout 75 20;', - - ' ignore_invalid_headers on;', - - ' index index.php;', - - ' fastcgi_index index.php;', - ' fastcgi_read_timeout 6h;', - ' fastcgi_buffers 32 64k;', - - ' server_tokens off;' -/* - - ' server {', - ' server_name _;', - ' access_log /emergence/logs/access.log main;', - ' error_log /emergence/logs/error.log info;', - ' }', -*/ - ); - + // configure each site _.each(me.controller.sites.sites, function(site, handle) { var hostnames = site.hostnames.slice(), - siteDir = me.controller.sites.options.sitesDir+'/'+handle, + siteDir = me.controller.sites.sitesDir+'/'+handle, logsDir = siteDir+'/logs', siteConfig = [], sslHostnames, sslHostname; @@ -277,9 +186,9 @@ exports.NginxService.prototype.makeConfig = function() { ' error_log '+logsDir+'/error.log notice;', ' location / {', - ' include '+me.options.miscConfigDir+'/fastcgi_params;', - ' fastcgi_param HTTPS $php_https;', ' fastcgi_pass '+phpSocketPath+';', + ' include '+me.packagePath+'/config/fastcgi_params;', + ' fastcgi_param HTTPS $php_https;', ' fastcgi_param PATH_INFO $fastcgi_script_name;', ' fastcgi_param SITE_ROOT '+siteDir+';', ' fastcgi_param SCRIPT_FILENAME '+phpBootstrapDir+'/web.php;', @@ -290,7 +199,7 @@ exports.NginxService.prototype.makeConfig = function() { // append config config.push( ' server {', - ' listen '+me.options.bindHost+':'+me.options.bindPort+';', + ' listen '+me.bindHost+':'+me.bindPort+';', ' server_name '+hostnames.join(' ')+';', ' set $php_https "";' ); @@ -316,7 +225,7 @@ exports.NginxService.prototype.makeConfig = function() { for (sslHostname in sslHostnames) { config.push( ' server {', - ' listen '+me.options.bindHost+':443;', + ' listen '+me.bindHost+':443;', ' server_name '+sslHostname+';', ' set $php_https on;', @@ -333,10 +242,6 @@ exports.NginxService.prototype.makeConfig = function() { } }); - config.push( - '}' // end http block - ); - return config.join('\n'); }; diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js index 008e021..29d142a 100644 --- a/kernel-lib/services/php-fpm.js +++ b/kernel-lib/services/php-fpm.js @@ -15,31 +15,21 @@ exports.PhpFpmService = function(name, controller, options) { // call parent constructor exports.PhpFpmService.super_.apply(me, arguments); - // default options - me.options.bootstrapDir = me.options.bootstrapDir || path.resolve(__dirname, '../../php-bootstrap'); - me.options.configPath = me.options.configPath || controller.options.configDir + '/php-fpm.conf'; - me.options.execPath = me.options.execPath || '/usr/bin/php-fpm'; - me.options.statScripts = me.options.statScripts || false; - me.options.runDir = me.options.runDir || controller.options.runDir + '/php-fpm'; - me.options.logsDir = me.options.logsDir || controller.options.logsDir + '/php-fpm'; - me.options.pidPath = me.options.pidPath || me.options.runDir + '/php-fpm.pid'; - me.options.socketPath = me.options.socketPath || me.options.runDir + '/php-fpm.sock'; - me.options.errorLogPath = me.options.errorLogPath || me.options.logsDir + '/errors.log'; - me.options.user = me.options.user || controller.options.user; - me.options.group = me.options.group || controller.options.group; - - // create required directories - if (!fs.existsSync(me.options.runDir)) { - fs.mkdirSync(me.options.runDir, '775'); - } + // initialize configuration + me.packagePath = options.packagePath; + me.execPath = me.packagePath + '/sbin/php-fpm'; + + me.configPath = '/hab/svc/emergence-kernel/config/php-fpm'; + me.pidPath = '/hab/svc/emergence-kernel/var/run/php-fpm.pid'; + me.socketPath = '/hab/svc/emergence-kernel/var/run/php-fpm.sock'; + me.errorLogPath = '/hab/svc/emergence-kernel/var/log/php-fpm.err'; + me.dataDir = '/hab/svc/emergence-kernel/var/mariadb'; + me.bootstrapDir = '/hab/svc/emergence-kernel/static/php-bootstrap'; - if (!fs.existsSync(me.options.logsDir)) { - fs.mkdirSync(me.options.logsDir, '775'); - } // check for existing master process - if (fs.existsSync(me.options.pidPath)) { - me.pid = parseInt(fs.readFileSync(me.options.pidPath, 'ascii')); + if (fs.existsSync(me.pidPath)) { + me.pid = parseInt(fs.readFileSync(me.pidPath, 'ascii')); console.log(me.name+': found existing PID: '+me.pid); me.status = 'online'; } @@ -55,16 +45,14 @@ util.inherits(exports.PhpFpmService, require('./abstract.js').AbstractService); exports.PhpFpmService.prototype.start = function() { var me = this; - console.log(me.name+': spawning daemon: '+me.options.execPath); + console.log(me.name+': spawning daemon: '+me.execPath); if (me.pid) { console.log(me.name+': already running with PID '+me.pid); return false; } - this.writeConfig(); - - me.proc = spawn(me.options.execPath, ['--fpm-config', me.options.configPath]); + me.proc = spawn(me.execPath, ['--fpm-config', me.configPath]); me.proc.on('exit', function (code) { @@ -75,8 +63,8 @@ exports.PhpFpmService.prototype.start = function() { } // look for pid - if (fs.existsSync(me.options.pidPath)) { - me.pid = parseInt(fs.readFileSync(me.options.pidPath, 'ascii')); + if (fs.existsSync(me.pidPath)) { + me.pid = parseInt(fs.readFileSync(me.pidPath, 'ascii')); console.log(me.name+': found new PID: '+me.pid); me.status = 'online'; } else { @@ -131,8 +119,6 @@ exports.PhpFpmService.prototype.restart = function() { return false; } - this.writeConfig(); - try { process.kill(me.pid, 'SIGUSR2'); } catch (error) { @@ -146,67 +132,17 @@ exports.PhpFpmService.prototype.restart = function() { }; -exports.PhpFpmService.prototype.writeConfig = function() { - fs.writeFileSync(this.options.configPath, this.makeConfig()); -}; - -exports.PhpFpmService.prototype.makeConfig = function() { - var me = this, - config = []; - - config.push( - '[global]', - 'pid = '+me.options.pidPath, - 'error_log = '+me.options.errorLogPath - ); - - config.push( - '[www]', - 'user = '+me.options.user, - 'group = '+me.options.group, - 'catch_workers_output = on', - 'listen = '+me.options.socketPath, - 'listen.owner = '+me.options.user, - 'listen.group = '+me.options.group, - 'pm = dynamic', - 'pm.max_children = '+(me.options.maxClients||50), - 'pm.start_servers = '+(me.options.startServers||5), - 'pm.min_spare_servers = '+(me.options.minSpareServers||1), - 'pm.max_spare_servers = '+Math.round((me.options.maxClients||50)/(me.options.startServers||5)) - ); - - if (me.options.statusPath) { - config.push('pm.status_path = '+me.options.statusPath); - } - - config.push( - 'php_admin_flag[short_open_tag] = on', - 'php_admin_value[apc.shm_size] = 512M', - 'php_admin_value[apc.shm_segments] = 1', - 'php_admin_value[apc.slam_defense] = 0', - 'php_admin_value[apc.stat] = '+(me.options.statScripts?'1':'0'), - 'php_admin_value[opcache.validate_timestamps] = '+(me.options.statScripts?'1':'0'), - 'php_admin_value[upload_max_filesize] = '+(me.options.uploadMaxSize ? me.options.uploadMaxSize : '200M'), - 'php_admin_value[post_max_size] = '+(me.options.postMaxSize ? me.options.postMaxSize : '200M'), - 'php_admin_value[memory_limit] = '+(me.options.memoryLimit ? me.options.memoryLimit : '200M'), - 'php_admin_value[error_reporting] = '+(me.options.errorReporting ? me.options.errorReporting : 'E_ALL & ~E_NOTICE'), - 'php_admin_value[date.timezone] = '+(me.options.defaultTimezone ? me.options.defaultTimezone : 'America/New_York') - ); - - return config.join('\n'); -}; - exports.PhpFpmService.prototype.onSiteUpdated = function(siteData) { var me = this, - siteRoot = me.controller.sites.options.sitesDir + '/' + siteData.handle, + siteRoot = me.controller.sites.sitesDir + '/' + siteData.handle, phpClient; console.log(me.name+': clearing config cache for '+siteRoot); // Connect to FPM worker pool phpClient = new phpfpm({ - sockFile: me.options.socketPath, - documentRoot: me.options.bootstrapDir + '/' + sockFile: me.socketPath, + documentRoot: me.bootstrapDir + '/' }); // Clear cached site.json diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js index 956d58f..488dc60 100644 --- a/kernel-lib/sites.js +++ b/kernel-lib/sites.js @@ -6,7 +6,6 @@ var _ = require('underscore'), events = require('events'), posix = require('posix'), spawn = require('child_process').spawn, - hostile = require('hostile'), phpShellScript = path.resolve(__dirname, '../bin/shell'); @@ -21,25 +20,19 @@ exports.Sites = function(config) { // call events constructor events.EventEmitter.call(me); - // initialize options and apply defaults - me.options = options || {}; - me.options.sitesDir = me.options.sitesDir || '/emergence/sites'; - me.options.dataUID = me.options.dataUID || posix.getpwnam(config.user).uid; - me.options.dataGID = me.options.dataGID || posix.getgrnam(config.group).gid; - me.options.dataMode = me.options.dataMode || '775'; + // initialize configuration + me.sitesDir = '/hab/svc/emergence-kernel/data/sites'; + me.dataUid = posix.getpwnam('hab').uid; + me.dataGid = posix.getgrnam('hab').gid; - // create required directories - if (!fs.existsSync(me.options.sitesDir)) { - fs.mkdirSync(me.options.sitesDir, '775'); - } // load sites - console.log('Loading sites from '+me.options.sitesDir+'...'); + console.log('Loading sites from '+me.sitesDir+'...'); me.sites = {}; - _.each(fs.readdirSync(me.options.sitesDir), function(handle) { + _.each(fs.readdirSync(me.sitesDir), function(handle) { try { - me.sites[handle] = JSON.parse(fs.readFileSync(me.options.sitesDir+'/'+handle+'/site.json', 'ascii')); + me.sites[handle] = JSON.parse(fs.readFileSync(me.sitesDir+'/'+handle+'/site.json', 'ascii')); me.sites[handle].handle = handle; console.log('-Loaded: '+me.sites[handle].primary_hostname); } catch (error) { @@ -86,7 +79,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) { } var handle = request.path[1], - siteDir = me.options.sitesDir + '/' + handle, + siteDir = me.sitesDir + '/' + handle, siteConfigPath = siteDir + '/site.json', params = JSON.parse(request.content), siteData; @@ -145,7 +138,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) { console.log('Executing shell post for ' + request.path[1] + ':'); console.log(requestData); - phpProc = spawn(phpShellScript, [request.path[1]]); + phpProc = spawn(phpShellScript, [request.path[1]], { uid: me.dataUid, gid: me.dataGid }); phpProcInitialized = false; phpProc.stderr.on('data', function(data) { @@ -178,7 +171,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) { console.log('Creating developer for ' + request.path[1] + ':' + phpShellScript); response.writeHead(200, {'Content-Type':'application/json'}); - phpProc = spawn(phpShellScript, [request.path[1], '--stdin']); + phpProc = spawn(phpShellScript, [request.path[1], '--stdin'], { uid: me.dataUid, gid: me.dataGid }); phpProc.stdout.on('data', function(data) { console.log('php-cli stdout: ' + data); @@ -220,16 +213,12 @@ exports.Sites.prototype.handleRequest = function(request, response, server) { cfgResult = me.writeSiteConfig(requestData); if (cfgResult.isNew) { - // write primary hostname to /etc/hosts - hostile.set('127.0.0.1', cfgResult.site.primary_hostname); - console.log('added ' + cfgResult.site.primary_hostname + ' to /etc/hosts'); - // notify plugins me.emit('siteCreated', cfgResult.site, requestData, { databaseReady: function() { // execute onSiteCreated within site's container console.log('Executing Site::onSiteCreated() via php-cli'); - phpProc = spawn(phpShellScript, [cfgResult.site.handle]); + phpProc = spawn(phpShellScript, [cfgResult.site.handle], { uid: me.dataUid, gid: me.dataGid }); phpProc.stdout.on('data', function(data) { console.log('php-cli stdout: ' + data); }); phpProc.stderr.on('data', function(data) { console.log('php-cli stderr: ' + data); }); @@ -307,7 +296,7 @@ exports.Sites.prototype.writeSiteConfig = function(requestData) { } // create site directory - var siteDir = me.options.sitesDir+'/'+siteData.handle, + var siteDir = me.sitesDir+'/'+siteData.handle, dataDir = siteDir + '/data', siteDataDir = siteDir + '/site-data', siteConfigPath = siteDir + '/site.json'; @@ -318,13 +307,13 @@ exports.Sites.prototype.writeSiteConfig = function(requestData) { } if (!fs.existsSync(dataDir)) { - fs.mkdirSync(dataDir, me.options.dataMode); - fs.chownSync(dataDir, me.options.dataUID, me.options.dataGID); + fs.mkdirSync(dataDir, '775'); + fs.chownSync(dataDir, me.dataUid, me.dataGid); } if (!fs.existsSync(siteDataDir)) { - fs.mkdirSync(siteDataDir, me.options.dataMode); - fs.chownSync(siteDataDir, me.options.dataUID, me.options.dataGID); + fs.mkdirSync(siteDataDir, '775'); + fs.chownSync(siteDataDir, me.dataUid, me.dataGid); } // write site config to file @@ -341,7 +330,7 @@ exports.Sites.prototype.writeSiteConfig = function(requestData) { exports.Sites.prototype.updateSiteConfig = function(handle, changes) { var me = this, - siteDir = me.options.sitesDir+'/'+handle, + siteDir = me.sitesDir+'/'+handle, filename = siteDir+'/site.json', siteData = this.sites[handle], create_user; diff --git a/package.json b/package.json index cebbe24..eb80d45 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "node-static": "0.6.x", "http-auth": "1.2.7", "posix": "^4.0.2", - "hostile": "^1.0.2", "shelljs": "^0.5.3", "semver": "^5.1.0", "underscore-cli": "^0.2.19", diff --git a/php-bootstrap/mail.php b/php-bootstrap/mail.php index 92cffbe..b3e00a0 100644 --- a/php-bootstrap/mail.php +++ b/php-bootstrap/mail.php @@ -10,26 +10,26 @@ // get hostmap // TODO: have the kernel maintain this file -$hostmapPath = '/emergence/services/hostmap.inc.php'; +$hostmapPath = '/hab/svc/emergence-kernel/var/hostmap.inc.php'; if (is_readable($hostmapPath)) { $hostmap = require($hostmapPath); } else { $hostmap = []; - - foreach (glob('/emergence/sites/*', GLOB_ONLYDIR) AS $sitePath) { + + foreach (glob('/hab/svc/emergence-kernel/data/sites/*', GLOB_ONLYDIR) AS $sitePath) { $configPath = "$sitePath/site.json"; if (!is_readable($configPath)) { continue; } - + $config = @json_decode(file_get_contents($configPath), true); - + if (!$config) { continue; } - + $hostnames = array_unique(array_merge([$config['primary_hostname']], $config['hostnames'])); - + foreach ($hostnames AS $hostname) { $hostmap['/^' . str_replace('\\*', '.*', preg_quote($hostname)) . '$/i'] = basename($sitePath); } @@ -58,7 +58,7 @@ // bootstrap emergence require('bootstrap.inc.php'); Site::$debug = true; -Site::initialize("/emergence/sites/$siteHandle", $hostname); +Site::initialize("/hab/svc/emergence-kernel/data/sites/$siteHandle", $hostname); // delegate remainder of request to email router From 41cd8758d7ff29cba9f9fe4717774caa75f7c950 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 16:56:24 -0400 Subject: [PATCH 25/56] Restore nginx defaults to do_build --- service-plans/nginx/config/nginx.conf | 9 --------- service-plans/nginx/plan.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/service-plans/nginx/config/nginx.conf b/service-plans/nginx/config/nginx.conf index 7410ac0..fd8526e 100644 --- a/service-plans/nginx/config/nginx.conf +++ b/service-plans/nginx/config/nginx.conf @@ -1,15 +1,6 @@ -pid {{pkg.svc_var_path}}/nginx.pid; -lock_file {{pkg.svc_var_path}}/nginx.lock; - include worker.include; http { - client_body_temp_path {{pkg.svc_var_path}}/client-body; - proxy_temp_path {{pkg.svc_var_path}}/proxy; - fastcgi_temp_path {{pkg.svc_var_path}}/fastcgi; - scgi_temp_path {{pkg.svc_var_path}}/scgi; - uwsgi_temp_path {{pkg.svc_var_path}}/uwsgi; - include http.include; server { diff --git a/service-plans/nginx/plan.sh b/service-plans/nginx/plan.sh index 0c30feb..2970c0e 100644 --- a/service-plans/nginx/plan.sh +++ b/service-plans/nginx/plan.sh @@ -20,11 +20,19 @@ pkg_exposes=(port) do_build() { ./configure --prefix="$pkg_prefix" \ + --conf-path="$pkg_svc_config_path/nginx.conf" \ --sbin-path="$pkg_prefix/bin/nginx" \ + --pid-path="$pkg_svc_var_path/nginx.pid" \ + --lock-path="$pkg_svc_var_path/nginx.lock" \ --user=hab \ --group=hab \ --http-log-path=/dev/stdout \ --error-log-path=stderr \ + --http-client-body-temp-path="$pkg_svc_var_path/client-body" \ + --http-proxy-temp-path="$pkg_svc_var_path/proxy" \ + --http-fastcgi-temp-path="$pkg_svc_var_path/fastcgi" \ + --http-scgi-temp-path="$pkg_svc_var_path/scgi" \ + --http-uwsgi-temp-path="$pkg_svc_var_path/uwsgi" \ --with-ipv6 \ --with-pcre \ --with-pcre-jit \ From eadf8df4f789af60e16215a39a54989d7579244b Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 17:09:21 -0400 Subject: [PATCH 26/56] Tweak nginx config --- habitat/config/nginx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/habitat/config/nginx b/habitat/config/nginx index ad80380..e5462f5 100644 --- a/habitat/config/nginx +++ b/habitat/config/nginx @@ -1,18 +1,20 @@ -user {{pkg.svc_user}}; -worker_processes auto; -pid {{pkg.svc_var_path}}/run/nginx.pid; -error_log {{pkg.svc_var_path}}/log/nginx.err info; +user hab hab; +worker_processes auto; events { worker_connections 1024; use epoll; } +pid {{pkg.svc_var_path}}/run/nginx.pid; +lock_file {{pkg.svc_var_path}}/run/nginx.lock; +error_log {{pkg.svc_var_path}}/log/nginx.err info; + http { client_body_temp_path {{pkg.svc_var_path}}/tmp/nginx/client-body; - fastcgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/fastcgi; proxy_temp_path {{pkg.svc_var_path}}/tmp/nginx/proxy; - scgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/scgi_temp_path; + fastcgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/fastcgi; + scgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/scgi; uwsgi_temp_path {{pkg.svc_var_path}}/tmp/nginx/uwsgi; include {{pkgPathFor "emergence/nginx"}}/config/mime.types; From 4441ec47d670cc06cee83f561878124fd34ee2f5 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 17:09:40 -0400 Subject: [PATCH 27/56] Ensure rX permissions on svc/static tree --- habitat/hooks/init | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/habitat/hooks/init b/habitat/hooks/init index 79d6c27..b8dc771 100755 --- a/habitat/hooks/init +++ b/habitat/hooks/init @@ -35,5 +35,8 @@ done popd > /dev/null # initialize static asset links -ln -sf "{{pkg.path}}/app/php-bootstrap" "{{pkg.svc_static_path}}/" -ln -sf "{{pkg.path}}/app/kernel-www" "{{pkg.svc_static_path}}/" \ No newline at end of file +pushd "{{pkg.svc_static_path}}" > /dev/null +chmod g+rX -R . +ln -sf "{{pkg.path}}/app/php-bootstrap" +ln -sf "{{pkg.path}}/app/kernel-www" +popd > /dev/null \ No newline at end of file From 5dc20feabda4fee35dfbdf8e287b5e615ace713a Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 17:42:34 -0400 Subject: [PATCH 28/56] Pass bindPort into kernel configuration --- habitat/config/config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/habitat/config/config.json b/habitat/config/config.json index 988c768..bc3c4f1 100644 --- a/habitat/config/config.json +++ b/habitat/config/config.json @@ -8,7 +8,8 @@ "web": { "type": "nginx", "packagePath": "{{pkgPathFor "emergence/nginx"}}", - "bindHost": "{{cfg.web.host}}" + "bindHost": "{{cfg.web.host}}", + "bindPort": "{{cfg.web.port}}" }, "sql": { "type": "mariadb", From 87ef825ac37791708defb50b8d9b349ccf9a59d3 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 18:06:21 -0400 Subject: [PATCH 29/56] Add default site with generic welcome message --- default-site/index.html | 9 +++++++++ habitat/config/nginx | 13 +++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 default-site/index.html diff --git a/default-site/index.html b/default-site/index.html new file mode 100644 index 0000000..1b25d78 --- /dev/null +++ b/default-site/index.html @@ -0,0 +1,9 @@ + + + emergence + + +

Welcome to emergence!

+

No site is configure (yet) at this address. If you are the webmaster, visit the emergence control panel and create a site

+ + \ No newline at end of file diff --git a/habitat/config/nginx b/habitat/config/nginx index e5462f5..7ae3810 100644 --- a/habitat/config/nginx +++ b/habitat/config/nginx @@ -52,8 +52,17 @@ http { fastcgi_buffers 32 64k; server_tokens off; - index index.php; - fastcgi_index index.php; + server { + listen {{cfg.web.host}}:{{cfg.web.port}}; + server_name localhost; + access_log off; + + location / { + root {{pkg.path}}/app/default-site; + index index.html; + error_page 404 index.html; + } + } include {{pkg.svc_var_path}}/config/nginx.sites; } \ No newline at end of file From 6c0e1f5b8633cbf8c7ab6f8407fe9482ba074fd4 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 18:50:30 -0400 Subject: [PATCH 30/56] Only enforce origin for CORS preflight requests --- php-bootstrap/lib/Site.class.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/php-bootstrap/lib/Site.class.php b/php-bootstrap/lib/Site.class.php index 626d0e2..55a95b1 100644 --- a/php-bootstrap/lib/Site.class.php +++ b/php-bootstrap/lib/Site.class.php @@ -165,19 +165,19 @@ public static function handleRequest() } // handle CORS headers - if (isset($_SERVER['HTTP_ORIGIN'])) { - $hostname = strtolower(parse_url($_SERVER['HTTP_ORIGIN'], PHP_URL_HOST)); - if ($hostname == strtolower(static::$hostname) || static::$permittedOrigins == '*' || in_array($hostname, static::$permittedOrigins)) { - header('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']); - header('Access-Control-Allow-Credentials: true'); - //header('Access-Control-Max-Age: 86400') - } else { - header('HTTP/1.1 403 Forbidden'); - exit(); + if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS' && isset($_SERVER['HTTP_ACCESS_CONTROL_REQUEST_METHOD'])) { + if (isset($_SERVER['HTTP_ORIGIN'])) { + $hostname = strtolower(parse_url($_SERVER['HTTP_ORIGIN'], PHP_URL_HOST)); + if ($hostname == strtolower(static::$hostname) || static::$permittedOrigins == '*' || in_array($hostname, static::$permittedOrigins)) { + header('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']); + header('Access-Control-Allow-Credentials: true'); + //header('Access-Control-Max-Age: 86400') + } else { + header('HTTP/1.1 403 Forbidden'); + die('CORS access denied for given origin'); + } } - } - if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS' && isset($_SERVER['HTTP_ACCESS_CONTROL_REQUEST_METHOD'])) { header('Access-Control-Allow-Methods: ' . $_SERVER['HTTP_ACCESS_CONTROL_REQUEST_METHOD']); if (isset($_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS'])) { From a27b0ca8f8f47076393bdded1ad53855c5f140b3 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 18:50:50 -0400 Subject: [PATCH 31/56] Enable gettext extension for PHP --- service-plans/php/plan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/service-plans/php/plan.sh b/service-plans/php/plan.sh index 9eb0167..fad071c 100644 --- a/service-plans/php/plan.sh +++ b/service-plans/php/plan.sh @@ -66,6 +66,7 @@ do_build() { --enable-fpm \ --with-fpm-user=hab \ --with-fpm-group=hab \ + --with-gettext="$(pkg_path_for glibc)" \ --enable-apcu \ --enable-mbstring \ --enable-opcache \ From af85cc00a2f3a9cac673cbe152525810e2b1af51 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 19:13:00 -0400 Subject: [PATCH 32/56] Copy php5 plan from habitat core-plans --- service-plans/php5/plan.sh | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 service-plans/php5/plan.sh diff --git a/service-plans/php5/plan.sh b/service-plans/php5/plan.sh new file mode 100644 index 0000000..3bc1dea --- /dev/null +++ b/service-plans/php5/plan.sh @@ -0,0 +1,59 @@ +pkg_name=php5 +pkg_distname=php +pkg_origin=core +pkg_version=5.6.23 +pkg_maintainer="The Habitat Maintainers " +pkg_license=('PHP-3.01') +pkg_upstream_url=http://php.net/ +pkg_description="PHP is a popular general-purpose scripting language that is especially suited to web development." +pkg_source=https://php.net/get/${pkg_distname}-${pkg_version}.tar.bz2/from/this/mirror +pkg_filename=${pkg_distname}-${pkg_version}.tar.bz2 +pkg_dirname=${pkg_distname}-${pkg_version} +pkg_shasum=facd280896d277e6f7084b60839e693d4db68318bfc92085d3dc0251fd3558c7 +pkg_deps=( + core/coreutils + core/curl + core/glibc + core/libxml2 + core/openssl + core/zlib +) +pkg_build_deps=( + core/bison2 + core/gcc + core/make + core/re2c +) +pkg_bin_dirs=(bin sbin) +pkg_lib_dirs=(lib) +pkg_include_dirs=(include) +pkg_interpreters=(bin/php) + +do_build() { + ./configure --prefix="$pkg_prefix" \ + --enable-exif \ + --enable-fpm \ + --enable-mbstring \ + --enable-opcache \ + --with-curl="$(pkg_path_for curl)" \ + --with-libxml-dir="$(pkg_path_for libxml2)" \ + --with-openssl="$(pkg_path_for openssl)" \ + --with-xmlrpc \ + --with-zlib="$(pkg_path_for zlib)" + make +} + +do_install() { + do_default_install + + # Modify PHP-FPM config so it will be able to run out of the box. To run a real + # PHP-FPM application you would want to supply your own config with + # --fpm-config . + mv "$pkg_prefix/etc/php-fpm.conf.default" "$pkg_prefix/etc/php-fpm.conf" + # Run as the hab user by default, as it's more likely to exist than nobody. + sed -i "s/nobody/hab/g" "$pkg_prefix/etc/php-fpm.conf" +} + +do_check() { + make test +} From c37be9b1cbd12255a868347cf2ad2e7ed8ebacd6 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 19:13:23 -0400 Subject: [PATCH 33/56] Change php5 plan origin to emergence --- service-plans/php5/plan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service-plans/php5/plan.sh b/service-plans/php5/plan.sh index 3bc1dea..687138b 100644 --- a/service-plans/php5/plan.sh +++ b/service-plans/php5/plan.sh @@ -1,6 +1,6 @@ pkg_name=php5 pkg_distname=php -pkg_origin=core +pkg_origin=emergence pkg_version=5.6.23 pkg_maintainer="The Habitat Maintainers " pkg_license=('PHP-3.01') From a9873c258df4920fc2917d1d068688836fc736d9 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 19:24:43 -0400 Subject: [PATCH 34/56] Add needed extensions to php5 --- service-plans/php5/plan.sh | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/service-plans/php5/plan.sh b/service-plans/php5/plan.sh index 687138b..b3e0d88 100644 --- a/service-plans/php5/plan.sh +++ b/service-plans/php5/plan.sh @@ -15,10 +15,13 @@ pkg_deps=( core/curl core/glibc core/libxml2 + core/libjpeg-turbo + core/libpng core/openssl core/zlib ) pkg_build_deps=( + core/autoconf core/bison2 core/gcc core/make @@ -29,15 +32,52 @@ pkg_lib_dirs=(lib) pkg_include_dirs=(include) pkg_interpreters=(bin/php) +apcu_version=4.0.11 +apcu_source=https://github.com/krakjoe/apcu/archive/v${apcu_version}.tar.gz +apcu_filename=apcu-${apcu_version}.tar.gz +apcu_shasum=bf1d78d4211c6fde6d29bfa71947999efe0ba0c50bdc99af3f646e080f74e3a4 +apcu_dirname=apcu-${apcu_version} + +do_download() { + do_default_download + + download_file $apcu_source $apcu_filename $apcu_shasum +} + +do_verify() { + do_default_verify + + verify_file $apcu_filename $apcu_shasum +} + +do_unpack() { + do_default_unpack + + unpack_file $apcu_filename + mv "$HAB_CACHE_SRC_PATH/$apcu_dirname" "$HAB_CACHE_SRC_PATH/$pkg_dirname/ext/apcu" +} + do_build() { + rm aclocal.m4 + ./buildconf --force + ./configure --prefix="$pkg_prefix" \ --enable-exif \ --enable-fpm \ + --with-fpm-user=hab \ + --with-fpm-group=hab \ + --with-gettext="$(pkg_path_for glibc)" \ + --enable-apcu \ --enable-mbstring \ --enable-opcache \ + --with-mysqli=mysqlnd \ + --with-pdo-mysql=mysqlnd \ --with-curl="$(pkg_path_for curl)" \ + --with-gd \ + --with-jpeg-dir="$(pkg_path_for libjpeg-turbo)" \ --with-libxml-dir="$(pkg_path_for libxml2)" \ --with-openssl="$(pkg_path_for openssl)" \ + --with-png-dir="$(pkg_path_for libpng)" \ --with-xmlrpc \ --with-zlib="$(pkg_path_for zlib)" make From 5ba9b114bd9bf1b986d140fb6db221493e0b865c Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 19:24:58 -0400 Subject: [PATCH 35/56] Switch kernel to use php5 --- habitat/config/config.json | 2 +- habitat/plan.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/habitat/config/config.json b/habitat/config/config.json index bc3c4f1..d4e9ec1 100644 --- a/habitat/config/config.json +++ b/habitat/config/config.json @@ -17,7 +17,7 @@ }, "php": { "type": "php-fpm", - "packagePath": "{{pkgPathFor "emergence/php"}}" + "packagePath": "{{pkgPathFor "emergence/php5"}}" } } } \ No newline at end of file diff --git a/habitat/plan.sh b/habitat/plan.sh index 366a194..eadfbc9 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -14,5 +14,5 @@ pkg_build_deps=( pkg_deps=( emergence/nginx emergence/mariadb - emergence/php + emergence/php5 ) \ No newline at end of file From bade584248e9c62bde2ee4ab569a1f3be312bc3e Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 20:03:48 -0400 Subject: [PATCH 36/56] Pass timezone to kernel config.json and use in bin/shell --- bin/shell | 2 ++ habitat/config/config.json | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/shell b/bin/shell index 9de4013..48819c0 100755 --- a/bin/shell +++ b/bin/shell @@ -29,6 +29,8 @@ siteDir="/hab/svc/emergence-kernel/data/sites/$siteHandle" autoPrependScript=$(mktemp); echo " $autoPrependScript +echo "\$serverConfig = json_decode(file_get_contents('/hab/svc/emergence-kernel/config/config.json'), true);" >> $autoPrependScript +echo "date_default_timezone_set(\$serverConfig['services']['php']['timezone']);" >> $autoPrependScript echo "require('`dirname $DIR`/php-bootstrap/bootstrap.inc.php');" >> $autoPrependScript echo "Site::initialize('$siteDir');" >> $autoPrependScript diff --git a/habitat/config/config.json b/habitat/config/config.json index d4e9ec1..f165af2 100644 --- a/habitat/config/config.json +++ b/habitat/config/config.json @@ -17,7 +17,8 @@ }, "php": { "type": "php-fpm", - "packagePath": "{{pkgPathFor "emergence/php5"}}" + "packagePath": "{{pkgPathFor "emergence/php5"}}", + "timezone": "{{cfg.php.timezone}}" } } } \ No newline at end of file From 76fba33b57780499dee4228ef7e5cfd47e85f2c0 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 28 May 2017 20:04:02 -0400 Subject: [PATCH 37/56] Ensure config directory is iterable --- habitat/hooks/init | 3 +++ 1 file changed, 3 insertions(+) diff --git a/habitat/hooks/init b/habitat/hooks/init index b8dc771..2f022fe 100755 --- a/habitat/hooks/init +++ b/habitat/hooks/init @@ -10,6 +10,9 @@ if [ ! -f "$AUTHFILE" ]; then chmod 640 "$AUTHFILE" fi +# fix permissions on config directory +chmod g+rX -R "{{pkg.svc_config_path}}" + # initialize var directories pushd "{{pkg.svc_var_path}}" > /dev/null chmod g+rX -R . From fb2d9692545b7afde8013cdce69dd4d68e7b4294 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 16 Jun 2017 22:08:05 -0400 Subject: [PATCH 38/56] Use bash shell for init script and remove pushd/popd commands --- habitat/hooks/init | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/habitat/hooks/init b/habitat/hooks/init index 2f022fe..4aae482 100755 --- a/habitat/hooks/init +++ b/habitat/hooks/init @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # initialize users database AUTHFILE="{{pkg.svc_data_path}}/users.htpasswd" @@ -14,7 +14,7 @@ fi chmod g+rX -R "{{pkg.svc_config_path}}" # initialize var directories -pushd "{{pkg.svc_var_path}}" > /dev/null +cd "{{pkg.svc_var_path}}" chmod g+rX -R . for DIR in log run config tmp tmp/nginx; do if [ ! -d "$DIR" ]; then @@ -23,10 +23,9 @@ for DIR in log run config tmp tmp/nginx; do chown root:hab "$DIR" fi done -popd > /dev/null # initialize data directories -pushd "{{pkg.svc_data_path}}" > /dev/null +cd "{{pkg.svc_data_path}}" chmod g+rX -R . for DIR in sites services; do if [ ! -d "$DIR" ]; then @@ -35,11 +34,9 @@ for DIR in sites services; do chown root:hab "$DIR" fi done -popd > /dev/null # initialize static asset links -pushd "{{pkg.svc_static_path}}" > /dev/null +cd "{{pkg.svc_static_path}}" chmod g+rX -R . ln -sf "{{pkg.path}}/app/php-bootstrap" ln -sf "{{pkg.path}}/app/kernel-www" -popd > /dev/null \ No newline at end of file From a39ff95df56b027818fbfe5cbc42016523bd79eb Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 16 Jun 2017 22:38:25 -0400 Subject: [PATCH 39/56] Add bash to runtime dependencies --- habitat/plan.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/habitat/plan.sh b/habitat/plan.sh index eadfbc9..515cc2c 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -12,7 +12,8 @@ pkg_build_deps=( ) pkg_deps=( + core/bash emergence/nginx emergence/mariadb emergence/php5 -) \ No newline at end of file +) From a5a8edaf88e37ff94048815638fe74ef695ec005 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 17 Jun 2017 01:54:38 -0400 Subject: [PATCH 40/56] Apply php.ini to php-fpm to load zend opcache --- habitat/config/php-fpm | 14 +------------- habitat/config/php.ini | 17 +++++++++++++++++ kernel-lib/services/php-fpm.js | 5 +++-- 3 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 habitat/config/php.ini diff --git a/habitat/config/php-fpm b/habitat/config/php-fpm index bbdefa2..40c3439 100644 --- a/habitat/config/php-fpm +++ b/habitat/config/php-fpm @@ -20,16 +20,4 @@ pm.max_spare_servers = {{cfg.php.max_spare_servers}} {{~#if cfg.php.status_path}} pm.status_path = {{cfg.php.status_path}} -{{~/if}} - -php_admin_flag[short_open_tag] = on -php_admin_value[apc.shm_size] = {{cfg.php.shm_size}} -php_admin_value[apc.shm_segments] = 1 -php_admin_value[apc.slam_defense] = 0 -php_admin_value[apc.stat] = {{#if cfg.php.stat_scripts}}1{{else}}0{{/if}} -php_admin_value[opcache.validate_timestamps] = {{#if cfg.php.stat_scripts}}1{{else}}0{{/if}} -php_admin_value[upload_max_filesize] = {{cfg.php.upload_max_filesize}} -php_admin_value[post_max_size] = {{cfg.php.post_max_size}} -php_admin_value[memory_limit] = {{cfg.php.memory_limit}} -php_admin_value[error_reporting] = {{cfg.php.error_reporting}} -php_admin_value[date.timezone] = {{cfg.php.timezone}} \ No newline at end of file +{{~/if}} \ No newline at end of file diff --git a/habitat/config/php.ini b/habitat/config/php.ini new file mode 100644 index 0000000..e2d6706 --- /dev/null +++ b/habitat/config/php.ini @@ -0,0 +1,17 @@ +[PHP] +short_open_tag = on + +upload_max_filesize = {{cfg.php.upload_max_filesize}} +post_max_size = {{cfg.php.post_max_size}} +memory_limit = {{cfg.php.memory_limit}} +error_reporting = {{cfg.php.error_reporting}} +date.timezone = {{cfg.php.timezone}} + +apc.shm_size = {{cfg.php.shm_size}} +apc.shm_segments = 1 +apc.slam_defense = 0 +apc.stat = {{#if cfg.php.stat_scripts}}1{{else}}0{{/if}} + +zend_extension = opcache.so +opcache.enable = on +opcache.validate_timestamps = {{#if cfg.php.stat_scripts}}1{{else}}0{{/if}} \ No newline at end of file diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js index 29d142a..f35ff11 100644 --- a/kernel-lib/services/php-fpm.js +++ b/kernel-lib/services/php-fpm.js @@ -19,7 +19,8 @@ exports.PhpFpmService = function(name, controller, options) { me.packagePath = options.packagePath; me.execPath = me.packagePath + '/sbin/php-fpm'; - me.configPath = '/hab/svc/emergence-kernel/config/php-fpm'; + me.phpConfigPath = '/hab/svc/emergence-kernel/config/php.ini'; + me.fpmConfigPath = '/hab/svc/emergence-kernel/config/php-fpm'; me.pidPath = '/hab/svc/emergence-kernel/var/run/php-fpm.pid'; me.socketPath = '/hab/svc/emergence-kernel/var/run/php-fpm.sock'; me.errorLogPath = '/hab/svc/emergence-kernel/var/log/php-fpm.err'; @@ -52,7 +53,7 @@ exports.PhpFpmService.prototype.start = function() { return false; } - me.proc = spawn(me.execPath, ['--fpm-config', me.configPath]); + me.proc = spawn(me.execPath, ['-c', me.phpConfigPath, '--fpm-config', me.fpmConfigPath]); me.proc.on('exit', function (code) { From 398a8dacf15fc2d9ea2fd7436232b791a1c691da Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 17 Jun 2017 10:07:51 -0400 Subject: [PATCH 41/56] Add readline to php builds for CLI support --- service-plans/php/plan.sh | 2 ++ service-plans/php5/plan.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/service-plans/php/plan.sh b/service-plans/php/plan.sh index fad071c..38b7c45 100644 --- a/service-plans/php/plan.sh +++ b/service-plans/php/plan.sh @@ -26,6 +26,7 @@ pkg_build_deps=( core/gcc core/make core/re2c + core/readline ) pkg_bin_dirs=(bin sbin) pkg_lib_dirs=(lib) @@ -66,6 +67,7 @@ do_build() { --enable-fpm \ --with-fpm-user=hab \ --with-fpm-group=hab \ + --with-readline="$(pkg_path_for readline)" \ --with-gettext="$(pkg_path_for glibc)" \ --enable-apcu \ --enable-mbstring \ diff --git a/service-plans/php5/plan.sh b/service-plans/php5/plan.sh index b3e0d88..b5579cc 100644 --- a/service-plans/php5/plan.sh +++ b/service-plans/php5/plan.sh @@ -26,6 +26,7 @@ pkg_build_deps=( core/gcc core/make core/re2c + core/readline ) pkg_bin_dirs=(bin sbin) pkg_lib_dirs=(lib) @@ -66,6 +67,7 @@ do_build() { --enable-fpm \ --with-fpm-user=hab \ --with-fpm-group=hab \ + --with-readline="$(pkg_path_for readline)" \ --with-gettext="$(pkg_path_for glibc)" \ --enable-apcu \ --enable-mbstring \ From c2f0d0de0726f270da8679acc4d3a4dc6e542421 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 17 Jun 2017 10:08:08 -0400 Subject: [PATCH 42/56] Add coreutils to runtime deps for shell scripts --- habitat/plan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/habitat/plan.sh b/habitat/plan.sh index 515cc2c..265036a 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -12,6 +12,7 @@ pkg_build_deps=( ) pkg_deps=( + core/coreutils core/bash emergence/nginx emergence/mariadb From 36096f52df3ab56983f8e8a054df765f1015d774 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 17 Jun 2017 10:08:24 -0400 Subject: [PATCH 43/56] Fix bash interpreter in shell scripts --- habitat/plan.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/habitat/plan.sh b/habitat/plan.sh index 265036a..15c98a1 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -18,3 +18,8 @@ pkg_deps=( emergence/mariadb emergence/php5 ) + +do_install() { + do_default_install + fix_interpreter "$scaffolding_app_prefix/bin/*" core/bash bin/bash +} From f3ecd0f651b89152d15aa67fad414c6047c6f04b Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 17 Jun 2017 10:08:46 -0400 Subject: [PATCH 44/56] Use su instead of sudo for privilege drop --- bin/shell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/shell b/bin/shell index 48819c0..d1787c3 100755 --- a/bin/shell +++ b/bin/shell @@ -62,7 +62,7 @@ PHP_ARGS="-d auto_prepend_file=$autoPrependScript -d apc.enable_cli=on $INPUT_AR if [ "$(whoami)" == "hab" ]; then TERM=$TERM php $PHP_ARGS else - TERM=$TERM sudo -E -u hab -g hab php $PHP_ARGS + TERM=$TERM su hab -c "php $PHP_ARGS" fi From 04eb7b1278c49465104334d3ad5c02d5b6b8420a Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 17 Jun 2017 21:26:54 -0400 Subject: [PATCH 45/56] Update readme format and add habitat instructions --- README.md | 68 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index b68990d..d132a63 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,44 @@ -Emergence -========= +# Emergence [![Join the chat at https://gitter.im/JarvusInnovations/Emergence](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/JarvusInnovations/Emergence?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) Emergence is a NodeJS-powered server that provides a web interface for configuring and launching the services that power your website or application. It provides virtualized storage containers for your code and assets that are accessible via WebDAV and API. Each storage container maintains complete version history for all files and can be linked over the web to a parent container that files will be inherited from just-in-time. -Features ---------- -* Rich web interface provides for all setup and management -* Plugin-based support for system services to be configured and run - * Plugins included for nginx and mysql -* Versioned storage containers - * Inherit remote containers over http - * Copy-on-write - * Accessible remotely via WebDAV and locally via API -* PHP development framework - * Classes automatically loaded from storage container - * Lightweight MVC classes optimized for serial inheritance across sites - * Extendable templating system powered by Dwoo - - -Requirements -------------- -* NodeJS -* npm - * underscore - * node-static -* mysql -* nginx -* php-fpm - * php 5.3+ - * apc - * mysqli - - -Installation --------------- +## Features + +- Rich web interface provides for all setup and management +- Plugin-based support for system services to be configured and run + - Plugins included for nginx and mysql +- Versioned storage containers + - Inherit remote containers over http + - Copy-on-write + - Accessible remotely via WebDAV and locally via API +- PHP development framework + - Classes automatically loaded from storage container + - Lightweight MVC classes optimized for serial inheritance across sites + - Extendable templating system powered by Dwoo + + +## Installation + See http://emr.ge/docs +## Building with Habitat + +- `cd ~/Repositories/emergence` +- `hab studio enter` +- `build` +- Optionally export a docker container image: `hab pkg export docker emergence/emergence-kernel` + + +## Debugging with Habitat + +From within studio: `hab sup start emergence/emergence-kernel` + + +## Running with Docker + +- Start a container from new image: `docker run -it -p 9080:80 -p 9083:9083 --name myemergence emergence/emergence-kernel` -Visit http://serverhost:9083 in your browser From 8a2f14a0d8cadbb39adba628173e40b019c1a1ab Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 18 Jun 2017 23:30:23 -0400 Subject: [PATCH 46/56] Test existing kernel socket and delete if stale --- kernel-lib/server.js | 87 +++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/kernel-lib/server.js b/kernel-lib/server.js index 5a5b460..64061a0 100644 --- a/kernel-lib/server.js +++ b/kernel-lib/server.js @@ -1,4 +1,5 @@ -var http = require('http'), +var net = require('net'), + http = require('http'), util = require('util'), fs = require('fs'), path = require('path'), @@ -32,39 +33,63 @@ util.inherits(exports.Server, events.EventEmitter); exports.Server.prototype.start = function() { - // create authenticator - this.httpAuth = require('http-auth')({ - authRealm: 'Emergence Node Management', - authFile: this.usersPath + var me = this, + testConnection; + + // test if socket connection is already open + if (!me.socketPath || !fs.existsSync(me.socketPath)) { + _doStart(); + return; + } + + testConnection = net.createConnection(me.socketPath); + + testConnection.on('connect', function() { + testConnection.close(); + throw 'Socket file already exists and is live: '+me.socketPath; }); - // create static fileserver - this.fileServer = new static.Server(this.staticDir); + testConnection.on('error', function() { + console.log('Deleting stale socket file:', me.socketPath); + fs.unlinkSync(me.socketPath); + _doStart(); + }); - // listen on web port - if (this.sslKey && this.sslCert) { - this.webServer = require('https').createServer({ - key: fs.readFileSync(this.sslKey), - cert: fs.readFileSync(this.sslCert) - }, this.handleWebRequest.bind(this)).listen(this.port, this.host); + function _doStart() { + // create authenticator + me.httpAuth = require('http-auth')({ + authRealm: 'Emergence Node Management', + authFile: me.usersPath + }); - this.webProtocol = 'https'; - } else { - this.webServer = require('http').createServer(this.handleWebRequest.bind(this)).listen(this.port, this.host); + // create static fileserver + me.fileServer = new static.Server(me.staticDir); - this.webProtocol = 'http'; - } + // listen on web port + if (me.sslKey && me.sslCert) { + me.webServer = require('https').createServer({ + key: fs.readFileSync(me.sslKey), + cert: fs.readFileSync(me.sslCert) + }, me.handleWebRequest.bind(me)).listen(me.port, me.host); - // listen on unix socket - if (this.socketPath) { - this.socketServer = require('http').createServer(this.handleRequest.bind(this)).listen(this.socketPath); - fs.chmodSync(this.socketPath, '400'); - } + me.webProtocol = 'https'; + } else { + me.webServer = require('http').createServer(me.handleWebRequest.bind(me)).listen(me.port, me.host); + + me.webProtocol = 'http'; + } + + // listen on unix socket + if (me.socketPath) { + me.socketServer = require('http').createServer(me.handleRequest.bind(me)).listen(me.socketPath); + fs.chmodSync(me.socketPath, '400'); + } - // clean up on exit - process.on('exit', this.close.bind(this)); + // clean up on exit + process.on('exit', me.close.bind(me)); - console.log('Management server listening on '+this.webProtocol+'://'+this.host+':'+this.port); + console.log('Management server listening on '+me.webProtocol+'://'+me.host+':'+me.port); + } }; exports.createServer = function(paths, options) { @@ -127,13 +152,15 @@ exports.Server.prototype.handleRequest = function(request, response) { }; exports.Server.prototype.close = function(options, error) { + var me = this; + console.log('Shutting down management server...'); - if (this.webServer) { - this.webServer.close(); + if (me.webServer) { + me.webServer.close(); } - if (this.socketServer) { - this.socketServer.close(); + if (me.socketServer) { + me.socketServer.close(); } }; \ No newline at end of file From 9ff4506a8310e5d8d87f2681cd3c3831f3ea8135 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 19 Jun 2017 21:06:17 -0400 Subject: [PATCH 47/56] Add git to runtime dependencies --- habitat/plan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/habitat/plan.sh b/habitat/plan.sh index 15c98a1..edbe9f7 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -14,6 +14,7 @@ pkg_build_deps=( pkg_deps=( core/coreutils core/bash + core/git emergence/nginx emergence/mariadb emergence/php5 From d242847108307999f22200fdc2e7dfd4fa8e2683 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 19 Jun 2017 21:06:47 -0400 Subject: [PATCH 48/56] Link hab and git binaries for PHP scripts PATH doesn't get down to them --- habitat/hooks/init | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/habitat/hooks/init b/habitat/hooks/init index 4aae482..ada58be 100755 --- a/habitat/hooks/init +++ b/habitat/hooks/init @@ -40,3 +40,7 @@ cd "{{pkg.svc_static_path}}" chmod g+rX -R . ln -sf "{{pkg.path}}/app/php-bootstrap" ln -sf "{{pkg.path}}/app/kernel-www" + +# initialize bin links +hab pkg binlink core/hab hab +hab pkg binlink core/git git \ No newline at end of file From 69b444d8545cf39efa362560a205c84f88e87039 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 19 Jun 2017 21:07:04 -0400 Subject: [PATCH 49/56] Update git-shell script for habitat environment --- bin/git-shell | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/bin/git-shell b/bin/git-shell index 64d26c6..dfb7731 100755 --- a/bin/git-shell +++ b/bin/git-shell @@ -21,7 +21,6 @@ SITE_HANDLE=$1 SITE_LAYER=$2 UNDERSCORE=$DIR/../node_modules/.bin/underscore -EMERGENCE_USER=`cat /hab/svc/emergence-kernel/config/config.json | $UNDERSCORE extract user --outfmt text` BLACK="\033[01;30m" MAGENTA="\033[1;31m" @@ -43,7 +42,6 @@ PS2="\[$ORANGE\]→ \[$RESET\]" \ # check environment -test $EUID -eq 0 || die "emergence-git-shell must be run with sudo" test -x "$UNDERSCORE" || die "unable to execute $UNDERSCORE" test -n "$SITE_HANDLE" -a -n "$SITE_LAYER"|| die "usage: emergence-git-shell site_handle layer_id" @@ -89,6 +87,20 @@ if [ -n "$GIT_USER_DIRTY" ]; then fi +# setup environment +export PS1 +export PS2 +export SITE_HOSTNAME +export SITE_LAYER +export GIT_SSH + +export HISTFILE="$GIT_DIR.bash_history" +export GIT_AUTHOR_NAME="$GIT_NAME" +export GIT_AUTHOR_EMAIL="$GIT_EMAIL" +export GIT_COMMITTER_NAME="$GIT_NAME" +export GIT_COMMITTER_EMAIL="$GIT_EMAIL" + + # execute subshell echo "Opening git shell at $GIT_DIR" cd $GIT_DIR @@ -97,16 +109,8 @@ git status echo "Committing as $GIT_NAME <$GIT_EMAIL>" -exec sudo -u $EMERGENCE_USER \ - HOME="$GIT_DIR" \ - HISTFILE="$GIT_DIR.bash_history" \ - GIT_SSH="$GIT_SSH" \ - GIT_AUTHOR_NAME="$GIT_NAME" \ - GIT_AUTHOR_EMAIL="$GIT_EMAIL" \ - GIT_COMMITTER_NAME="$GIT_NAME" \ - GIT_COMMITTER_EMAIL="$GIT_EMAIL" \ - PS1="$PS1" \ - PS2="$PS2" \ - SITE_HOSTNAME="$SITE_HOSTNAME" \ - SITE_LAYER="$SITE_LAYER" \ - /bin/bash --norc \ No newline at end of file +if [ "$(whoami)" == "hab" ]; then + HOME="$GIT_DIR" exec bash --norc +else + exec su hab -c "HOME='$GIT_DIR' bash --norc" +fi \ No newline at end of file From 295aebf9c4c51c44f0202a0085f1193b7aeae195 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 19 Jun 2017 21:08:39 -0400 Subject: [PATCH 50/56] Use local underscore executable --- bin/{create-site.sh => create-site} | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) rename bin/{create-site.sh => create-site} (86%) diff --git a/bin/create-site.sh b/bin/create-site similarity index 86% rename from bin/create-site.sh rename to bin/create-site index 1375463..10df73f 100755 --- a/bin/create-site.sh +++ b/bin/create-site @@ -1,5 +1,21 @@ #!/bin/bash + +# determine location of this script +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + + +# get path to underscore +UNDERSCORE=$DIR/../node_modules/.bin/underscore + + +# parse args DEFAULT_PRECACHE_TREES='dwoo-plugins,event-handlers,html-templates,php-classes,php-config,php-migrations,site-root,site-tasks' DEFAULT_GENERATE_TABLES='UserSession,User' @@ -77,10 +93,6 @@ case $i in esac done -if [ "$GENERATE_TABLES" ] || [ "$PRECACHE_TREES" ] || [ "$ENTER_SHELL" ]; then - command -v underscore >/dev/null 2>&1 || { echo >&2 "underscore must be installed to use --precache-trees or --generate-tables"; exit 1; } -fi - if [ -z "$KERNEL_SOCKET" ]; then KERNEL_SOCKET="/hab/svc/emergence-kernel/var/run/kernel.sock" fi @@ -145,7 +157,7 @@ if [ -z "$PRECACHE_TREES" ] && [ -z "$GENERATE_TABLES" ] && [ -z "$ENTER_SHELL" exit 0 fi -SITE_HANDLE=$(echo "${RESPONSE}" | underscore --outfmt=text extract data.handle) +SITE_HANDLE=$(echo "${RESPONSE}" | $UNDERSCORE --outfmt=text extract data.handle) if [ "$PRECACHE_TREES" ]; then emergence-shell $SITE_HANDLE 1>&2 << END_OF_PHP From 9a33fb3062d5e9d9eebe474aa4851c78b0660c0a Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 19 Jun 2017 21:49:06 -0400 Subject: [PATCH 51/56] Add curl runtime dependency --- habitat/plan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/habitat/plan.sh b/habitat/plan.sh index edbe9f7..54edf9f 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -15,6 +15,7 @@ pkg_deps=( core/coreutils core/bash core/git + core/curl emergence/nginx emergence/mariadb emergence/php5 From 2052038d4fec0aa3a0502ae70be62f1eeb9a8168 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 19 Jun 2017 21:51:00 -0400 Subject: [PATCH 52/56] Use new curl syntax for unix socket URLs --- bin/create-site | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/create-site b/bin/create-site index 10df73f..d2167e3 100755 --- a/bin/create-site +++ b/bin/create-site @@ -148,7 +148,7 @@ RESPONSE=$( --unix-socket "${KERNEL_SOCKET}" \ -H "Content-Type: application/json" \ -d "${REQUEST_BODY}" \ - "http:/sites" + "http://localhost/sites" ) echo "$RESPONSE" From 70dc78d2c4452be0a9589f3e7358168c4684a744 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 21 Jun 2017 17:18:00 -0400 Subject: [PATCH 53/56] Add ssh-keygen to bin --- habitat/hooks/init | 3 ++- habitat/plan.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/habitat/hooks/init b/habitat/hooks/init index ada58be..1ac5aec 100755 --- a/habitat/hooks/init +++ b/habitat/hooks/init @@ -43,4 +43,5 @@ ln -sf "{{pkg.path}}/app/kernel-www" # initialize bin links hab pkg binlink core/hab hab -hab pkg binlink core/git git \ No newline at end of file +hab pkg binlink core/git git +hab pkg binlink core/openssh ssh-keygen \ No newline at end of file diff --git a/habitat/plan.sh b/habitat/plan.sh index 54edf9f..312bc9d 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -16,6 +16,7 @@ pkg_deps=( core/bash core/git core/curl + core/openssh emergence/nginx emergence/mariadb emergence/php5 From 8278f5f48ab28338586e390fb114db023a253908 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 21 Jun 2017 17:54:56 -0400 Subject: [PATCH 54/56] Add ssh client to bin --- habitat/hooks/init | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/habitat/hooks/init b/habitat/hooks/init index 1ac5aec..032f8d4 100755 --- a/habitat/hooks/init +++ b/habitat/hooks/init @@ -44,4 +44,5 @@ ln -sf "{{pkg.path}}/app/kernel-www" # initialize bin links hab pkg binlink core/hab hab hab pkg binlink core/git git -hab pkg binlink core/openssh ssh-keygen \ No newline at end of file +hab pkg binlink core/openssh ssh-keygen +hab pkg binlink core/openssh ssh \ No newline at end of file From c912890fbd6212f543ad52a509b1e3bef4fe9eb9 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 6 Sep 2017 18:20:19 -0400 Subject: [PATCH 55/56] WIP: Upgrade mariadb to 10.2 --- service-plans/mariadb/plan.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/service-plans/mariadb/plan.sh b/service-plans/mariadb/plan.sh index 8b18464..ab6bf42 100644 --- a/service-plans/mariadb/plan.sh +++ b/service-plans/mariadb/plan.sh @@ -1,11 +1,12 @@ pkg_name=mariadb pkg_origin=emergence -pkg_version=10.1.18 +pkg_version=10.2.6 pkg_description="An open source monitoring software for networks and applications" pkg_maintainer="The Habitat Maintainers " pkg_license=('GPL-2.0') -pkg_source=http://ftp.hosteurope.de/mirror/archive.mariadb.org//${pkg_name}-${pkg_version}/source/${pkg_name}-${pkg_version}.tar.gz -pkg_shasum=d7336907e9ff44496d6453f92526b25bd253638a64a051ca879f953499873b73 +pkg_source=https://github.com/MariaDB/server/archive/${pkg_name}-${pkg_version}.tar.gz +pkg_shasum=64dc9152daee2b396828963298edc55f36ad1add65a1f19d3db5ff0962d46842 +pkg_dirname="server-${pkg_name}-${pkg_version}" pkg_deps=(core/ncurses core/gcc-libs core/zlib core/sed) pkg_build_deps=(core/gcc core/make core/coreutils core/cmake) pkg_bin_dirs=(bin) @@ -32,11 +33,16 @@ do_build() { -DCMAKE_PREFIX_PATH="$(pkg_path_for core/ncurses)" \ -DCMAKE_BUILD_TYPE=Release \ -DWITH_READLINE=OFF + make + + return $? } do_install() { + attach make install + attach rm -rf "${pkg_prefix}/mysql-test" rm -rf "${pkg_prefix}/bin/mysql_client_test" rm -rf "${pkg_prefix}/bin/mysql_test" From eba358b83533b93709d49065ded7b549b2dd6e45 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 31 Dec 2017 02:05:01 -0500 Subject: [PATCH 56/56] Commit TODOs and dev scripts --- README.md | 21 +++++++++++++++++++++ clean-runtime.sh | 8 ++++++++ debug.toml | 2 ++ rebuild.sh | 8 ++++++++ reset-emergence.sh | 4 ++++ run-kernel.sh | 7 +++++++ run-service.sh | 5 +++++ run-studio.sh | 6 ++++++ 8 files changed, 61 insertions(+) create mode 100755 clean-runtime.sh create mode 100644 debug.toml create mode 100755 rebuild.sh create mode 100755 reset-emergence.sh create mode 100755 run-kernel.sh create mode 100755 run-service.sh create mode 100755 run-studio.sh diff --git a/README.md b/README.md index d132a63..a20f4af 100644 --- a/README.md +++ b/README.md @@ -42,3 +42,24 @@ From within studio: `hab sup start emergence/emergence-kernel` - Start a container from new image: `docker run -it -p 9080:80 -p 9083:9083 --name myemergence emergence/emergence-kernel` + +## Habitat Migration Todo + +- [X] Use habitat config system to generate base service configs, add `include .../var/nginx.sites` to nginx, stick dynamic ish there +- [ ] Compare stock services configs with hab-provided configs +- [ ] Pull Request mariadb fix +- [ ] Get nginx running with configured external config +- [X] Move stock nginx config bodies to .include files (e.g. http.include) +- [X] Add default static nginx site to nginx.conf before sites include +- [ ] Review all initialized permissions +- [ ] Remove shelljs +- [ ] Upgrade dwoo and see if php7 works + +- Get docker container working + - [X] Run new docker container + - [X] Check that Zend is loaded now + - [X] get shell wrapper working + - [X] Figure out why cookies dont work -- set from console for now + - [X] Commit changes + +- [ ] PR readline into core-plans php and php5 diff --git a/clean-runtime.sh b/clean-runtime.sh new file mode 100755 index 0000000..645ffb5 --- /dev/null +++ b/clean-runtime.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +hab sup term +hab pkg exec core/busybox-static killall php-fpm +hab pkg exec core/busybox-static killall nginx +hab pkg exec core/busybox-static killall mysqld +rm /hab/svc/emergence-kernel/var/run/kernel.sock +find /hab/svc/emergence-kernel/var/run/ -type f -delete \ No newline at end of file diff --git a/debug.toml b/debug.toml new file mode 100644 index 0000000..486049c --- /dev/null +++ b/debug.toml @@ -0,0 +1,2 @@ +[php] +stat_scripts=true diff --git a/rebuild.sh b/rebuild.sh new file mode 100755 index 0000000..c48424a --- /dev/null +++ b/rebuild.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# reset runtime env +./clean-runtime.sh + +rm -R /hab/svc/*/ + +exec build \ No newline at end of file diff --git a/reset-emergence.sh b/reset-emergence.sh new file mode 100755 index 0000000..0d656db --- /dev/null +++ b/reset-emergence.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +./clean-runtime.sh +rm -R /hab/svc/emergence-kernel \ No newline at end of file diff --git a/run-kernel.sh b/run-kernel.sh new file mode 100755 index 0000000..365d6af --- /dev/null +++ b/run-kernel.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +./clean-runtime.sh + +export NODE_PATH="`hab pkg path emergence/emergence-kernel`/app/node_modules/" + +exec hab pkg exec core/node node bin/kernel \ No newline at end of file diff --git a/run-service.sh b/run-service.sh new file mode 100755 index 0000000..42167f0 --- /dev/null +++ b/run-service.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./clean-runtime.sh + +exec hab svc start emergence/emergence-kernel \ No newline at end of file diff --git a/run-studio.sh b/run-studio.sh new file mode 100755 index 0000000..59daa06 --- /dev/null +++ b/run-studio.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +export HAB_DOCKER_OPTS="-p 9080:80 -p 9083:9083" +export HAB_ORIGIN=emergence + +exec hab studio enter