Merge pull request #53 from dice-group/develop

Developed version 1.0.8

.gitignore

@@ -6,9 +6,11 @@ target
 .settings
 .idea
 squirrel.iml
-/data
+/deployment/data
+/deployment/scenarios/1/worker1
 scenarios/1/worker*
 /dependency-reduced-pom.xml
 /foundUris.lck
 /foundUris.properties
 /foundUris.script
+/bin/

.travis.yml

@@ -1 +1,9 @@
+sudo: required
+
 language: java
+
+services:
+  - docker
+
+before_install:
+- docker pull rethinkdb:2.3.5

Dockerfile

@@ -1,29 +1,13 @@
-FROM java
-#FROM ubuntu:trusty
+FROM openjdk:8u151-jdk
 
-#RUN apt-get update && apt-get -y install python-software-properties && apt-get -y install software-properties-common && add-apt-repository ppa:webupd8team/java -y && apt-get update
-#RUN apt-get install -y supervisor
-# Accept the Java licenses
-# RUN echo oracle-java6-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
-# RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
-#RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
-#RUN apt-get install -y oracle-java8-installer && apt-get install -y oracle-java8-set-default
+RUN apt-get update && apt-get install -y netcat
 
-#COPY ./src /data/squirrel/src
-#COPY ./repository /data/squirrel/repository
-#COPY ./pom.xml /data/squirrel/pom.xml
 COPY ./target/squirrel.jar /data/squirrel/squirrel.jar
 WORKDIR /data/squirrel
-#RUN mvn clean compile assembly:single
 
-#COPY ./data /data/squirrel/data
-#COPY ./scripts /data/squirrel/scripts
-
-#ENV PATH $PATH:/data/squirrel/scripts
-
-# Frontier tcp port
-#EXPOSE 60000
-# Sink tcp port
-#EXPOSE 60001
+ADD entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
 
 VOLUME ["/var/squirrel/data"]
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

Makefile

@@ -7,4 +7,4 @@ dockerize: build
 	docker build -t squirrel .
 
 clean:
-	rm -rf data/worker*
+	rm -rf data/worker* && rm -rf deployment/scenarios/1/worker*

README.md

@@ -3,8 +3,7 @@ Squirrel searches and collects Linked Data
 
 # Running with docker
 ```
+  $ make build dockerize
   $ docker-compose build
   $ docker-compose up
 ```
-
-This software is provided under a dual license. For non-commercial purposes, the terms of the LGPL3.0 license hold (printed in the LICENSE file). For commercial purposes, please contact the financial department of AKSW at [email protected]

data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

deployment/Makefile

@@ -0,0 +1,9 @@
+default: clean
+	docker-compose up -d
+	docker-compose logs -f
+
+clean:
+	sudo rm -rf data/
+
+network:
+	docker network create -d bridge -o com.docker.network.bridge.enable_ip_masquerade=true --attachable --subnet 172.17.200.0/24 squirrel

deployment/README.md

@@ -0,0 +1,22 @@
+# Deployment instructions
+
+SSH into server and clone this repo:
+```
+ssh akswnc5.aksw.uni-leipzig.de
+git clone https://github.com/dice-group/Squirrel && cd Squirrel
+```
+
+Before using these instructions, you need to build squirrel docker image from repository root:
+```
+$ make dockerize
+```
+
+To deploy squirrel do:
+```
+cd deployment && make
+```
+
+If squirrel was deployed previously make sure to clean up the docker containers:
+```
+docker-compose down
+```

docker-compose-simulation-scenario1.yml → deployment/docker-compose-simulation-scenario1.yml

@@ -1,44 +1,43 @@
 version: '2'
 
 services:
-
-  frontier:
-    image: squirrel
-    container_name: frontier
-    environment:
-      - HOBBIT_RABBIT_HOST=rabbit
-      - SEED_FILE=/var/squirrel/data/seeds.txt
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
-#    networks:
-#      - squirrel
-    volumes:
-      - ./scenarios/1:/var/squirrel/data
-    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.FrontierComponent
-
   rethinkdb:
     image: rethinkdb:2.3.5
-#    networks:
-#      - squirrel
 #    volumes:
 #      - ./scenarios/1/rethinkdb:/data
+    ports:
+      - "28015:28015"
 
   # message bus
   rabbit:
     image: rabbitmq:management
     container_name: rabbit
     hostname: rabbit
-#    networks:
-#      - squirrel
+    ports:
+      - "8081:15672"
+      # Forwarding the port for testing
+      - "5672:5672"
+
+  frontier:
+    image: squirrel
+    container_name: frontier
+    environment:
+      HOBBIT_RABBIT_HOST: rabbit
+      SEED_FILE: /var/squirrel/data/seeds.txt
+      RDB_HOST_NAME: rethinkdb
+      RDB_PORT: 28015
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
+    volumes:
+      - ./scenarios/1:/var/squirrel/data
+    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.FrontierComponent
 
   worker1:
     image: squirrel
     container_name: worker1
     environment:
-      - HOBBIT_RABBIT_HOST=rabbit
-      - OUTPUT_FOLDER=/var/squirrel/data
-#    networks:
-#      - squirrel
+      HOBBIT_RABBIT_HOST: rabbit
+      OUTPUT_FOLDER: /var/squirrel/data
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
     volumes:
         - ./scenarios/1/worker1:/var/squirrel/data
     command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.WorkerComponent
@@ -47,13 +46,12 @@ services:
     image: squirrel
     container_name: NodeA
     environment:
-      - RESOURCE_MODEL=/var/squirrel/data/nodeA.ttl
-      - RESOURCE_MODEL_LANG=N3
-      - PORT=80
-      - DUMP_FILE_NAME=dump.gz
-      - USE_DEREF=true
-#    networks:
-#      - squirrel
+      RESOURCE_MODEL: /var/squirrel/data/nodeA.ttl
+      RESOURCE_MODEL_LANG: N3
+      PORT: 80
+      DUMP_FILE_NAME: dump.gz
+      USE_DEREF: "true"
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
     volumes:
         - ./scenarios/1:/var/squirrel/data
     command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.SimpleHttpServerComponent
@@ -62,12 +60,11 @@ services:
     image: squirrel
     container_name: NodeB
     environment:
-      - RESOURCE_MODEL=/var/squirrel/data/nodeB.ttl
-      - RESOURCE_MODEL_LANG=N3
-      - PORT=80
-      - USE_DEREF=true
-#    networks:
-#      - squirrel
+      RESOURCE_MODEL: /var/squirrel/data/nodeB.ttl
+      RESOURCE_MODEL_LANG: N3
+      PORT: 80
+      USE_DEREF: "true"
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
     volumes:
         - ./scenarios/1:/var/squirrel/data
     command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.SimpleHttpServerComponent
@@ -76,18 +73,12 @@ services:
     image: squirrel
     container_name: NodeC
     environment:
-      - RESOURCE_MODEL=/var/squirrel/data/nodeC.ttl
-      - RESOURCE_MODEL_LANG=N3
-      - PORT=80
-      - DUMP_FILE_NAME=dump.gz
-      - USE_DEREF=false
-#    networks:
-#      - squirrel
+      RESOURCE_MODEL: /var/squirrel/data/nodeC.ttl
+      RESOURCE_MODEL_LANG: N3
+      PORT: 80
+      DUMP_FILE_NAME: dump.gz
+      USE_DEREF: "false"
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
     volumes:
         - ./scenarios/1:/var/squirrel/data
     command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.SimpleHttpServerComponent
-
-#networks:
-#  squirrel:
-#    external:
-#      name: squirrel

deployment/docker-compose.yml

@@ -0,0 +1,71 @@
+version: "2"
+
+services:
+  rethinkdb:
+    image: rethinkdb:2.3.5
+    volumes:
+      - ./data/rethinkdb:/data
+    networks:
+      - squirrel
+
+  # message bus
+  rabbit:
+    image: rabbitmq:management
+    networks:
+      - squirrel
+
+  frontier:
+    image: squirrel
+    environment:
+      HOBBIT_RABBIT_HOST: rabbit
+      SEED_FILE: /var/squirrel/data/seeds.txt
+      RDB_HOST_NAME: rethinkdb
+      RDB_PORT: 28015
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
+    volumes:
+      - ./data/frontier:/var/squirrel/data
+      - ./seeds.txt:/var/squirrel/data/seeds.txt:ro
+    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.FrontierComponent
+    networks:
+      - squirrel
+
+  worker1:
+    image: squirrel
+    environment:
+      HOBBIT_RABBIT_HOST: rabbit
+      OUTPUT_FOLDER: /var/squirrel/data
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
+    volumes:
+        - ./data/worker1:/var/squirrel/data
+    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.WorkerComponent
+    networks:
+      - squirrel
+
+  worker2:
+    image: squirrel
+    environment:
+      HOBBIT_RABBIT_HOST: rabbit
+      OUTPUT_FOLDER: /var/squirrel/data
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
+    volumes:
+        - ./data/worker2:/var/squirrel/data
+    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.WorkerComponent
+    networks:
+      - squirrel
+
+  worker3:
+    image: squirrel
+    environment:
+      HOBBIT_RABBIT_HOST: rabbit
+      OUTPUT_FOLDER: /var/squirrel/data
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
+    volumes:
+        - ./data/worker3:/var/squirrel/data
+    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.WorkerComponent
+    networks:
+      - squirrel
+
+networks:
+  squirrel:
+    external:
+      name: squirrel