Blurred lines AR CR: Watch out for your snacks

This tutorial explains how to build an immersive installation where virtual bugs move physical elements. Our idea was to create a scene where virtual and physical elements interact together, as if they were all part of the conventional reality, leaving the spectator unsure of what is real and what is not.

Concept

Hardware Needed

2x Raspberry Pi
1x Adafruit Servo HAT
2x 180 degree Servo Motor
1x PowerBank stick
Breadboard
Jumper wires
Makey Makey board kit
Projector (1280x720)

Software and Frameworks

For the Raspberry Pis: Current distro with [Python 2.7] for SpaceBrew and [Python 3] for the Adafruit ServoHAT (https://www.python.org/) and Spacebrew, WiFi/network is assumed to be working; the Servo HAT needs some libraries and configuration to work - work through the Adafruit tutorial
For the AR scene: Unity 2017.3.0f3 with Vuforia and the YUXI examples
One Spacebrew server (we had a local instance running at 192.168.1.182)

Unity assets

For the insects: Set of 7 arthropods

System Overview

Creating the scene

STEP 1: Wiring the Raspberry Pi / Pi Hat with the servos

STEP 2: Connect the Raspberry Pi to Spacebrew

To connect to Spacebrew

#!/usr/bin/env python
import sys
import time
import subprocess
from pySpacebrew.spacebrew import Spacebrew


CHECK_FREQ = 0.1 # How often to check the hardware


def servoFirstMove():
  print("Calling python3...")
  subprocess.call(["python3", "servoBrew-4.py"])
  print("Call done.")

def servoOtherMove():
  print("Calling python3...")
  subprocess.call(["python3", "servoBrew-5.py"])
  print("Call done.")


# Setup Spacebrew with Publishers and Subscribers
brew = Spacebrew("Salvador's Hands", description="",  server="192.168.1.182", port=9000)
brew.addSubscriber("Salvadore's move", "boolean")
brew.addSubscriber("Another trick up his sleeve", "boolean")


firstMoveOn = False 
otherMoveOn = False

def handleFirstMove(value):
    global firstMoveOn
    print("Received *first* move: "+str(value))
    if (value == 'true' or str(value) == 'True'):
      firstMoveOn = True

def handleOtherMove(value):
    global otherMoveOn
    print("Received *other* move: "+str(value))
    if (value == 'true' or str(value) == 'True'):
      otherMoveOn = True


# for handling messages coming through spacebrew 
brew.subscribe("Salvadore's move", handleFirstMove)
brew.subscribe("Another trick up his sleeve", handleOtherMove)

try:
    brew.start()
    print("Press Ctrl-C to quit.")
    while True:
      if (firstMoveOn):
        servoFirstMove()
        firstMoveOn = False

      if (otherMoveOn):
        print("Doint the *OTHER* move")
        servoOtherMove()
        otherMoveOn = False

      time.sleep(CHECK_FREQ)
      
    
finally:
  print("Goodbye :-)")

Calling/moving servo no 1.

#!/usr/bin/env python
import RPi.GPIO as GPIO
import sys
import time
from pySpacebrew.spacebrew import Spacebrew

from adafruit_servokit import ServoKit
kit = ServoKit(channels=16)

print("Servo positions initialized.")

def moveServos():
  kit.servo[0].angle = 180
  kit.servo[7].angle = 180
  time.sleep(0.5)
  kit.servo[0].angle = 0
  kit.servo[7].angle = 0

  print("I did Salvadore's other trick that he has up his sleeve. Nice, isn't it?")
  
moveServos()


GPIO.cleanup()

Calling/moving servo no 2.

#!/usr/bin/env python
import RPi.GPIO as GPIO
import sys
import time
from pySpacebrew.spacebrew import Spacebrew

from adafruit_servokit import ServoKit
kit = ServoKit(channels=16)

print("Servo positions initialized.")

def moveServos():
  kit.servo[0].angle = 0
  kit.servo[7].angle = 0
  #time.sleep(0.5)
  kit.servo[0].angle = 30
  time.sleep(1)
  kit.servo[0].angle = 0
  kit.servo[7].angle = 30
  time.sleep(0.5)
  kit.servo[0].angle = 30
  kit.servo[7].angle = 0
  time.sleep(1.5)
  kit.servo[0].angle = 0
  kit.servo[7].angle = 30
  time.sleep(0.5)
  kit.servo[0].angle = 0
  kit.servo[7].angle = 0
  time.sleep(0.5)
  kit.servo[0].angle = 180
  kit.servo[7].angle = 180
  time.sleep(2.0)
  kit.servo[0].angle = 0
  kit.servo[7].angle = 0

  print("I did Salvadore's moves. Nice, isn't it?")
  
moveServos()


GPIO.cleanup()

STEP 3: Unity scene

For controlling the bug's movements:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.AI;


public class InsectMovement_New : MonoBehaviour {

	//	public GameObject myPrefab;
	NavMeshAgent agent;
	public Transform hotspot1;
	public Transform hotspot2;
	public Transform hotspot3;
	public Transform origin;
	bool moveRandom = false;
	AudioSource crawlingSound;
	bool isAtOrigin = true;

	//	Animator anim;

	// Use this for initialization
	void Start () {
		agent = GetComponent<NavMeshAgent>();
		crawlingSound = GetComponent<AudioSource> ();
		//		anim = GetComponent<Animator>();

	}


	// Update is called once per frame
	void Update () {

		isAtOrigin = Instantiate_New.atOrigin;

		if (Input.GetKey ("space")) {

			Vector3 destPos = new Vector3 (hotspot1.position.x, 0.1f, hotspot1.position.z);
			agent.SetDestination (destPos);
//			Debug.Log (destPos);
		
		} else if (Input.GetKey (KeyCode.UpArrow)) {
		
			Vector3 destPos2 = new Vector3 (hotspot2.position.x, 0.1f, hotspot2.position.z);
			agent.SetDestination (destPos2);
//			Debug.Log (destPos2);
		}

		else if (Input.GetKey (KeyCode.DownArrow)) {

			Vector3 destPos3 = new Vector3 (hotspot3.position.x, 0.1f, hotspot3.position.z);
			agent.SetDestination (destPos3);
//			Debug.Log (destPos3);
		}


		if (agent.remainingDistance < 0.1f) {
			moveRandom = true;
		}
		else {
			moveRandom = false;
			crawlingSound.Play(0);
		}


		if (agent.remainingDistance == 0 || agent.remainingDistance > 9000)
			moveRandom = true;


//		Debug.Log ("Distance remaining: " + agent.remainingDistance);
//		Debug.Log ("moving randomly: " + moveRandom);

		if(moveRandom && !isAtOrigin) {
//			Debug.Log ("random");
			Vector3 newPos2 = RandomNavSphere (transform.position, 2, -1);
			agent.SetDestination (newPos2);
		}
			
	}

	public static Vector3 RandomNavSphere(Vector3 origin, float dist, int layermask) {
		Vector3 randDirection = Random.insideUnitSphere * dist;

		randDirection += origin;

		NavMeshHit navHit;

		NavMesh.SamplePosition (randDirection, out navHit, dist, layermask);

		return navHit.position;
	}

	public void moveToOrigin(){
		moveRandom = false;
		agent.SetDestination (origin.transform.position);
		Debug.Log ("moving to origin");

	}
}

Events sending data to spacebrew:

using UnityEngine;
using System.Collections;

public class SpacebrewEvents : MonoBehaviour {

	SpacebrewClient sbClient;

	// Use this for initialization
	void Start () {
		GameObject go = GameObject.Find ("SpacebrewObject"); // the name of your client object
		sbClient = go.GetComponent <SpacebrewClient> ();

		// register an event with the client and a callback function here.
		// COMMON GOTCHA: THIS MUST MATCH THE NAME VALUE YOU TYPED IN THE EDITOR!!
		sbClient.addEventListener (this.gameObject, "mystring");
	}

	// Update is called once per frame
	void Update () {
	}

	public void OnSpacebrewEvent(SpacebrewClient.SpacebrewMessage _msg) {
		print ("Received Spacebrew Message");
		print (_msg.value);
	}

	public void SlowServoMsg(){
		print ("Sending Spacebrew Message");
		sbClient.sendMessage("slowServo", "boolean", "true");
	}

	public void FastServoMsg(){
		print ("Sending Spacebrew Message");
		sbClient.sendMessage("fastServo", "boolean", "true");
	}
		

	public IEnumerator FastServoWithDelayMsg() {
		yield return new WaitForSeconds(3.0F);
		SlowServoMsg ();
	}

}

STEP 4: Spacebrew final connections

Make sure that the Raspberry Pi is powered and running.
Make sure that the unity scene is running.

Then:

Connect the servos (Unity) with the switches (MakeyMakey)

(Note: Frida and Miro are for test / debugging purposes only and not required to run the installation.)

Some of our process:

Bodystorming
Physical prototype 1: Chips movement + Electronics
Physical prototype 2: Chips movement + Electronics
Physical prototype 3: Chips movement + Electronics
Physical prototype 4:
Unity prototype 1:

Provide feedback

Saved searches

Use saved searches to filter your results more quickly