confluent-commands.txt

# https://github.com/alonsoir/kafka-streams-machine-learning-examples/blob/master/tensorflow-image-recognition
# packaging jar file

	mvn clean package

# running project (in a terminal)

	java -cp target/tensorflow-image-recognition-CP53_AK23-jar-with-dependencies.jar com.github.megachucky.kafka.streams.machinelearning.Kafka_Streams_TensorFlow_Image_Recognition_Example

# Run GRPC-TensorFlow-Server docker container

	docker run -it -p 9000:9000 tgowda/inception_serving_tika

	Inside the container, start the Tensorflow Serving server - this deploys the TensorFlow model for Image Recognition

	root@8311ea4e8074:/# /serving/server.sh

# Run Confluent platform (Kafka + Zookeeper)

	confluent local start

# CREATE necessary topics for  

	kafka-topics --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic ImageInputTopic

	kafka-topics --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic ImageOutputTopic

# push some pictures to ImageInputTopic


	echo -e "/Users/aironman/gitProjects/kafka-streams-machine-learning-examples/tensorflow-image-recognition/src/main/resources/TensorFlow_Images/trained_airplane_1.jpg" | kafkacat -b localhost:9092 -P -t ImageInputTopic

	echo -e "/Users/aironman/gitProjects/kafka-streams-machine-learning-examples/tensorflow-image-recognition/src/main/resources/TensorFlow_Images/trained_airplane_2.jpg" | kafkacat -b localhost:9092 -P -t ImageInputTopic


	echo -e "/Users/aironman/gitProjects/kafka-streams-machine-learning-examples/tensorflow-image-recognition/src/main/resources/TensorFlow_Images/trained_butterfly.jpg" | kafkacat -b localhost:9092 -P -t ImageInputTopic

	echo -e "/Users/aironman/gitProjects/kafka-streams-machine-learning-examples/tensorflow-image-recognition/src/main/resources/TensorFlow_Images/new_airplane.jpg" | kafkacat -b localhost:9092 -P -t ImageInputTopic

# consume predictions...

    kafka-console-consumer --bootstrap-server localhost:9092 --topic ImageOutputTopic --from-beginning


# or, in the running jar file terminal...

	Last login: Tue Nov 19 12:25:55 on ttys006
	aironman@MacBook-Pro-de-Alonso tensorflow-image-recognition % fish
	Welcome to fish, the friendly interactive shell
	aironman@MacBook-Pro-de-Alonso ~/g/k/tensorflow-image-recognition> java -cp target/tensorflow-image-recognition-CP53_AK23-jar-with-dependencies.jar com.github.megachucky.kafka.streams.machinelearning.Kafka_Streams_TensorFlow_Image_Recognition_Example
	                                                                   
	SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
	SLF4J: Defaulting to no-operation (NOP) logger implementation
	SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
	Image Recognition Microservice is running...
	Input to Kafka Topic ImageInputTopic; Output to Kafka Topic ImageOutputTopic
	2019-11-19 12:37:44.005196: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations.
	2019-11-19 12:37:44.005228: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations.
	2019-11-19 12:37:44.005233: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations.
	2019-11-19 12:37:44.005236: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations.
	BEST MATCH: space shuttle (47,19% likely)
	BEST MATCH: airliner (63,22% likely)
	BEST MATCH: cabbage butterfly (26,23% likely)
	BEST MATCH: airliner (54,36% likely)

# https://docs.confluent.io/current/quickstart/cos-quickstart.html#cos-quickstart


	docker run -it -p 9000:9000 tgowda/inception_serving_tika

	curl -L https://cnfl.io/cli | sh -s -- -b /Users/aironman/confluent-5.3.1/confluent-cli/bin

	/Users/aironman/confluent-5.3.1/bin/confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:latest

# fish way
	
	set -x JAVA_HOME (/usr/libexec/java_home -v 1.8)

	set -x CONFLUENT_CLI_HOME /Users/aironman/confluent-5.3.1/confluent-cli

	set -x CONFLUENT_HOME /Users/aironman/confluent-5.3.1

# not necessary because java and javac is located within /usr/bin/java

	set PATH $JAVA_HOME $PATH

	set PATH $CONFLUENT_CLI_HOME/bin $PATH

	set PATH $CONFLUENT_HOME/bin $PATH

	confluent local start

#creating some kafka topics 
	
	kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic users


	kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic pageviews

# Install a Kafka Connector and Generate Sample Data
# Run one instance of the Kafka Connect Datagen connector to produce Kafka data to the pageviews topic in AVRO format
	
	wget https://github.com/confluentinc/kafka-connect-datagen/raw/master/config/connector_pageviews_cos.config
	
	curl -X POST -H "Content-Type: application/json" --data @connector_pageviews_cos.config http://localhost:8083/connectors

# Run another instance of the Kafka Connect Datagen connector to produce Kafka data to the users topic in AVRO format

	wget https://github.com/confluentinc/kafka-connect-datagen/raw/master/config/connector_users_cos.config
	curl -X POST -H "Content-Type: application/json" --data @connector_users_cos.config http://localhost:8083/connectors

# Create and Write to a Stream and Table using KSQL
# Start the KSQL CLI in your terminal with this command

	LOG_DIR=./ksql_logs 
	set LOG_DIR ./ksql_logs

	$CONFLUENT_HOME/bin/ksql

# Run these commands within ksql session

	CREATE STREAM pageviews (viewtime BIGINT, userid VARCHAR, pageid VARCHAR) WITH (KAFKA_TOPIC='pageviews', VALUE_FORMAT='AVRO');

	SHOW STREAMS;

	CREATE TABLE users (registertime BIGINT, gender VARCHAR, regionid VARCHAR, userid VARCHAR) WITH (KAFKA_TOPIC='users', VALUE_FORMAT='AVRO', KEY = 'userid');

	SHOW TABLES;

	SET 'auto.offset.reset'='earliest';

	SELECT pageid FROM pageviews LIMIT 3;

	CREATE STREAM pageviews_female AS SELECT users.userid AS userid, pageid, regionid, gender FROM pageviews LEFT JOIN users ON pageviews.userid = users.userid WHERE gender = 'FEMALE';

	CREATE STREAM pageviews_female_like_89 WITH (kafka_topic='pageviews_enriched_r8_r9', value_format='AVRO') AS SELECT * FROM pageviews_female WHERE regionid LIKE '%_8' OR regionid LIKE '%_9';

	CREATE TABLE pageviews_regions AS SELECT gender, regionid , COUNT(*) AS numusers FROM pageviews_female WINDOW TUMBLING (size 30 second) GROUP BY gender, regionid HAVING COUNT(*) > 1;

# Monitor Streaming Data

	DESCRIBE EXTENDED pageviews_female_like_89;

	show queries;

	EXPLAIN CTAS_PAGEVIEWS_REGIONS_2;


# confluent local stop

# confluent local destroy