-
Notifications
You must be signed in to change notification settings - Fork 2.4k
/
setup-project
executable file
·35 lines (35 loc) · 1.34 KB
/
setup-project
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#Download spark 1.0.2
echo "Setup spark"
wget http:https://d3kbcqa49mib13.cloudfront.net/spark-1.0.2-bin-hadoop2.tgz
tar -xvf spark-1.0.2-bin-hadoop2.tgz
echo "Installing protobuf"
sudo apt-get install protobuf-compiler
echo $?
# Set up cassandra
echo "Setting up dsc (cassandra"
echo "deb http:https://debian.datastax.com/community stable main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list
curl -L http:https://debian.datastax.com/debian/repo_key | sudo apt-key add -
sudo apt-get update
yes | sudo apt-get install -y --force-yes dsc21
yes | sudo apt-get install -y --force-yes cassandra-tools
echo $?
echo "install urllib3"
sudo pip install urllib3
echo "set up hive directories"
export IAM=`whoami`
sudo mkdir -p /user/hive && sudo chown -R $IAM /user/hive
echo "done with setup"
# Set up kafka
echo "Setting up kafka"
curl http:https://mirrors.ibiblio.org/apache/kafka/0.8.1.1/kafka_2.9.2-0.8.1.1.tgz > kafka.tgz
tar -xzf kafka.tgz
echo "Starting zookeeper"
./bin/zookeeper-server-start.sh config/zookeeper.properties &
echo "Starting kafka"
sleep 5
./bin/kafka-server-start.sh config/server.properties &
sleep 5
# publish a pandas topic to kafka
./bin/kafka-topics.sh --zookeeper localhost:2181 --topic pandas --partition 1 --replication-factor 1 --create
./bin/kafka-topics.sh --zookeeper localhost:2181 --topic logs --partition 1 --replication-factor 1 --create
echo $?