forked from PipelineAI/pipeline
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
130 lines (101 loc) · 4.22 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
FROM ubuntu:14.04
ENV SCALA_VERSION=2.10.4
EXPOSE 80 4042 9160 9042 9200 7077 38080 38081 6060 6061 8090 10000 50070 50090 9092 6066 9000 19999 6379 6081 7474 8787 5601 8989 7979 4040 6379
RUN \
apt-get update \
&& apt-get install -y curl \
&& apt-get install -y wget \
&& apt-get install -y vim \
# && apt-get update \
# Start in Home Dir (/root)
&& cd ~ \
# Git
&& apt-get install -y git \
# Retrieve Latest Datasets, Configs, and Start Scripts
&& git clone https://github.com/fluxcapacitor/pipeline.git \
&& chmod a+rx pipeline/*.sh \
# SSH
&& apt-get install -y openssh-server \
# .profile Shell Environment Variables
&& mv ~/.profile ~/.profile.orig \
&& ln -s ~/pipeline/config/bash/.profile ~/.profile \
# Java
&& apt-get install -y default-jdk \
# Apache2 Httpd
&& apt-get install -y apache2 \
# SBT
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/sbt-0.13.8.tgz \
&& tar xvzf sbt-0.13.8.tgz \
&& rm sbt-0.13.8.tgz \
&& cd pipeline \
&& rm -rf /root/.ivy2 \
&& ../sbt/bin/sbt clean clean-files package \
# Start from ~
&& cd ~ \
# Ganglia
&& DEBIAN_FRONTEND=noninteractive apt-get install -y ganglia-monitor rrdtool gmetad ganglia-webfrontend \
# MySql (Required by Hive Metastore)
# Generic Install? https://dev.mysql.com/doc/refman/5.7/en/binary-installation.html
&& DEBIAN_FRONTEND=noninteractive apt-get -y install mysql-server \
&& apt-get install -y mysql-client \
&& apt-get install -y libmysql-java \
# Python Data Science Libraries
&& apt-get install -y python-matplotlib \
&& apt-get install -y python-numpy \
&& apt-get install -y python-scipy \
&& apt-get install -y python-sklearn \
&& apt-get install -y python-dateutil \
&& apt-get install -y python-pandas-lib \
&& apt-get install -y python-numexpr \
&& apt-get install -y python-statsmodels \
# R
&& apt-get install -y r-base \
&& apt-get install -y r-base-dev \
# Logstash
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/logstash-1.5.3.tar.gz \
&& tar xvzf logstash-1.5.3.tar.gz \
&& rm logstash-1.5.3.tar.gz \
# Kibana
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/kibana-4.1.1-linux-x64.tar.gz \
&& tar xvzf kibana-4.1.1-linux-x64.tar.gz \
&& rm kibana-4.1.1-linux-x64.tar.gz \
# Apache Cassandra
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/apache-cassandra-2.2.0-bin.tar.gz \
&& tar xvzf apache-cassandra-2.2.0-bin.tar.gz \
&& rm apache-cassandra-2.2.0-bin.tar.gz \
# Apache Kafka (Confluent Distribution)
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/confluent-1.0-2.10.4.tar.gz \
&& tar xvzf confluent-1.0-2.10.4.tar.gz \
&& rm confluent-1.0-2.10.4.tar.gz \
# ElasticSearch
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/elasticsearch-1.7.1.tar.gz \
&& tar xvzf elasticsearch-1.7.1.tar.gz \
&& rm elasticsearch-1.7.1.tar.gz \
# Apache Spark
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/spark-1.4.1-bin-fluxcapacitor.tgz \
&& tar xvzf spark-1.4.1-bin-fluxcapacitor.tgz \
&& rm spark-1.4.1-bin-fluxcapacitor.tgz \
# Apache Zeppelin
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/zeppelin-0.5.1-spark-1.4.1-hadoop-2.6.0-fluxcapacitor.tar.gz \
&& tar xvzf zeppelin-0.5.1-spark-1.4.1-hadoop-2.6.0-fluxcapacitor.tar.gz \
&& rm zeppelin-0.5.1-spark-1.4.1-hadoop-2.6.0-fluxcapacitor.tar.gz \
# Tachyon (Required by Spark Notebook)
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/tachyon-0.6.4-bin.tar.gz \
&& tar xvfz tachyon-0.6.4-bin.tar.gz \
&& rm tachyon-0.6.4-bin.tar.gz \
# Spark Notebook
&& apt-get install -y screen \
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/spark-notebook-0.6.0-scala-2.10.4-spark-1.4.1-hadoop-2.6.0-with-hive-with-parquet.tgz \
&& tar xvzf spark-notebook-0.6.0-scala-2.10.4-spark-1.4.1-hadoop-2.6.0-with-hive-with-parquet.tgz \
&& rm spark-notebook-0.6.0-scala-2.10.4-spark-1.4.1-hadoop-2.6.0-with-hive-with-parquet.tgz \
# Redis
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/redis-3.0.3.tar.gz \
&& tar -xzvf redis-3.0.3.tar.gz \
&& rm redis-3.0.3.tar.gz \
&& cd redis-3.0.3 \
&& make install \
&& cd ~ \
# Apache Hadoop
&& wget https://s3.amazonaws.com/fluxcapacitor.com/packages/hadoop-2.6.0.tar.gz \
&& tar xvzf hadoop-2.6.0.tar.gz \
&& rm hadoop-2.6.0.tar.gz \