diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java
index aa453a0ef644f..3a3aac17fc149 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java
@@ -38,7 +38,16 @@ public abstract class AbstractJobResult implements IOReadableWritable {
* @author warneke
*/
public enum ReturnCode {
- SUCCESS, ERROR
+
+ /**
+ * The success return code.
+ */
+ SUCCESS,
+
+ /**
+ * The error return code.
+ */
+ ERROR
};
/**
@@ -156,23 +165,23 @@ public boolean equals(final Object obj) {
return true;
}
-
+
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
-
+
long hashCode = 0;
-
- if(this.returnCode != null) {
+
+ if (this.returnCode != null) {
hashCode += this.returnCode.hashCode();
}
- if(this.description != null) {
+ if (this.description != null) {
hashCode += this.description.hashCode();
}
-
+
return (int) (hashCode % Integer.MAX_VALUE);
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java
index ffbb78f174254..f73ccd686ac8d 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java
@@ -33,7 +33,7 @@ public class JobCancelResult extends AbstractJobResult {
* @param description
* the optional error description
*/
- public JobCancelResult(ReturnCode returnCode, String description) {
+ public JobCancelResult(final ReturnCode returnCode, final String description) {
super(returnCode, description);
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java
index 2be224bfd2a3e..b4b2718f742ec 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java
@@ -38,6 +38,7 @@
/**
* The job client is able to submit, control, and abort jobs.
+ *
* This class is thread-safe.
*
* @author warneke
@@ -45,7 +46,7 @@
public class JobClient {
/**
- * The logging object used for debugging
+ * The logging object used for debugging.
*/
private static final Log LOG = LogFactory.getLog(JobClient.class);
@@ -94,7 +95,7 @@ public static class JobCleanUp extends Thread {
* @param jobClient
* the job client this clean up object belongs to
*/
- public JobCleanUp(JobClient jobClient) {
+ public JobCleanUp(final JobClient jobClient) {
this.jobClient = jobClient;
}
@@ -134,7 +135,7 @@ public void run() {
* @throws IOException
* thrown on error while initializing the RPC connection to the job manager
*/
- public JobClient(JobGraph jobGraph) throws IOException {
+ public JobClient(final JobGraph jobGraph) throws IOException {
this(jobGraph, new Configuration());
}
@@ -150,7 +151,7 @@ public JobClient(JobGraph jobGraph) throws IOException {
* @throws IOException
* thrown on error while initializing the RPC connection to the job manager
*/
- public JobClient(JobGraph jobGraph, Configuration configuration) throws IOException {
+ public JobClient(final JobGraph jobGraph, final Configuration configuration) throws IOException {
final String address = configuration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
final int port = configuration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
@@ -163,8 +164,7 @@ public JobClient(JobGraph jobGraph, Configuration configuration) throws IOExcept
this.configuration = configuration;
this.jobCleanUp = new JobCleanUp(this);
}
-
-
+
/**
* Constructs a new job client object and instantiates a local
* RPC proxy for the {@link JobSubmissionProtocol}.
@@ -177,16 +177,18 @@ public JobClient(JobGraph jobGraph, Configuration configuration) throws IOExcept
* IP/Port of the jobmanager (not taken from provided configuration object).
* @throws IOException
* thrown on error while initializing the RPC connection to the job manager
- */
- public JobClient(JobGraph jobGraph, Configuration configuration, InetSocketAddress jobManagerAddress) throws IOException {
-
+ */
+ public JobClient(final JobGraph jobGraph, final Configuration configuration,
+ final InetSocketAddress jobManagerAddress)
+ throws IOException {
- this.jobSubmitClient = (JobManagementProtocol) RPC.getProxy(JobManagementProtocol.class, jobManagerAddress, NetUtils
- .getSocketFactory());
+ this.jobSubmitClient = (JobManagementProtocol) RPC.getProxy(JobManagementProtocol.class, jobManagerAddress,
+ NetUtils
+ .getSocketFactory());
this.jobGraph = jobGraph;
this.configuration = configuration;
this.jobCleanUp = new JobCleanUp(this);
- }
+ }
/**
* Close the JobClient
.
@@ -328,8 +330,11 @@ public void submitJobAndWait() throws IOException, JobExecutionException {
} else if (jobStatus == JobStatus.CANCELED || jobStatus == JobStatus.FAILED) {
Runtime.getRuntime().removeShutdownHook(this.jobCleanUp);
LOG.info(jobEvent.getOptionalMessage());
- throw new JobExecutionException(jobEvent.getOptionalMessage(),
- (jobStatus == JobStatus.CANCELED) ? true : false);
+ if (jobStatus == JobStatus.CANCELED) {
+ throw new JobExecutionException(jobEvent.getOptionalMessage(), true);
+ } else {
+ throw new JobExecutionException(jobEvent.getOptionalMessage(), false);
+ }
}
}
}
@@ -353,7 +358,7 @@ public void submitJobAndWait() throws IOException, JobExecutionException {
* @throws IOException
* thrown after the error message is written to the log
*/
- private void logErrorAndRethrow(String errorMessage) throws IOException {
+ private void logErrorAndRethrow(final String errorMessage) throws IOException {
LOG.error(errorMessage);
throw new IOException(errorMessage);
@@ -366,7 +371,7 @@ private void logErrorAndRethrow(String errorMessage) throws IOException {
* @param sleepTime
* the sleep time in milliseconds after which old events shall be removed from the processed event queue
*/
- private void cleanUpOldEvents(long sleepTime) {
+ private void cleanUpOldEvents(final long sleepTime) {
long mostRecentTimestamp = 0;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java
index 6e6cbb86d4130..58f6f0d0c8db5 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java
@@ -28,6 +28,9 @@ public class JobExecutionException extends Exception {
*/
private static final long serialVersionUID = 2818087325120827525L;
+ /**
+ * Indicates whether the job has been aborted as a result of user request.
+ */
private final boolean canceledByUser;
/**
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java
index 631f4cc968a52..4104eed9f8072 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java
@@ -46,7 +46,9 @@ public class JobProgressResult extends AbstractJobResult {
* @param events
* the job events to be transported within this object
*/
- public JobProgressResult(ReturnCode returnCode, String description, SerializableArrayList events) {
+ public JobProgressResult(final ReturnCode returnCode, final String description,
+ final SerializableArrayList events) {
+
super(returnCode, description);
this.events = events;
@@ -65,7 +67,7 @@ public JobProgressResult() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
super.read(in);
this.events.read(in);
@@ -75,7 +77,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
super.write(out);
this.events.write(out);
@@ -90,36 +92,36 @@ public Iterator getEvents() {
return this.events.iterator();
}
-
+
/**
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
-
- if(!super.equals(obj)) {
+ public boolean equals(final Object obj) {
+
+ if (!super.equals(obj)) {
return false;
}
-
- if(!(obj instanceof JobProgressResult)) {
+
+ if (!(obj instanceof JobProgressResult)) {
return false;
}
-
+
final JobProgressResult jpr = (JobProgressResult) obj;
-
- if(!this.events.equals(jpr.events)) {
+
+ if (!this.events.equals(jpr.events)) {
return false;
}
-
+
return true;
}
-
+
/**
* {@inheritDoc}
*/
@Override
public int hashCode() {
-
+
return super.hashCode();
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java
index 34d19846cd523..0322b46c4eb04 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java
@@ -37,7 +37,7 @@ public class JobSubmissionResult extends AbstractJobResult {
* @param description
* the error description
*/
- public JobSubmissionResult(ReturnCode returnCode, String description) {
+ public JobSubmissionResult(final ReturnCode returnCode, final String description) {
super(returnCode, description);
}
@@ -54,7 +54,7 @@ public JobSubmissionResult() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
super.read(in);
}
@@ -62,7 +62,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
super.write(out);
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java
index 669d7ab480377..700aca1ca3972 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java
@@ -77,7 +77,7 @@ public final class ConfigConstants {
public static final int DEFAULT_JOB_MANAGER_IPC_PORT = 6123;
/**
- * The default network port the task manager expects incoming IPC connections
+ * The default network port the task manager expects incoming IPC connections.
*/
public static final int DEFAULT_TASK_MANAGER_IPC_PORT = 6122;
@@ -97,7 +97,7 @@ public final class ConfigConstants {
public static final long DEFAULT_MEMORY_MANAGER_MIN_UNRESERVED_MEMORY = 256 * 1024 * 1024;
/**
- * The default directory for temporary files of the task manager
+ * The default directory for temporary files of the task manager.
*/
public static final String DEFAULT_TASK_MANAGER_TMP_PATH = System.getProperty("java.io.tmpdir");
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java
index 1d4365d97ee1f..6283a67dae80a 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java
@@ -59,7 +59,7 @@ public class Configuration implements IOReadableWritable {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public String getString(String key, String defaultValue) {
+ public String getString(final String key, final String defaultValue) {
synchronized (this.confData) {
@@ -88,10 +88,11 @@ public String getString(String key, String defaultValue) {
* @see #setClass(String, Class)
*/
@SuppressWarnings("unchecked")
- public Class getClass(String key, Class extends T> defaultValue, Class ancestor) {
+ public Class getClass(final String key, final Class extends T> defaultValue, final Class ancestor) {
String className = getString(key, null);
- if (className == null)
+ if (className == null) {
return (Class) defaultValue;
+ }
try {
return (Class) Class.forName(className);
} catch (ClassNotFoundException e) {
@@ -111,7 +112,7 @@ public Class getClass(String key, Class extends T> defaultValue, Class<
* if the class identified by the associated value cannot be resolved
* @see #setClass(String, Class)
*/
- public Class> getClass(String key, Class> defaultValue) {
+ public Class> getClass(final String key, final Class> defaultValue) {
return getClass(key, defaultValue, Object.class);
}
@@ -126,7 +127,7 @@ public Class> getClass(String key, Class> defaultValue) {
* @see #getClass(String, Class)
* @see #getClass(String, Class, Class)
*/
- public void setClass(String key, Class> klazz) {
+ public void setClass(final String key, final Class> klazz) {
setString(key, klazz.getName());
}
@@ -139,7 +140,7 @@ public void setClass(String key, Class> klazz) {
* @param value
* the value of the key/value pair to be added
*/
- public void setString(String key, String value) {
+ public void setString(final String key, final String value) {
if (key == null || value == null) {
// TODO: should probably throw an NullPointerException
@@ -161,7 +162,7 @@ public void setString(String key, String value) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public int getInteger(String key, int defaultValue) {
+ public int getInteger(final String key, final int defaultValue) {
int retVal = defaultValue;
@@ -188,7 +189,7 @@ public int getInteger(String key, int defaultValue) {
* @param value
* the value of the key/value pair to be added
*/
- public void setInteger(String key, int value) {
+ public void setInteger(final String key, final int value) {
if (key == null) {
LOG.warn("Cannot set integer: Given key is null!");
@@ -199,7 +200,7 @@ public void setInteger(String key, int value) {
this.confData.put(key, Integer.toString(value));
}
}
-
+
/**
* Returns the value associated with the given key as a long.
*
@@ -209,7 +210,7 @@ public void setInteger(String key, int value) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public long getLong(String key, long defaultValue) {
+ public long getLong(final String key, final long defaultValue) {
try {
synchronized (this.confData) {
String val = this.confData.get(key);
@@ -217,8 +218,7 @@ public long getLong(String key, long defaultValue) {
return Long.parseLong(val);
}
}
- }
- catch (NumberFormatException e) {
+ } catch (NumberFormatException e) {
LOG.debug(e);
}
@@ -234,7 +234,7 @@ public long getLong(String key, long defaultValue) {
* @param value
* the value of the key/value pair to be added
*/
- public void setLong(String key, long value) {
+ public void setLong(final String key, final long value) {
if (key == null) {
LOG.warn("Cannot set integer: Given key is null!");
return;
@@ -254,7 +254,7 @@ public void setLong(String key, long value) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public boolean getBoolean(String key, boolean defaultValue) {
+ public boolean getBoolean(final String key, final boolean defaultValue) {
boolean retVal = defaultValue;
@@ -277,7 +277,7 @@ public boolean getBoolean(String key, boolean defaultValue) {
* @param value
* the value of the key/value pair to be added
*/
- public void setBoolean(String key, boolean value) {
+ public void setBoolean(final String key, final boolean value) {
if (key == null) {
LOG.warn("Cannot set boolean: Given key is null!");
@@ -288,7 +288,7 @@ public void setBoolean(String key, boolean value) {
this.confData.put(key, Boolean.toString(value));
}
}
-
+
/**
* Returns the value associated with the given key as a float.
*
@@ -298,9 +298,8 @@ public void setBoolean(String key, boolean value) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public float getFloat(String key, float defaultValue) {
- synchronized (this.confData)
- {
+ public float getFloat(final String key, final float defaultValue) {
+ synchronized (this.confData) {
String val = this.confData.get(key);
return val == null ? defaultValue : Float.parseFloat(val);
}
@@ -315,7 +314,7 @@ public float getFloat(String key, float defaultValue) {
* @param value
* the value of the key/value pair to be added
*/
- public void setFloat(String key, float value) {
+ public void setFloat(final String key, final float value) {
if (key == null) {
LOG.warn("Cannot set boolean: Given key is null!");
@@ -353,7 +352,7 @@ public Set keySet() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
synchronized (this.confData) {
@@ -371,7 +370,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
synchronized (this.confData) {
@@ -396,7 +395,7 @@ public int hashCode() {
}
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (this == obj) {
return true;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java
index 3b2ff9be144b6..c5751f35ecf41 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java
@@ -45,7 +45,7 @@
*
* @author warneke
*/
-public class GlobalConfiguration {
+public final class GlobalConfiguration {
/**
* The log object used for debugging.
@@ -72,7 +72,7 @@ public class GlobalConfiguration {
*
* @return the global configuration object
*/
- private synchronized static GlobalConfiguration get() {
+ private static synchronized GlobalConfiguration get() {
if (configuration == null) {
configuration = new GlobalConfiguration();
@@ -96,7 +96,7 @@ private GlobalConfiguration() {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public static String getString(String key, String defaultValue) {
+ public static String getString(final String key, final String defaultValue) {
return get().getStringInternal(key, defaultValue);
}
@@ -110,7 +110,7 @@ public static String getString(String key, String defaultValue) {
* defaultValue the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- private String getStringInternal(String key, String defaultValue) {
+ private String getStringInternal(final String key, final String defaultValue) {
synchronized (this.confData) {
@@ -131,7 +131,7 @@ private String getStringInternal(String key, String defaultValue) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public static int getInteger(String key, int defaultValue) {
+ public static int getInteger(final String key, final int defaultValue) {
return get().getIntegerInternal(key, defaultValue);
}
@@ -145,7 +145,7 @@ public static int getInteger(String key, int defaultValue) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- private int getIntegerInternal(String key, int defaultValue) {
+ private int getIntegerInternal(final String key, final int defaultValue) {
int retVal = defaultValue;
@@ -157,7 +157,10 @@ private int getIntegerInternal(String key, int defaultValue) {
}
}
} catch (NumberFormatException e) {
- // Nothing to do here
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(StringUtils.stringifyException(e));
+ }
}
return retVal;
@@ -172,7 +175,7 @@ private int getIntegerInternal(String key, int defaultValue) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- public static boolean getBoolean(String key, boolean defaultValue) {
+ public static boolean getBoolean(final String key, final boolean defaultValue) {
return get().getBooleanInternal(key, defaultValue);
}
@@ -186,7 +189,7 @@ public static boolean getBoolean(String key, boolean defaultValue) {
* the default value which is returned in case there is no value associated with the given key
* @return the (default) value associated with the given key
*/
- private boolean getBooleanInternal(String key, boolean defaultValue) {
+ private boolean getBooleanInternal(final String key, final boolean defaultValue) {
boolean retVal = defaultValue;
@@ -207,7 +210,7 @@ private boolean getBooleanInternal(String key, boolean defaultValue) {
* @param configDir
* the directory which contains the configuration files
*/
- public static void loadConfiguration(String configDir) {
+ public static void loadConfiguration(final String configDir) {
if (configDir == null) {
@@ -223,9 +226,9 @@ public static void loadConfiguration(String configDir) {
}
// get all XML files in the directory
- File[] files = confDirFile.listFiles(new FilenameFilter() {
+ final File[] files = confDirFile.listFiles(new FilenameFilter() {
@Override
- public boolean accept(File dir, String name) {
+ public boolean accept(final File dir, final String name) {
return dir == confDirFile && name != null && name.endsWith(".xml");
}
@@ -253,7 +256,7 @@ public boolean accept(File dir, String name) {
* @param uri
* the URI pointing to the XML document
*/
- private void loadResource(String uri) {
+ private void loadResource(final String uri) {
final DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
// Ignore comments in the XML file
@@ -397,7 +400,7 @@ public static Configuration getConfiguration() {
* array of keys specifying the subset of pairs to copy.
* @return the {@link Configuration} object including the key/value pairs
*/
- public static Configuration getConfiguration(String[] keys) {
+ public static Configuration getConfiguration(final String[] keys) {
return get().getConfigurationInternal(keys);
}
@@ -409,7 +412,7 @@ public static Configuration getConfiguration(String[] keys) {
* array of keys specifying the subset of pairs to copy.
* @return the {@link Configuration} object including the key/value pairs
*/
- private Configuration getConfigurationInternal(String[] keys) {
+ private Configuration getConfigurationInternal(final String[] keys) {
Configuration conf = new Configuration();
@@ -450,7 +453,7 @@ private Configuration getConfigurationInternal(String[] keys) {
* @param conf
* the {@link Configuration} object to merge into the global configuration
*/
- public static void includeConfiguration(Configuration conf) {
+ public static void includeConfiguration(final Configuration conf) {
get().includeConfigurationInternal(conf);
}
@@ -461,7 +464,7 @@ public static void includeConfiguration(Configuration conf) {
* @param conf
* the {@link Configuration} object to merge into the global configuration
*/
- private void includeConfigurationInternal(Configuration conf) {
+ private void includeConfigurationInternal(final Configuration conf) {
if (conf == null) {
LOG.error("Given configuration object is null, ignoring it...");
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java
index 2602eabe1c341..53af7b813a44a 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java
@@ -32,9 +32,9 @@
public abstract class AbstractEvent implements IOReadableWritable {
/**
- * Auxiliary object which helps to convert a {@link Date} object to the given string representation
+ * Auxiliary object which helps to convert a {@link Date} object to the given string representation.
*/
- private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
+ private static final SimpleDateFormat DATA_FORMATTER = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
/**
* The timestamp of the event.
@@ -47,7 +47,7 @@ public abstract class AbstractEvent implements IOReadableWritable {
* @param timestamp
* the timestamp of the event.
*/
- public AbstractEvent(long timestamp) {
+ public AbstractEvent(final long timestamp) {
this.timestamp = timestamp;
}
@@ -63,7 +63,7 @@ public AbstractEvent() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
// Read the timestamp
this.timestamp = in.readLong();
@@ -73,7 +73,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
// Write the timestamp
out.writeLong(this.timestamp);
@@ -96,9 +96,9 @@ public long getTimestamp() {
* the timestamp in milliseconds since the beginning of "the epoch"
* @return the string unified representation of the timestamp
*/
- public static String timestampToString(long timestamp) {
+ public static String timestampToString(final long timestamp) {
- return dateFormatter.format(new Date(timestamp));
+ return DATA_FORMATTER.format(new Date(timestamp));
}
@@ -106,7 +106,7 @@ public static String timestampToString(long timestamp) {
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (obj instanceof AbstractEvent) {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java
index c414fd7497588..ac6eda30bfd14 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java
@@ -51,7 +51,7 @@ public class JobEvent extends AbstractEvent {
* @param optionalMessage
* an optional message that shall be attached to this event, possibly null
*/
- public JobEvent(long timestamp, JobStatus currentJobStatus, String optionalMessage) {
+ public JobEvent(final long timestamp, final JobStatus currentJobStatus, final String optionalMessage) {
super(timestamp);
this.currentJobStatus = currentJobStatus;
@@ -73,7 +73,7 @@ public JobEvent() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
super.read(in);
// Read job status
@@ -87,7 +87,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
super.write(out);
// Write job status
@@ -128,7 +128,7 @@ public String toString() {
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (!super.equals(obj)) {
return false;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java
index fef5d39a0d28f..45213848d682e 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java
@@ -81,8 +81,9 @@ public class VertexEvent extends AbstractEvent {
* @param description
* an optional description
*/
- public VertexEvent(long timestamp, JobVertexID jobVertexID, String jobVertexName, int totalNumberOfSubtasks,
- int indexOfSubtask, ExecutionState currentExecutionState, String description) {
+ public VertexEvent(final long timestamp, final JobVertexID jobVertexID, final String jobVertexName,
+ final int totalNumberOfSubtasks, final int indexOfSubtask, final ExecutionState currentExecutionState,
+ final String description) {
super(timestamp);
this.jobVertexID = jobVertexID;
this.jobVertexName = jobVertexName;
@@ -112,7 +113,7 @@ public VertexEvent() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
super.read(in);
@@ -128,7 +129,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
super.write(out);
@@ -210,7 +211,7 @@ public String toString() {
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (!super.equals(obj)) {
return false;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java
index 169562fa98635..c1fd6d4dc89f5 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java
@@ -43,7 +43,7 @@ public class EventNotificationManager {
* @param eventType
* the event type the given listener object wants to be notified about
*/
- public void subscribeToEvent(EventListener eventListener, Class extends AbstractTaskEvent> eventType) {
+ public void subscribeToEvent(final EventListener eventListener, final Class extends AbstractTaskEvent> eventType) {
synchronized (this.subscriptions) {
@@ -65,7 +65,7 @@ public void subscribeToEvent(EventListener eventListener, Class extends Abstra
* @param eventType
* the event type to remove the subscription for
*/
- public void unsubscribeFromEvent(EventListener eventListener, Class extends AbstractEvent> eventType) {
+ public void unsubscribeFromEvent(final EventListener eventListener, final Class extends AbstractEvent> eventType) {
synchronized (this.subscriptions) {
@@ -87,7 +87,7 @@ public void unsubscribeFromEvent(EventListener eventListener, Class extends Ab
* @param event
* the event to deliver
*/
- public void deliverEvent(AbstractTaskEvent event) {
+ public void deliverEvent(final AbstractTaskEvent event) {
synchronized (this.subscriptions) {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/IntegerTaskEvent.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/IntegerTaskEvent.java
index 24144613287c8..24b2c6a8978d4 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/IntegerTaskEvent.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/IntegerTaskEvent.java
@@ -43,8 +43,14 @@
*/
public class IntegerTaskEvent extends AbstractTaskEvent {
+ /**
+ * The integer value transported by this integer task event.
+ */
private int value = -1;
+ /**
+ * Default constructor (should only be used for deserialization).
+ */
public IntegerTaskEvent() {
// default constructor implementation.
// should only be used for deserialization
@@ -54,6 +60,7 @@ public IntegerTaskEvent() {
* Constructs a new integer task event.
*
* @param value
+ * the integer value to be transported inside this integer task event
*/
public IntegerTaskEvent(final int value) {
this.value = value;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/Environment.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/Environment.java
index 817763f5d9701..c3c0bd5fbf106 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/Environment.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/Environment.java
@@ -166,8 +166,8 @@ public class Environment implements Runnable, IOReadableWritable {
* @param runtimeConfiguration
* the configuration object which was attached to the original {@link JobVertex}
*/
- public Environment(JobID jobID, String taskName, Class extends AbstractInvokable> invokableClass,
- Configuration runtimeConfiguration) {
+ public Environment(final JobID jobID, final String taskName,
+ final Class extends AbstractInvokable> invokableClass, final Configuration runtimeConfiguration) {
this.jobID = jobID;
this.taskName = taskName;
this.invokableClass = invokableClass;
@@ -216,7 +216,7 @@ public JobID getJobID() {
* @param executionListener
* the object to be notified for important events during the task execution
*/
- public void registerExecutionListener(ExecutionListener executionListener) {
+ public void registerExecutionListener(final ExecutionListener executionListener) {
synchronized (this.executionListeners) {
@@ -233,7 +233,7 @@ public void registerExecutionListener(ExecutionListener executionListener) {
* @param executionListener
* the lister object to be unregistered
*/
- public void unregisterExecutionListener(ExecutionListener executionListener) {
+ public void unregisterExecutionListener(final ExecutionListener executionListener) {
synchronized (this.executionListeners) {
this.executionListeners.remove(executionListener);
@@ -267,7 +267,7 @@ public boolean hasUnboundOutputGates() {
* the index of the unbound output gate
* @return the unbound output gate with the given ID, or null
if no such gate exists
*/
- public OutputGate extends Record> getUnboundOutputGate(int gateID) {
+ public OutputGate extends Record> getUnboundOutputGate(final int gateID) {
if (this.unboundOutputGates.size() == 0) {
LOG.debug("No unbound output gates");
@@ -283,7 +283,7 @@ public OutputGate extends Record> getUnboundOutputGate(int gateID) {
* the index of the unbound input gate
* @return the unbound input gate with the given ID, or null
if no such gate exists
*/
- public InputGate extends Record> getUnboundInputGate(int gateID) {
+ public InputGate extends Record> getUnboundInputGate(final int gateID) {
if (this.unboundInputGates.size() == 0) {
LOG.debug("No unbound input gates");
@@ -440,7 +440,7 @@ private void activateInputChannels() throws IOException, InterruptedException {
* @param outputGate
* the output gate to be registered with the environment
*/
- public void registerOutputGate(OutputGate extends Record> outputGate) {
+ public void registerOutputGate(final OutputGate extends Record> outputGate) {
LOG.debug("Registering output gate");
this.outputGates.add(outputGate);
}
@@ -451,7 +451,7 @@ public void registerOutputGate(OutputGate extends Record> outputGate) {
* @param inputGate
* the input gate to be registered with the environment
*/
- public void registerInputGate(InputGate extends Record> inputGate) {
+ public void registerInputGate(final InputGate extends Record> inputGate) {
LOG.debug("Registering input gate");
this.inputGates.add(inputGate);
}
@@ -481,7 +481,7 @@ public int getNumberOfInputGates() {
* the index of the input gate to return
* @return the input gate at index pos
or null
if no such index exists
*/
- public InputGate extends Record> getInputGate(int pos) {
+ public InputGate extends Record> getInputGate(final int pos) {
if (pos < this.inputGates.size()) {
return this.inputGates.get(pos);
}
@@ -496,7 +496,7 @@ public InputGate extends Record> getInputGate(int pos) {
* the index of the output gate to return
* @return the output gate at index pos
or null
if no such index exists
*/
- public OutputGate extends Record> getOutputGate(int pos) {
+ public OutputGate extends Record> getOutputGate(final int pos) {
if (pos < this.outputGates.size()) {
return this.outputGates.get(pos);
}
@@ -510,7 +510,11 @@ public OutputGate extends Record> getOutputGate(int pos) {
public void startExecution() {
if (this.executingThread == null) {
- this.executingThread = new Thread(this, this.taskName);
+ if (this.taskName != null) {
+ this.executingThread = new Thread(this, this.taskName);
+ } else {
+ this.executingThread = new Thread(this);
+ }
this.executingThread.start();
}
}
@@ -560,7 +564,7 @@ public void cancelExecution() {
*/
@SuppressWarnings("unchecked")
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
// Read job vertex id
this.jobID = new JobID();
@@ -715,7 +719,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
// Write out job vertex id
if (this.jobID == null) {
@@ -915,7 +919,7 @@ public IOManager getIOManager() {
* @param memoryManager
* the new {@link IOManager}
*/
- public void setIOManager(IOManager ioManager) {
+ public void setIOManager(final IOManager ioManager) {
this.ioManager = ioManager;
}
@@ -934,7 +938,7 @@ public MemoryManager getMemoryManager() {
* @param memoryManager
* the new {@link MemoryManager}
*/
- public void setMemoryManager(MemoryManager memoryManager) {
+ public void setMemoryManager(final MemoryManager memoryManager) {
this.memoryManager = memoryManager;
}
@@ -963,7 +967,7 @@ public int getCurrentNumberOfSubtasks() {
* @param currentNumberOfSubtasks
* the current number of subtasks the respective task is split into
*/
- public void setCurrentNumberOfSubtasks(int currentNumberOfSubtasks) {
+ public void setCurrentNumberOfSubtasks(final int currentNumberOfSubtasks) {
this.currentNumberOfSubtasks = currentNumberOfSubtasks;
}
@@ -984,12 +988,12 @@ public int getIndexInSubtaskGroup() {
* @param indexInSubtaskGroup
* the index of this subtask in the subtask group
*/
- public void setIndexInSubtaskGroup(int indexInSubtaskGroup) {
+ public void setIndexInSubtaskGroup(final int indexInSubtaskGroup) {
this.indexInSubtaskGroup = indexInSubtaskGroup;
}
- public void changeExecutionState(ExecutionState newExecutionState, String optionalMessage) {
+ public void changeExecutionState(final ExecutionState newExecutionState, final String optionalMessage) {
// Ignore state changes in final states
if (this.executionState == ExecutionState.CANCELED || this.executionState == ExecutionState.FINISHED
@@ -1132,7 +1136,7 @@ public InputSplitProvider getInputSplitProvider() {
* @param userThread
* the user thread which has been started
*/
- public void userThreadStarted(Thread userThread) {
+ public void userThreadStarted(final Thread userThread) {
synchronized (this.executionListeners) {
final Iterator it = this.executionListeners.iterator();
@@ -1149,7 +1153,7 @@ public void userThreadStarted(Thread userThread) {
* @param userThread
* the user thread which has finished
*/
- public void userThreadFinished(Thread userThread) {
+ public void userThreadFinished(final Thread userThread) {
synchronized (this.executionListeners) {
final Iterator it = this.executionListeners.iterator();
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java
index d8ad91e887736..5e803a53d03a9 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java
@@ -63,7 +63,7 @@ public final class ExecutionSignature {
* @param signature
* the byte buffer containing the signature.
*/
- private ExecutionSignature(byte[] signature) {
+ private ExecutionSignature(final byte[] signature) {
this.signature = signature;
}
@@ -76,7 +76,8 @@ private ExecutionSignature(byte[] signature) {
* the ID of the job
* @return the cryptographic signature of this vertex
*/
- public static ExecutionSignature createSignature(Class extends AbstractInvokable> invokableClass, JobID jobID) {
+ public static ExecutionSignature createSignature(final Class extends AbstractInvokable> invokableClass,
+ final JobID jobID) {
// First, try to load message digest algorithm, if necessary
if (messageDigest == null) {
@@ -117,7 +118,7 @@ public static ExecutionSignature createSignature(Class extends AbstractInvokab
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (obj instanceof ExecutionSignature) {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheManager.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheManager.java
index 41fad94ce711f..310448fb3d11e 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheManager.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheManager.java
@@ -45,7 +45,7 @@
*
* @author warneke
*/
-public class LibraryCacheManager {
+public final class LibraryCacheManager {
/**
* The instance of the library cache manager accessible through a singleton pattern.
@@ -100,7 +100,7 @@ public class LibraryCacheManager {
* @throws IOException
* thrown if access to the file system can not be obtained or the requested hashing algorithm does not exist
*/
- private synchronized static LibraryCacheManager get() throws IOException {
+ private static synchronized LibraryCacheManager get() throws IOException {
// Lazy initialization
if (libraryManager == null) {
@@ -116,8 +116,7 @@ private synchronized static LibraryCacheManager get() throws IOException {
* @throws IOException
* thrown if access to the file system can not be obtained or the requested hashing algorithm does not exist
*/
- private LibraryCacheManager()
- throws IOException {
+ private LibraryCacheManager() throws IOException {
// Check if the library cache directory exists, otherwise create it
final String tmp = System.getProperty("java.io.tmpdir");
@@ -156,43 +155,6 @@ public static void setLocalMode() throws IOException {
get().localMode = true;
}
- /**
- * Creates a mapping between a job vertex ID and the corresponding graph ID. The mapping is required to
- * unambiguously translate
- * the client path of a library to its internal cache name.
- *
- * @param vertexID
- * the job vertex ID for the mapping
- * @param graphID
- * the graph ID for the mapping
- * @throws IOException
- * thrown if the library cache manager could not be instantiated
- */
- /*
- * public static void createMapping(JobVertexID vertexID, JobGraphID graphID) throws IOException {
- * LibraryCacheManager lib = get();
- * lib.createMappingInternal(vertexID, graphID);
- * }
- */
-
- /**
- * Creates a mapping between a job vertex ID and the corresponding graph ID. The mapping is required to
- * unambiguously translate
- * the client path of a library to its internal cache name.
- *
- * @param vertexID
- * the job vertex ID for the mapping
- * @param graphID
- * the graph ID for the mapping
- */
- /*
- * private void createMappingInternal(JobVertexID vertexID, JobGraphID graphID) {
- * synchronized(this.vertexIDToGraphIDMap) {
- * this.vertexIDToGraphIDMap.put(vertexID, graphID);
- * }
- * }
- */
-
/**
* Registers a job ID with a set of library paths that are required to run the job. The library paths are given in
* terms
@@ -208,7 +170,7 @@ public static void setLocalMode() throws IOException {
* thrown if the library cache manager could not be instantiated, no mapping between the job ID and a job ID
* exists or the requested library is not in the cache.
*/
- public static void register(JobID id, Path[] clientPaths) throws IOException {
+ public static void register(final JobID id, final Path[] clientPaths) throws IOException {
final LibraryCacheManager lib = get();
lib.registerInternal(id, clientPaths);
@@ -228,7 +190,7 @@ public static void register(JobID id, Path[] clientPaths) throws IOException {
* @throws IOException
* thrown if no mapping between the job ID and a job ID exists or the requested library is not in the cache.
*/
- private void registerInternal(JobID id, Path[] clientPaths) throws IOException {
+ private void registerInternal(final JobID id, final Path[] clientPaths) throws IOException {
final String[] cacheNames = new String[clientPaths.length];
synchronized (this.clientPathToCacheName) {
@@ -258,7 +220,7 @@ private void registerInternal(JobID id, Path[] clientPaths) throws IOException {
* thrown if the library cache manager could not be instantiated or one of the requested libraries is not in
* the cache
*/
- public static void register(JobID id, String[] requiredJarFiles) throws IOException {
+ public static void register(final JobID id, final String[] requiredJarFiles) throws IOException {
final LibraryCacheManager lib = get();
lib.registerInternal(id, requiredJarFiles);
@@ -276,7 +238,7 @@ public static void register(JobID id, String[] requiredJarFiles) throws IOExcept
* @throws IOException
* thrown if one of the requested libraries is not in the cache
*/
- private void registerInternal(JobID id, String[] requiredJarFiles) throws IOException {
+ private void registerInternal(final JobID id, final String[] requiredJarFiles) throws IOException {
// Check if library manager entry for this id already exists
synchronized (this.libraryManagerEntries) {
@@ -293,8 +255,9 @@ private void registerInternal(JobID id, String[] requiredJarFiles) throws IOExce
for (int i = 0; i < requiredJarFiles.length; i++) {
final Path p = contains(requiredJarFiles[i]);
- if (p == null)
+ if (p == null) {
throw new IOException(requiredJarFiles[i] + " does not exist in the library cache");
+ }
// Add file to the URL array
try {
@@ -319,7 +282,7 @@ private void registerInternal(JobID id, String[] requiredJarFiles) throws IOExce
* @throws IOException
* thrown if the library cache manager could not be instantiated
*/
- public static void unregister(JobID id) throws IOException {
+ public static void unregister(final JobID id) throws IOException {
final LibraryCacheManager lib = get();
lib.unregisterInternal(id);
@@ -331,7 +294,7 @@ public static void unregister(JobID id) throws IOException {
* @param id
* the job ID to unregister
*/
- private void unregisterInternal(JobID id) {
+ private void unregisterInternal(final JobID id) {
// TODO: the library cache manager (LCM) was designed to be a singleton object
// Running Nephele is local mode confuses the LCM and it deallocates libraries
@@ -357,7 +320,7 @@ private void unregisterInternal(JobID id) {
* thrown if the library cache manager could not be instantiated or no access to the file system could be
* obtained
*/
- public static Path contains(String cacheName) throws IOException {
+ public static Path contains(final String cacheName) throws IOException {
final LibraryCacheManager lib = get();
return lib.containsInternal(cacheName);
@@ -372,7 +335,7 @@ public static Path contains(String cacheName) throws IOException {
* @throws IOException
* thrown if no access to the file system could be obtained
*/
- private Path containsInternal(String cacheName) throws IOException {
+ private Path containsInternal(final String cacheName) throws IOException {
// Create a path object from the external name string
final Path p = new Path(this.libraryCachePath + "/" + cacheName);
@@ -396,7 +359,7 @@ private Path containsInternal(String cacheName) throws IOException {
* @throws IOException
* thrown if the library cache manager could not be instantiated
*/
- public static ClassLoader getClassLoader(JobID id) throws IOException {
+ public static ClassLoader getClassLoader(final JobID id) throws IOException {
if (id == null) {
return null;
@@ -416,7 +379,7 @@ public static ClassLoader getClassLoader(JobID id) throws IOException {
* @throws IOException
* thrown if the library cache manager could not be instantiated
*/
- private ClassLoader getClassLoaderInternal(JobID id) {
+ private ClassLoader getClassLoaderInternal(final JobID id) {
synchronized (this.libraryManagerEntries) {
if (!this.libraryManagerEntries.containsKey(id)) {
@@ -436,7 +399,7 @@ private ClassLoader getClassLoaderInternal(JobID id) {
* @throws IOException
* thrown if the library cache manager could not be instantiated
*/
- public static String[] getRequiredJarFiles(JobID id) throws IOException {
+ public static String[] getRequiredJarFiles(final JobID id) throws IOException {
if (id == null) {
return new String[0];
@@ -454,7 +417,7 @@ public static String[] getRequiredJarFiles(JobID id) throws IOException {
* the ID of the job to return the names of required libraries for.
* @return the names of the required libraries or null
if the specified job ID is unknown
*/
- private String[] getRequiredJarFilesInternal(JobID id) {
+ private String[] getRequiredJarFilesInternal(final JobID id) {
LibraryManagerEntry entry = null;
@@ -479,7 +442,7 @@ private String[] getRequiredJarFilesInternal(JobID id) {
* @throws IOException
* thrown if an error occurs while writing the data
*/
- public static void writeLibraryToStream(String libraryFileName, DataOutput out) throws IOException {
+ public static void writeLibraryToStream(final String libraryFileName, final DataOutput out) throws IOException {
final LibraryCacheManager lib = get();
lib.writeLibraryToStreamInternal(libraryFileName, out);
@@ -496,7 +459,7 @@ public static void writeLibraryToStream(String libraryFileName, DataOutput out)
* @throws IOException
* thrown if an error occurs while writing the data
*/
- private void writeLibraryToStreamInternal(String libraryFileName, DataOutput out) throws IOException {
+ private void writeLibraryToStreamInternal(final String libraryFileName, final DataOutput out) throws IOException {
if (libraryFileName == null) {
throw new IOException("libraryName is null!");
@@ -535,7 +498,7 @@ private void writeLibraryToStreamInternal(String libraryFileName, DataOutput out
* @throws IOException
* throws if an error occurs while reading from the stream
*/
- public static void readLibraryFromStream(DataInput in) throws IOException {
+ public static void readLibraryFromStream(final DataInput in) throws IOException {
final LibraryCacheManager lib = get();
lib.readLibraryFromStreamInternal(in);
@@ -550,7 +513,7 @@ public static void readLibraryFromStream(DataInput in) throws IOException {
* @throws IOException
* throws if an error occurs while reading from the stream
*/
- private void readLibraryFromStreamInternal(DataInput in) throws IOException {
+ private void readLibraryFromStreamInternal(final DataInput in) throws IOException {
final String libraryFileName = StringRecord.readString(in);
@@ -564,7 +527,7 @@ private void readLibraryFromStreamInternal(DataInput in) throws IOException {
throw new IOException("Submitted jar file " + libraryFileName + " is too large");
}
- final byte buf[] = new byte[(int) length];
+ final byte[] buf = new byte[(int) length];
in.readFully(buf);
final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName);
@@ -596,7 +559,8 @@ private void readLibraryFromStreamInternal(DataInput in) throws IOException {
* thrown if the library cache manager could not be instantiated or an error occurred while reading the
* library data from the input stream
*/
- public static void addLibrary(JobID jobID, Path name, long size, DataInput in) throws IOException {
+ public static void addLibrary(final JobID jobID, final Path name, final long size, final DataInput in)
+ throws IOException {
final LibraryCacheManager lib = get();
lib.addLibraryInternal(jobID, name, size, in);
@@ -617,14 +581,15 @@ public static void addLibrary(JobID jobID, Path name, long size, DataInput in) t
* @throws IOException
* thrown if an error occurred while reading the library data from the input stream
*/
- private void addLibraryInternal(JobID jobID, Path name, long size, DataInput in) throws IOException {
+ private void addLibraryInternal(final JobID jobID, final Path name, final long size, final DataInput in)
+ throws IOException {
if (size > (long) Integer.MAX_VALUE) {
throw new IOException("Submitted jar file " + name + " is too large");
}
// Map the entire jar file to memory
- final byte buf[] = new byte[(int) size];
+ final byte[] buf = new byte[(int) size];
in.readFully(buf);
// Reset and calculate message digest from jar file
@@ -685,7 +650,7 @@ private static class LibraryManagerEntry {
* an array with the names of required libraries by the corresponding job (URL objects required by the
* class loader)
*/
- public LibraryManagerEntry(JobID id, String[] requiredJarFiles, URL[] urls) {
+ public LibraryManagerEntry(final JobID id, final String[] requiredJarFiles, URL[] urls) {
String[] temp = requiredJarFiles;
if (temp == null) {
@@ -749,7 +714,7 @@ private static class LibraryTranslationKey {
* @param clientPath
* the client path
*/
- public LibraryTranslationKey(JobID jobID, Path clientPath) {
+ public LibraryTranslationKey(final JobID jobID, final Path clientPath) {
this.jobID = jobID;
this.clientPath = clientPath;
@@ -770,7 +735,7 @@ public int hashCode() {
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (obj == null) {
return false;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java
index 15c67b56cbe26..c90c19f6aa4c4 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java
@@ -51,7 +51,7 @@ public String[] getRequiredLibraries() {
* @param requiredLibraries
* the names of libraries whose cache status is to be retrieved
*/
- public void setRequiredLibraries(String[] requiredLibraries) {
+ public void setRequiredLibraries(final String[] requiredLibraries) {
this.requiredLibraries = requiredLibraries;
}
@@ -59,7 +59,7 @@ public void setRequiredLibraries(String[] requiredLibraries) {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
// Read required jar files
this.requiredLibraries = new String[in.readInt()];
@@ -73,7 +73,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
if (this.requiredLibraries == null) {
throw new IOException("requiredLibraries is null");
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java
index 386bbdb6b3144..73da0c6f635de 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java
@@ -48,7 +48,7 @@ public class LibraryCacheProfileResponse implements IOReadableWritable {
* @param request
* the library cache profile request the response belongs to
*/
- public LibraryCacheProfileResponse(LibraryCacheProfileRequest request) {
+ public LibraryCacheProfileResponse(final LibraryCacheProfileRequest request) {
this.requiredLibraries = request.getRequiredLibraries();
this.cached = new boolean[this.requiredLibraries.length];
@@ -69,7 +69,7 @@ public LibraryCacheProfileResponse() {
* @param cached
* true
if the library at the given position is in the local cache, false
otherwise
*/
- public void setCached(int pos, boolean cached) {
+ public void setCached(final int pos, final boolean cached) {
if (pos < this.cached.length) {
this.cached[pos] = cached;
@@ -84,7 +84,7 @@ public void setCached(int pos, boolean cached) {
* @return true
if the library at the given position is in the local cache, false
* otherwise
*/
- public boolean isCached(int pos) {
+ public boolean isCached(final int pos) {
if (pos < this.cached.length) {
return this.cached[pos];
@@ -97,7 +97,7 @@ public boolean isCached(int pos) {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
// Read the names of the required jar files
this.requiredLibraries = new String[in.readInt()];
@@ -117,7 +117,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
if (this.requiredLibraries == null) {
throw new IOException("requiredLibraries is null");
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java
index 8714871d0c509..9b0bde097ca9b 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java
@@ -39,7 +39,7 @@ public class LibraryCacheUpdate implements IOReadableWritable {
* @param libraryFileName
* the name of the library that should be transported within this object.
*/
- public LibraryCacheUpdate(String libraryFileName) {
+ public LibraryCacheUpdate(final String libraryFileName) {
this.libraryFileName = libraryFileName;
}
@@ -53,7 +53,7 @@ public LibraryCacheUpdate() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
LibraryCacheManager.readLibraryFromStream(in);
}
@@ -62,7 +62,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
if (this.libraryFileName == null) {
throw new IOException("libraryFileName is null");
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java
index f97964f8dc991..e80fcbef00360 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java
@@ -55,7 +55,7 @@ public class FileInputSplit implements InputSplit {
* List of hosts (hostnames) containing the block, possibly null
.
*/
private String[] hosts;
-
+
/**
* The logical number of the split.
*/
@@ -64,6 +64,8 @@ public class FileInputSplit implements InputSplit {
/**
* Constructs a split with host information.
*
+ * @param num
+ * the number of this input split
* @param file
* the file name
* @param start
@@ -73,7 +75,7 @@ public class FileInputSplit implements InputSplit {
* @param hosts
* the list of hosts containing the block, possibly null
*/
- public FileInputSplit(int num, Path file, long start, long length, String[] hosts) {
+ public FileInputSplit(final int num, final Path file, final long start, final long length, final String[] hosts) {
this.partitionNumber = num;
this.file = file;
this.start = start;
@@ -127,11 +129,12 @@ public String[] getHostNames() {
}
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
* @see eu.stratosphere.nephele.template.InputSplit#getPartitionNumber()
*/
@Override
- public int getPartitionNumber() {
+ public int getSplitNumber() {
return this.partitionNumber;
}
@@ -147,11 +150,10 @@ public String toString() {
* {@inheritDoc}
*/
@Override
- public void write(final DataOutput out) throws IOException
- {
+ public void write(final DataOutput out) throws IOException {
// write partition number
out.writeInt(this.partitionNumber);
-
+
// write file
if (this.file != null) {
out.writeBoolean(true);
@@ -163,7 +165,7 @@ public void write(final DataOutput out) throws IOException
// write start and length
out.writeLong(this.start);
out.writeLong(this.length);
-
+
// write hosts
if (this.hosts == null) {
out.writeBoolean(false);
@@ -180,11 +182,10 @@ public void write(final DataOutput out) throws IOException
* {@inheritDoc}
*/
@Override
- public void read(final DataInput in) throws IOException
- {
+ public void read(final DataInput in) throws IOException {
// read partition number
this.partitionNumber = in.readInt();
-
+
// read file path
boolean isNotNull = in.readBoolean();
if (isNotNull) {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java
index 6d3532f936bee..73ed1062d019a 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java
@@ -49,7 +49,7 @@ public abstract class FileSystem {
/**
* Object used to protect calls to specific methods.
*/
- private static final Object synchronizationObject = new Object();
+ private static final Object SYNCHRONIZATION_OBJECT = new Object();
/**
* An auxiliary class to identify a file system by its scheme
@@ -78,7 +78,7 @@ public static class FSKey {
* @param authority
* the authority of the file system
*/
- public FSKey(String scheme, String authority) {
+ public FSKey(final String scheme, final String authority) {
this.scheme = scheme;
this.authority = authority;
}
@@ -87,7 +87,7 @@ public FSKey(String scheme, String authority) {
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (obj instanceof FSKey) {
final FSKey key = (FSKey) obj;
@@ -187,7 +187,7 @@ public static FileSystem get(final URI uri) throws IOException {
FileSystem fs = null;
- synchronized (synchronizationObject) {
+ synchronized (SYNCHRONIZATION_OBJECT) {
if (uri.getScheme() == null) {
throw new IOException("FileSystem: Scheme is null");
@@ -313,7 +313,7 @@ public long getDefaultBlockSize() {
* @param f
* source file
*/
- public boolean exists(Path f) throws IOException {
+ public boolean exists(final Path f) throws IOException {
try {
return (getFileStatus(f) != null);
@@ -386,7 +386,7 @@ public abstract FSDataOutputStream create(Path f, boolean overwrite, int bufferS
* @return the number of block's thie file/directory consists of
* @throws IOException
*/
- public int getNumberOfBlocks(FileStatus file) throws IOException {
+ public int getNumberOfBlocks(final FileStatus file) throws IOException {
int numberOfBlocks = 0;
@@ -400,7 +400,7 @@ public int getNumberOfBlocks(FileStatus file) throws IOException {
}
// file is a directory
- FileStatus[] files = this.listStatus(file.getPath());
+ final FileStatus[] files = this.listStatus(file.getPath());
for (int i = 0; i < files.length; i++) {
if (!files[i].isDir()) {
@@ -411,7 +411,7 @@ public int getNumberOfBlocks(FileStatus file) throws IOException {
return numberOfBlocks;
}
- private int getNumberOfBlocks(long length, long blocksize) {
+ private int getNumberOfBlocks(final long length, final long blocksize) {
if (blocksize != 0) {
int numberOfBlocks;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java
index 897d8a31690ea..0723f6b6f7d7e 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java
@@ -31,7 +31,7 @@ public class LocalBlockLocation implements BlockLocation {
private final String[] hosts;
- public LocalBlockLocation(String host, long length) {
+ public LocalBlockLocation(final String host, final long length) {
this.hosts = new String[] { host };
this.length = length;
}
@@ -62,12 +62,11 @@ public long getOffset() {
return 0;
}
- /*
- * (non-Javadoc)
- * @see java.lang.Comparable#compareTo(java.lang.Object)
+ /**
+ * {@inheritDoc}
*/
@Override
- public int compareTo(BlockLocation o) {
+ public int compareTo(final BlockLocation o) {
return 0;
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java
index 191e5fbac57eb..caf152e5a5139 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java
@@ -47,8 +47,7 @@ public class LocalDataInputStream extends FSDataInputStream {
* @throws IOException
* thrown if the data input stream cannot be created
*/
- public LocalDataInputStream(File file)
- throws IOException {
+ public LocalDataInputStream(final File file) throws IOException {
this.fis = new FileInputStream(file);
this.position = 0;
@@ -58,7 +57,7 @@ public LocalDataInputStream(File file)
* {@inheritDoc}
*/
@Override
- public void seek(long desired) throws IOException {
+ public void seek(final long desired) throws IOException {
this.fis.getChannel().position(desired);
this.position = desired;
@@ -82,7 +81,7 @@ public int read() throws IOException {
* {@inheritDoc}
*/
@Override
- public int read(byte[] buffer, int offset, int length) throws IOException {
+ public int read(final byte[] buffer, final int offset, final int length) throws IOException {
final int value = this.fis.read(buffer, offset, length);
if (value > 0) {
@@ -113,7 +112,7 @@ public int available() throws IOException {
* {@inheritDoc}
*/
@Override
- public long skip(long n) throws IOException {
+ public long skip(final long n) throws IOException {
return this.fis.skip(n);
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java
index 655ab0d63f6cb..ed9eec460f51e 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java
@@ -47,8 +47,7 @@ public class LocalDataOutputStream extends FSDataOutputStream {
* @throws IOException
* thrown if the data output stream cannot be created
*/
- public LocalDataOutputStream(File file)
- throws IOException {
+ public LocalDataOutputStream(final File file) throws IOException {
this.fos = new FileOutputStream(file);
this.position = 0;
@@ -58,7 +57,7 @@ public LocalDataOutputStream(File file)
* {@inheritDoc}
*/
@Override
- public void write(int b) throws IOException {
+ public void write(final int b) throws IOException {
fos.write(b);
position++;
}
@@ -67,7 +66,7 @@ public void write(int b) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(byte b[], int off, int len) throws IOException {
+ public void write(final byte[] b, final int off, final int len) throws IOException {
fos.write(b, off, len);
position += len; // update position
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java
index d91884929ad94..79ee643c3ff7b 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java
@@ -47,7 +47,7 @@ public class LocalFileStatus implements FileStatus {
* @param fs
* the file system the corresponding file has been read from
*/
- public LocalFileStatus(File f, FileSystem fs) {
+ public LocalFileStatus(final File f, final FileSystem fs) {
this.file = f;
this.path = new Path(fs.getUri().getScheme() + ":" + f.getAbsolutePath());
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java
index 361c6d1110d9f..4984e1a981da6 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java
@@ -84,9 +84,10 @@ public LocalFileSystem() {
* {@inheritDoc}
*/
@Override
- public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {
+ public BlockLocation[] getFileBlockLocations(final FileStatus file, final long start, final long len)
+ throws IOException {
- BlockLocation[] blockLocations = new BlockLocation[1];
+ final BlockLocation[] blockLocations = new BlockLocation[1];
blockLocations[0] = new LocalBlockLocation(this.hostName, file.getLen());
return blockLocations;
@@ -129,7 +130,7 @@ public Path getWorkingDirectory() {
* {@inheritDoc}
*/
@Override
- public void initialize(URI name) throws IOException {
+ public void initialize(final URI name) throws IOException {
// TODO Auto-generated method stub
}
@@ -138,7 +139,7 @@ public void initialize(URI name) throws IOException {
* {@inheritDoc}
*/
@Override
- public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ public FSDataInputStream open(final Path f, final int bufferSize) throws IOException {
// TODO Auto-generated method stub
return null;
}
@@ -147,7 +148,7 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
* {@inheritDoc}
*/
@Override
- public FSDataInputStream open(Path f) throws IOException {
+ public FSDataInputStream open(final Path f) throws IOException {
final File file = pathToFile(f);
@@ -169,7 +170,7 @@ private File pathToFile(Path path) {
* {@inheritDoc}
*/
@Override
- public FileStatus[] listStatus(Path f) throws IOException {
+ public FileStatus[] listStatus(final Path f) throws IOException {
final File localf = pathToFile(f);
FileStatus[] results;
@@ -197,7 +198,7 @@ public FileStatus[] listStatus(Path f) throws IOException {
* {@inheritDoc}
*/
@Override
- public boolean delete(Path f, boolean recursive) throws IOException {
+ public boolean delete(final Path f, final boolean recursive) throws IOException {
final File file = pathToFile(f);
if (file.isFile()) {
@@ -218,13 +219,13 @@ public boolean delete(Path f, boolean recursive) throws IOException {
* @throws IOException
* thrown if an error occurred while deleting the files/directories
*/
- private boolean delete(File f) throws IOException {
+ private boolean delete(final File f) throws IOException {
if (f.isDirectory()) {
final File[] files = f.listFiles();
for (int i = 0; i < files.length; i++) {
- boolean del = delete(files[i]);
+ final boolean del = delete(files[i]);
if (del == false) {
return false;
}
@@ -246,7 +247,7 @@ private boolean delete(File f) throws IOException {
* @throws IOException
* thrown if an error occurred while creating the directory/directories
*/
- public boolean mkdirs(Path f) throws IOException {
+ public boolean mkdirs(final Path f) throws IOException {
final Path parent = f.getParent();
final File p2f = pathToFile(f);
@@ -257,8 +258,8 @@ public boolean mkdirs(Path f) throws IOException {
* {@inheritDoc}
*/
@Override
- public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize)
- throws IOException {
+ public FSDataOutputStream create(final Path f, final boolean overwrite, final int bufferSize,
+ final short replication, final long blockSize) throws IOException {
if (exists(f) && !overwrite) {
throw new IOException("File already exists:" + f);
@@ -277,9 +278,8 @@ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, shor
* {@inheritDoc}
*/
@Override
- public FSDataOutputStream create(Path f, boolean overwrite) throws IOException {
+ public FSDataOutputStream create(final Path f, final boolean overwrite) throws IOException {
return create(f, overwrite, 0, (short) 0, 0);
}
-
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java
index a83705dfdb54c..9095e2a2fa8d7 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java
@@ -42,7 +42,7 @@ public abstract class AbstractID implements IOReadableWritable {
/**
* Constructs a new ID with a specific bytes value.
*/
- public AbstractID(byte[] bytes) {
+ public AbstractID(final byte[] bytes) {
if(bytes.length == SIZE) {
System.arraycopy(bytes, 0, this.bytes, 0, SIZE);
@@ -65,16 +65,16 @@ public AbstractID() {
* @param src
* the bytes the ID consists of
*/
- private void setBytes(byte[] src) {
+ private void setBytes(final byte[] src) {
if (src == null) {
return;
}
- if(src.length != SIZE) {
+ if (src.length != SIZE) {
return;
}
-
+
System.arraycopy(src, 0, this.bytes, 0, SIZE);
}
@@ -93,7 +93,7 @@ private byte[] getBytes() {
* @param src
* the source ID
*/
- public void setID(AbstractID src) {
+ public void setID(final AbstractID src) {
setBytes(src.getBytes());
}
@@ -101,7 +101,7 @@ public void setID(AbstractID src) {
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (!(obj instanceof AbstractID)) {
return false;
@@ -163,7 +163,7 @@ public int hashCode() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
in.readFully(this.bytes);
}
@@ -172,7 +172,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
// Write the particular bytes
out.write(this.bytes);
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java
index b39e8241e356d..1efb01e678121 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java
@@ -27,7 +27,8 @@ public class BipartiteDistributionPattern implements DistributionPattern {
* {@inheritDoc}
*/
@Override
- public boolean createWire(int nodeLowerStage, int nodeUpperStage, int sizeSetLowerStage, int sizeSetUpperStage) {
+ public boolean createWire(final int nodeLowerStage, final int nodeUpperStage, final int sizeSetLowerStage,
+ final int sizeSetUpperStage) {
return true;
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java
index ecc63535d2084..7618befaad3e7 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java
@@ -15,7 +15,6 @@
package eu.stratosphere.nephele.io;
-import eu.stratosphere.nephele.io.channels.AbstractOutputChannel;
import eu.stratosphere.nephele.types.Record;
/**
@@ -39,5 +38,5 @@ public interface ChannelSelector extends IOReadableWritable {
* @return a (possibly empty) array of integer numbers which indicate the indices of the output channels through
* which the record shall be forwarded
*/
- int[] selectChannels(T record, int numberOfOutpuChannels);
+ int[] selectChannels(T record, int numberOfOutputChannels);
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java
index 79f81d3dbdbe7..504c87c89130e 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java
@@ -31,6 +31,9 @@
*/
public class DefaultChannelSelector implements ChannelSelector {
+ /**
+ * Stores the index of the channel to send the next record to.
+ */
private final int[] nextChannelToSendTo = new int[1];
/**
@@ -44,7 +47,7 @@ public DefaultChannelSelector() {
* {@inheritDoc}
*/
@Override
- public int[] selectChannels(T record, int numberOfOutpuChannels) {
+ public int[] selectChannels(final T record, final int numberOfOutpuChannels) {
this.nextChannelToSendTo[0] = (this.nextChannelToSendTo[0] + 1) % numberOfOutpuChannels;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java
index 4680f4b6de581..43fe817ae7f98 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java
@@ -117,4 +117,4 @@ record = recordClass.newInstance();
}
return record;
}
-}
\ No newline at end of file
+}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java
index 78abed20dd698..d80c21cb253e9 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java
@@ -30,7 +30,8 @@ public class PointwiseDistributionPattern implements DistributionPattern {
* {@inheritDoc}
*/
@Override
- public boolean createWire(int nodeLowerStage, int nodeUpperStage, int sizeSetLowerStage, int sizeSetUpperStage) {
+ public boolean createWire(final int nodeLowerStage, final int nodeUpperStage, final int sizeSetLowerStage,
+ final int sizeSetUpperStage) {
if (sizeSetLowerStage < sizeSetUpperStage) {
if (nodeLowerStage == (nodeUpperStage % sizeSetLowerStage)) {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java
index b1125b6f2c3cf..83507c6a640c3 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java
@@ -55,7 +55,7 @@ public class RecordBuffer {
* the maximum size of the buffer
*/
@SuppressWarnings("unchecked")
- public RecordBuffer(Class type, int maxSize) {
+ public RecordBuffer(final Class type, final int maxSize) {
this.buffer = (T[]) Array.newInstance(type, maxSize);
}
@@ -66,7 +66,7 @@ public RecordBuffer(Class type, int maxSize) {
* @param record
* the record to be added to the buffer.
*/
- public void put(T record) {
+ public void put(final T record) {
if (record == null) {
return;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java
index efdfeb9fb1eec..ad21366396fb3 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java
@@ -52,7 +52,7 @@ public class DeserializationBuffer {
/**
* Buffer to reconstruct the length field.
*/
- ByteBuffer lengthBuf = ByteBuffer.allocate(SIZEOFINT);
+ private ByteBuffer lengthBuf = ByteBuffer.allocate(SIZEOFINT);
/**
* Size of the record to be deserialized in bytes.
@@ -64,7 +64,7 @@ public class DeserializationBuffer {
/**
* Temporary buffer.
*/
- ByteBuffer tempBuffer = null;
+ private ByteBuffer tempBuffer = null;
/**
* Constructs a new deserialization buffer with the specified type.
@@ -72,10 +72,10 @@ public class DeserializationBuffer {
* @param type
* the type of the record the deserialization buffer can be used for
* @param propagateEndOfStream
- * true> if end of stream notifications during the
+ * true
if end of stream notifications during the
* deserialization process shall be propagated to the caller, false
otherwise
*/
- public DeserializationBuffer(RecordDeserializer deserializer, boolean propagateEndOfStream) {
+ public DeserializationBuffer(final RecordDeserializer deserializer, final boolean propagateEndOfStream) {
this.deserializer = deserializer;
this.propagateEndOfStream = propagateEndOfStream;
}
@@ -89,7 +89,7 @@ public DeserializationBuffer(RecordDeserializer deserializer, boolean propaga
* @throws IOException
* thrown if an error occurs while reading the data or deserializing the object
*/
- public T readData(T target, ReadableByteChannel readableByteChannel) throws IOException {
+ public T readData(final T target, final ReadableByteChannel readableByteChannel) throws IOException {
if (this.recordLength < 0) {
if (readableByteChannel.read(this.lengthBuf) == -1 && this.propagateEndOfStream) {
@@ -129,8 +129,8 @@ public T readData(T target, ReadableByteChannel readableByteChannel) throws IOEx
return null;
}
- deserializationBuffer.reset(tempBuffer.array(), this.recordLength);
- final T record = deserializer.deserialize(target, deserializationBuffer);
+ this.deserializationBuffer.reset(tempBuffer.array(), this.recordLength);
+ final T record = deserializer.deserialize(target, this.deserializationBuffer);
this.recordLength = -1;
this.lengthBuf.clear();
@@ -145,7 +145,7 @@ public T readData(T target, ReadableByteChannel readableByteChannel) throws IOEx
* the array of bytes used as input.
* @return the resulting integer
*/
- private int byteArrayToInt(byte[] arr) {
+ private int byteArrayToInt(final byte[] arr) {
int number = 0;
for (int i = 0; i < SIZEOFINT; ++i) {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java
index 8c785ef6f4eac..9a3b4b3df1d57 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java
@@ -47,10 +47,10 @@ public class SerializationBuffer {
* @param arr
* The byte buffer to store the data of the integer
*/
- private void integerToByteBuffer(int val, ByteBuffer byteBuffer) {
+ private void integerToByteBuffer(final int val, final ByteBuffer byteBuffer) {
for (int i = 0; i < SIZEOFINT; ++i) {
- int shift = i << (SIZEOFINT - 1); // i * 8
+ final int shift = i << (SIZEOFINT - 1); // i * 8
byteBuffer.put(SIZEOFINT - 1 - i, (byte) ((val & (0xff << shift)) >>> shift));
}
@@ -68,8 +68,9 @@ private void integerToByteBuffer(int val, ByteBuffer byteBuffer) {
*/
public boolean dataLeftFromPreviousSerialization() {
- if (leftInSerializationBuffer() > 0)
+ if (leftInSerializationBuffer() > 0) {
return true;
+ }
return false;
}
@@ -84,7 +85,7 @@ public boolean dataLeftFromPreviousSerialization() {
* @throws IOException
* thrown if an error occurs while writing to serialized data to the channel
*/
- public int read(WritableByteChannel writableByteChannel) throws IOException {
+ public int read(final WritableByteChannel writableByteChannel) throws IOException {
int bytesReadFromLengthBuf = 0;
@@ -126,17 +127,18 @@ private int leftInSerializationBuffer() {
* Thrown if data from a previous serialization process is still in the internal buffer and has not yet been
* transfered to a byte buffer
*/
- public void serialize(T record) throws IOException {
+ public void serialize(final T record) throws IOException {
// Check if there is data left in the buffer
- if (dataLeftFromPreviousSerialization())
+ if (dataLeftFromPreviousSerialization()) {
throw new IOException("Cannot write new data, " + leftInSerializationBuffer()
+ " bytes still left from previous call");
+ }
record.write(this.serializationBuffer); // serializationBuffer grows dynamically
// Now record is completely in serializationBuffer;
- integerToByteBuffer(serializationBuffer.getLength(), this.lengthBuf);
+ integerToByteBuffer(this.serializationBuffer.getLength(), this.lengthBuf);
}
public void clear() {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobInputVertex.java
new file mode 100644
index 0000000000000..4fe00f3d782ef
--- /dev/null
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobInputVertex.java
@@ -0,0 +1,40 @@
+/***********************************************************************************************************************
+ *
+ * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ **********************************************************************************************************************/
+
+package eu.stratosphere.nephele.jobgraph;
+
+/**
+ * An abstract base class for input vertices in Nephele.
+ *
+ * @author warneke
+ */
+public abstract class AbstractJobInputVertex extends AbstractJobVertex {
+
+ /**
+ * Constructs a new job input vertex with the given name.
+ *
+ * @param name
+ * the name of the new job input vertex
+ * @param id
+ * the ID of this vertex
+ * @param jobGraph
+ * the job graph this vertex belongs to
+ */
+ protected AbstractJobInputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
+ super(name, id, jobGraph);
+
+ jobGraph.addVertex(this);
+ }
+}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobOutputVertex.java
new file mode 100644
index 0000000000000..4718c026c7dd8
--- /dev/null
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobOutputVertex.java
@@ -0,0 +1,40 @@
+/***********************************************************************************************************************
+ *
+ * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ **********************************************************************************************************************/
+
+package eu.stratosphere.nephele.jobgraph;
+
+/**
+ * An abstract base class for output vertices in Nephele.
+ *
+ * @author warneke
+ */
+public abstract class AbstractJobOutputVertex extends AbstractJobVertex {
+
+ /**
+ * Constructs a new job output vertex with the given name.
+ *
+ * @param name
+ * the name of the new job output vertex
+ * @param id
+ * the ID of this vertex
+ * @param jobGraph
+ * the job graph this vertex belongs to
+ */
+ protected AbstractJobOutputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
+ super(name, id, jobGraph);
+
+ jobGraph.addVertex(this);
+ }
+}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java
index 50788c9fecd01..67d87e332c627 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java
@@ -21,6 +21,7 @@
import java.util.ArrayList;
import eu.stratosphere.nephele.configuration.Configuration;
+import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager;
import eu.stratosphere.nephele.io.IOReadableWritable;
import eu.stratosphere.nephele.io.channels.ChannelType;
import eu.stratosphere.nephele.io.compression.CompressionLevel;
@@ -87,6 +88,11 @@ public abstract class AbstractJobVertex implements IOReadableWritable {
*/
private final Configuration configuration = new Configuration();
+ /**
+ * The class of the invokable.
+ */
+ protected Class extends AbstractInvokable> invokableClass = null;
+
/**
* Constructs a new job vertex and assigns it with the given name.
*
@@ -97,7 +103,7 @@ public abstract class AbstractJobVertex implements IOReadableWritable {
* @param jobGraph
* the job graph this vertex belongs to
*/
- protected AbstractJobVertex(String name, JobVertexID id, JobGraph jobGraph) {
+ protected AbstractJobVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
this.name = name;
this.id = (id == null) ? new JobVertexID() : id;
@@ -112,7 +118,7 @@ protected AbstractJobVertex(String name, JobVertexID id, JobGraph jobGraph) {
* @throws JobGraphDefinitionException
* thrown if the given vertex cannot be connected to vertex
in the requested manner
*/
- public void connectTo(AbstractJobVertex vertex) throws JobGraphDefinitionException {
+ public void connectTo(final AbstractJobVertex vertex) throws JobGraphDefinitionException {
this.connectTo(vertex, null, null, -1, -1);
}
@@ -130,7 +136,7 @@ public void connectTo(AbstractJobVertex vertex) throws JobGraphDefinitionExcepti
* @throws JobGraphDefinitionException
* thrown if the given vertex cannot be connected to vertex
in the requested manner
*/
- public void connectTo(AbstractJobVertex vertex, int indexOfOutputGate, int indexOfInputGate)
+ public void connectTo(final AbstractJobVertex vertex, final int indexOfOutputGate, final int indexOfInputGate)
throws JobGraphDefinitionException {
this.connectTo(vertex, null, null, indexOfOutputGate, indexOfInputGate);
}
@@ -147,8 +153,8 @@ public void connectTo(AbstractJobVertex vertex, int indexOfOutputGate, int index
* @throws JobGraphDefinitionException
* thrown if the given vertex cannot be connected to vertex
in the requested manner
*/
- public void connectTo(AbstractJobVertex vertex, ChannelType channelType, CompressionLevel compressionLevel)
- throws JobGraphDefinitionException {
+ public void connectTo(final AbstractJobVertex vertex, final ChannelType channelType,
+ final CompressionLevel compressionLevel) throws JobGraphDefinitionException {
this.connectTo(vertex, channelType, compressionLevel, -1, -1);
}
@@ -170,8 +176,9 @@ public void connectTo(AbstractJobVertex vertex, ChannelType channelType, Compres
* @throws JobGraphDefinitionException
* thrown if the given vertex cannot be connected to vertex
in the requested manner
*/
- public void connectTo(AbstractJobVertex vertex, ChannelType channelType, CompressionLevel compressionLevel,
- int indexOfOutputGate, int indexOfInputGate) throws JobGraphDefinitionException {
+ public void connectTo(final AbstractJobVertex vertex, final ChannelType channelType,
+ final CompressionLevel compressionLevel, int indexOfOutputGate, int indexOfInputGate)
+ throws JobGraphDefinitionException {
if (vertex == null) {
throw new JobGraphDefinitionException("Target vertex is null!");
@@ -253,8 +260,8 @@ protected int getFirstFreeInputGateIndex() {
* @param indexOfInputGate
* index of the consuming task's input gate to be used
*/
- private void connectBacklink(AbstractJobVertex vertex, ChannelType channelType, CompressionLevel compressionLevel,
- int indexOfOutputGate, int indexOfInputGate) {
+ private void connectBacklink(final AbstractJobVertex vertex, final ChannelType channelType,
+ final CompressionLevel compressionLevel, final int indexOfOutputGate, final int indexOfInputGate) {
// Make sure the array is big enough
for (int i = this.backwardEdges.size(); i <= indexOfInputGate; i++) {
@@ -298,7 +305,7 @@ public int getNumberOfBackwardConnections() {
* the index of the edge
* @return the forward edge or null
if no edge exists at the specified index.
*/
- public JobEdge getForwardConnection(int index) {
+ public JobEdge getForwardConnection(final int index) {
if (index < this.forwardEdges.size()) {
return this.forwardEdges.get(index);
@@ -314,7 +321,7 @@ public JobEdge getForwardConnection(int index) {
* the index of the edge
* @return the backward edge or null
if no edge exists at the specified index
*/
- public JobEdge getBackwardConnection(int index) {
+ public JobEdge getBackwardConnection(final int index) {
if (index < this.backwardEdges.size()) {
return this.backwardEdges.get(index);
@@ -361,8 +368,9 @@ public JobVertexID getID() {
/**
* {@inheritDoc}
*/
+ @SuppressWarnings("unchecked")
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
if (jobGraph == null) {
throw new IOException("jobGraph is null, cannot deserialize");
@@ -418,13 +426,36 @@ public void read(DataInput in) throws IOException {
this.forwardEdges.add(null);
}
}
+
+ // Read the invokable class
+ final boolean isNotNull = in.readBoolean();
+ if (!isNotNull) {
+ return;
+ }
+
+ // Read the name of the class and try to instantiate the class object
+
+ final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID());
+ if (cl == null) {
+ throw new IOException("Cannot find class loader for vertex " + getID());
+ }
+
+ // Read the name of the expected class
+ final String className = StringRecord.readString(in);
+
+ try {
+ this.invokableClass = (Class extends AbstractInvokable>) Class.forName(className, true, cl);
+ } catch (ClassNotFoundException cnfe) {
+ throw new IOException("Class " + className + " not found in one of the supplied jar files: "
+ + StringUtils.stringifyException(cnfe));
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
// Instance type
StringRecord.writeString(out, this.instanceType);
@@ -463,6 +494,17 @@ public void write(DataOutput out) throws IOException {
out.writeInt(edge.getIndexOfInputGate());
}
}
+
+ // Write the invokable class
+ if (this.invokableClass == null) {
+ out.writeBoolean(false);
+ return;
+ }
+
+ out.writeBoolean(true);
+
+ // Write out the name of the class
+ StringRecord.writeString(out, this.invokableClass.getName());
}
/**
@@ -480,7 +522,7 @@ public JobGraph getJobGraph() {
* @param numberOfSubtasks
* the number of subtasks this vertex represents should be split into at runtime
*/
- public void setNumberOfSubtasks(int numberOfSubtasks) {
+ public void setNumberOfSubtasks(final int numberOfSubtasks) {
this.numberOfSubtasks = numberOfSubtasks;
}
@@ -500,7 +542,7 @@ public int getNumberOfSubtasks() {
* @param instanceType
* the instance type the task this vertex represents should run on
*/
- public void setInstanceType(String instanceType) {
+ public void setInstanceType(final String instanceType) {
this.instanceType = instanceType;
}
@@ -519,7 +561,7 @@ public String getInstanceType() {
* @param numberOfSubtasksPerInstance
* the number of subtasks that should be assigned to the same instance
*/
- public void setNumberOfSubtasksPerInstance(int numberOfSubtasksPerInstance) {
+ public void setNumberOfSubtasksPerInstance(final int numberOfSubtasksPerInstance) {
this.numberOfSubtasksPerInstance = numberOfSubtasksPerInstance;
}
@@ -538,7 +580,7 @@ public int getNumberOfSubtasksPerInstance() {
* @param vertex
* the vertex this vertex should share its instances with at runtime
*/
- public void setVertexToShareInstancesWith(AbstractJobVertex vertex) {
+ public void setVertexToShareInstancesWith(final AbstractJobVertex vertex) {
this.vertexToShareInstancesWith = vertex;
}
@@ -569,7 +611,23 @@ public Configuration getConfiguration() {
* @throws IllegalConfigurationException
* thrown if the respective tasks is not configured properly
*/
- public abstract void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException;
+ public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException {
+
+ if (invokable == null) {
+ throw new IllegalArgumentException("Argument invokable is null");
+ }
+
+ // see if the task itself has a valid configuration
+ // because this is user code running on the master, we embed it in a catch-all block
+ try {
+ invokable.checkConfiguration();
+ } catch (IllegalConfigurationException icex) {
+ throw icex; // simply forward
+ } catch (Throwable t) {
+ throw new IllegalConfigurationException("Checking the invokable's configuration caused an error: "
+ + StringUtils.stringifyException(t));
+ }
+ }
/**
* Returns the minimum number of subtasks the respective task
@@ -579,7 +637,14 @@ public Configuration getConfiguration() {
* an instance of the task this vertex represents
* @return the minimum number of subtasks the respective task must be split into at runtime
*/
- public abstract int getMinimumNumberOfSubtasks(AbstractInvokable invokable);
+ public int getMinimumNumberOfSubtasks(final AbstractInvokable invokable) {
+
+ if (invokable == null) {
+ throw new IllegalArgumentException("Argument invokable is null");
+ }
+
+ return invokable.getMinimumNumberOfSubtasks();
+ }
/**
* Returns the maximum number of subtasks the respective task
@@ -590,12 +655,22 @@ public Configuration getConfiguration() {
* @return the maximum number of subtasks the respective task can be split into at runtime, -1
for
* infinity
*/
- public abstract int getMaximumNumberOfSubtasks(AbstractInvokable invokable);
+ public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) {
+
+ if (invokable == null) {
+ throw new IllegalArgumentException("Argument invokable is null");
+ }
+
+ return invokable.getMaximumNumberOfSubtasks();
+ }
/**
* Returns the invokable class which represents the task of this vertex
*
* @return the invokable class, null
if it is not set
*/
- public abstract Class extends AbstractInvokable> getInvokableClass();
+ public Class extends AbstractInvokable> getInvokableClass() {
+
+ return this.invokableClass;
+ }
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java
index 4ef6ebdd08e54..83adde429febc 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java
@@ -58,8 +58,8 @@ public class JobEdge {
* @param indexOfInputGate
* index of the consuming task's input gate that this edge connects to
*/
- public JobEdge(AbstractJobVertex connectedVertex, ChannelType channelType, CompressionLevel compressionLevel,
- int indexOfInputGate) {
+ public JobEdge(final AbstractJobVertex connectedVertex, final ChannelType channelType,
+ final CompressionLevel compressionLevel, final int indexOfInputGate) {
this.connectedVertex = connectedVertex;
this.channelType = channelType;
this.compressionLevel = compressionLevel;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java
index 321523cea2aee..545ee03153700 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java
@@ -23,20 +23,19 @@
import eu.stratosphere.nephele.fs.FileSystem;
import eu.stratosphere.nephele.fs.Path;
import eu.stratosphere.nephele.template.AbstractFileInputTask;
-import eu.stratosphere.nephele.template.AbstractInputTask;
import eu.stratosphere.nephele.template.AbstractInvokable;
import eu.stratosphere.nephele.template.IllegalConfigurationException;
import eu.stratosphere.nephele.util.StringUtils;
/**
- * A JobFileInputVertex is a specific subtype of a {@link JobInputVertex} and is designed
+ * A JobFileInputVertex is a specific subtype of a {@link AbstractJobInputVertex} and is designed
* for Nephele tasks which read data from a local or distributed file system. As every job input vertex
* A JobFileInputVertex must not have any further input.
*
* @author warneke
*/
-public class JobFileInputVertex extends JobGenericInputVertex
-{
+public final class JobFileInputVertex extends AbstractJobInputVertex {
+
/**
* The path pointing to the input file/directory.
*/
@@ -52,7 +51,7 @@ public class JobFileInputVertex extends JobGenericInputVertex
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobFileInputVertex(String name, JobVertexID id, JobGraph jobGraph) {
+ public JobFileInputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
super(name, id, jobGraph);
}
@@ -64,7 +63,7 @@ public JobFileInputVertex(String name, JobVertexID id, JobGraph jobGraph) {
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobFileInputVertex(String name, JobGraph jobGraph) {
+ public JobFileInputVertex(final String name, final JobGraph jobGraph) {
super(name, null, jobGraph);
}
@@ -74,7 +73,7 @@ public JobFileInputVertex(String name, JobGraph jobGraph) {
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobFileInputVertex(JobGraph jobGraph) {
+ public JobFileInputVertex(final JobGraph jobGraph) {
super(null, null, jobGraph);
}
@@ -84,7 +83,7 @@ public JobFileInputVertex(JobGraph jobGraph) {
* @param path
* the path of the file the job file input vertex's task should read from
*/
- public void setFilePath(Path path) {
+ public void setFilePath(final Path path) {
this.path = path;
}
@@ -104,8 +103,8 @@ public Path getFilePath() {
* @param inputClass
* the class of the vertex's input task.
*/
- public void setFileInputClass(Class extends AbstractFileInputTask> inputClass) {
- this.inputClass = (Class extends AbstractInputTask>>) inputClass;
+ public void setFileInputClass(final Class extends AbstractFileInputTask> inputClass) {
+ this.invokableClass = inputClass;
}
/**
@@ -115,19 +114,18 @@ public void setFileInputClass(Class extends AbstractFileInputTask> inputClass)
*/
@SuppressWarnings("unchecked")
public Class extends AbstractFileInputTask> getFileInputClass() {
- return (Class extends AbstractFileInputTask>) this.inputClass;
+ return (Class extends AbstractFileInputTask>) this.invokableClass;
}
/**
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException
- {
+ public void read(final DataInput in) throws IOException {
super.read(in);
// Read path of the input file
- boolean isNotNull = in.readBoolean();
+ final boolean isNotNull = in.readBoolean();
if (isNotNull) {
this.path = new Path();
this.path.read(in);
@@ -138,8 +136,7 @@ public void read(DataInput in) throws IOException
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException
- {
+ public void write(final DataOutput out) throws IOException {
super.write(out);
// Write out the path of the input file
@@ -156,7 +153,7 @@ public void write(DataOutput out) throws IOException
* {@inheritDoc}
*/
@Override
- public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException {
+ public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException {
// Check if the user has specified a path
if (this.path == null) {
@@ -170,14 +167,14 @@ public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfig
if (f == null) {
throw new IOException(path.toString() + " led to a null object");
}
- }
- catch (IOException e) {
+ } catch (IOException e) {
throw new IllegalConfigurationException("Cannot access file or directory: "
+ StringUtils.stringifyException(e));
}
-
+
// register the path in the configuration
- invokable.getRuntimeConfiguration().setString(AbstractFileInputTask.INPUT_PATH_CONFIG_KEY, this.path.toString());
+ invokable.getRuntimeConfiguration()
+ .setString(AbstractFileInputTask.INPUT_PATH_CONFIG_KEY, this.path.toString());
// Finally, see if the task itself has a valid configuration
super.checkConfiguration(invokable);
@@ -187,7 +184,7 @@ public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfig
* {@inheritDoc}
*/
@Override
- public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) {
+ public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) {
int numberOfBlocks = -1;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java
index 2c275317564d9..150190add0484 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java
@@ -28,14 +28,14 @@
import eu.stratosphere.nephele.template.IllegalConfigurationException;
/**
- * A JobFileOutputVertex is a specific subtype of a {@link JobOutputVertex} and is designed
+ * A JobFileOutputVertex is a specific subtype of a {@link AbstractJobOutputVertex} and is designed
* for Nephele tasks which write data to a local or distributed file system. As every job output vertex
* A JobFileOutputVertex must not have any further output.
*
* @author warneke
*/
-public class JobFileOutputVertex extends JobGenericOutputVertex
-{
+public class JobFileOutputVertex extends AbstractJobOutputVertex {
+
/**
* The path pointing to the output file/directory.
*/
@@ -51,7 +51,7 @@ public class JobFileOutputVertex extends JobGenericOutputVertex
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobFileOutputVertex(String name, JobVertexID id, JobGraph jobGraph) {
+ public JobFileOutputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
super(name, id, jobGraph);
}
@@ -63,7 +63,7 @@ public JobFileOutputVertex(String name, JobVertexID id, JobGraph jobGraph) {
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobFileOutputVertex(String name, JobGraph jobGraph) {
+ public JobFileOutputVertex(final String name, final JobGraph jobGraph) {
super(name, null, jobGraph);
}
@@ -73,7 +73,7 @@ public JobFileOutputVertex(String name, JobGraph jobGraph) {
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobFileOutputVertex(JobGraph jobGraph) {
+ public JobFileOutputVertex(final JobGraph jobGraph) {
super(null, null, jobGraph);
}
@@ -83,7 +83,7 @@ public JobFileOutputVertex(JobGraph jobGraph) {
* @param path
* the path of the file the job file input vertex's task should write to
*/
- public void setFilePath(Path path) {
+ public void setFilePath(final Path path) {
this.path = path;
}
@@ -104,8 +104,8 @@ public Path getFilePath() {
* @param outputClass
* the class of the vertex's output task.
*/
- public void setFileOutputClass(Class extends AbstractFileOutputTask> outputClass) {
- this.outputClass = outputClass;
+ public void setFileOutputClass(final Class extends AbstractFileOutputTask> outputClass) {
+ this.invokableClass = outputClass;
}
/**
@@ -115,17 +115,16 @@ public void setFileOutputClass(Class extends AbstractFileOutputTask> outputCla
*/
@SuppressWarnings("unchecked")
public Class extends AbstractFileOutputTask> getFileOutputClass() {
- return (Class extends AbstractFileOutputTask>) this.outputClass;
+ return (Class extends AbstractFileOutputTask>) this.invokableClass;
}
/**
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException
- {
+ public void read(final DataInput in) throws IOException {
super.read(in);
-
+
// Read path of the input file
boolean isNotNull = in.readBoolean();
if (isNotNull) {
@@ -138,8 +137,7 @@ public void read(DataInput in) throws IOException
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException
- {
+ public void write(final DataOutput out) throws IOException {
super.write(out);
// Write out the path of the input file
@@ -155,7 +153,7 @@ public void write(DataOutput out) throws IOException
* {@inheritDoc}
*/
@Override
- public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException {
+ public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException {
// Check if the user has specified a path
if (this.path == null) {
@@ -169,8 +167,8 @@ public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfig
* {@inheritDoc}
*/
@Override
- public int getMaximumNumberOfSubtasks(AbstractInvokable invokable)
- {
+ public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) {
+
if (this.path == null) {
return 0;
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericInputVertex.java
deleted file mode 100644
index e39d90d8a2f39..0000000000000
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericInputVertex.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/***********************************************************************************************************************
- *
- * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- *
- **********************************************************************************************************************/
-
-package eu.stratosphere.nephele.jobgraph;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager;
-import eu.stratosphere.nephele.template.AbstractInputTask;
-import eu.stratosphere.nephele.template.AbstractInvokable;
-import eu.stratosphere.nephele.template.IllegalConfigurationException;
-import eu.stratosphere.nephele.types.StringRecord;
-import eu.stratosphere.nephele.util.StringUtils;
-
-public class JobGenericInputVertex extends JobInputVertex
-{
- /**
- * Class of input task.
- */
- protected Class extends AbstractInputTask>> inputClass = null;
-
- /**
- * Creates a new job input vertex with the specified name.
- *
- * @param name The name of the new job file input vertex.
- * @param id The ID of this vertex.
- * @param jobGraph The job graph this vertex belongs to.
- */
- public JobGenericInputVertex(String name, JobVertexID id, JobGraph jobGraph) {
- super(name, id, jobGraph);
- }
-
- /**
- * Creates a new job file input vertex with the specified name.
- *
- * @param name The name of the new job file input vertex.
- * @param jobGraph The job graph this vertex belongs to.
- */
- public JobGenericInputVertex(String name, JobGraph jobGraph) {
- super(name, null, jobGraph);
- }
-
- /**
- * Creates a new job file input vertex.
- *
- * @param jobGraph The job graph this vertex belongs to.
- */
- public JobGenericInputVertex(JobGraph jobGraph) {
- super(null, null, jobGraph);
- }
-
- /**
- * Sets the class of the vertex's input task.
- *
- * @param inputClass The class of the vertex's input task.
- */
- public void setInputClass(Class extends AbstractInputTask>> inputClass) {
- this.inputClass = inputClass;
- }
-
- /**
- * Returns the class of the vertex's input task.
- *
- * @return the class of the vertex's input task or null
if no task has yet been set
- */
- public Class extends AbstractInputTask>> getInputClass() {
- return this.inputClass;
- }
-
- /**
- * {@inheritDoc}
- */
- @SuppressWarnings("unchecked")
- @Override
- public void read(DataInput in) throws IOException
- {
- super.read(in);
-
- // Read class
- boolean isNotNull = in.readBoolean();
- if (isNotNull) {
- // Read the name of the class and try to instantiate the class object
- final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID());
- if (cl == null) {
- throw new IOException("Cannot find class loader for vertex " + getID());
- }
-
- // Read the name of the expected class
- final String className = StringRecord.readString(in);
-
- try {
- this.inputClass = (Class extends AbstractInputTask>>) Class.forName(className, true, cl).asSubclass(AbstractInputTask.class);
- }
- catch (ClassNotFoundException cnfe) {
- throw new IOException("Class " + className + " not found in one of the supplied jar files: "
- + StringUtils.stringifyException(cnfe));
- }
- catch (ClassCastException ccex) {
- throw new IOException("Class " + className + " is not a subclass of "
- + AbstractInputTask.class.getName() + ": " + StringUtils.stringifyException(ccex));
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void write(DataOutput out) throws IOException
- {
- super.write(out);
-
- // Write out the name of the class
- if (this.inputClass == null) {
- out.writeBoolean(false);
- } else {
- out.writeBoolean(true);
- StringRecord.writeString(out, this.inputClass.getName());
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException
- {
- // see if the task itself has a valid configuration
- // because this is user code running on the master, we embed it in a catch-all block
- try {
- invokable.checkConfiguration();
- }
- catch (IllegalConfigurationException icex) {
- throw icex; // simply forward
- }
- catch (Throwable t) {
- throw new IllegalConfigurationException("Checking the invokable's configuration caused an error: "
- + StringUtils.stringifyException(t));
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Class extends AbstractInvokable> getInvokableClass() {
-
- return this.inputClass;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public int getMaximumNumberOfSubtasks(AbstractInvokable invokable)
- {
- return invokable.getMaximumNumberOfSubtasks();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public int getMinimumNumberOfSubtasks(AbstractInvokable invokable) {
-
- return invokable.getMinimumNumberOfSubtasks();
- }
-}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericOutputVertex.java
deleted file mode 100644
index 6edbaba78bb80..0000000000000
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericOutputVertex.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/***********************************************************************************************************************
- *
- * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- *
- **********************************************************************************************************************/
-
-package eu.stratosphere.nephele.jobgraph;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager;
-import eu.stratosphere.nephele.template.AbstractInvokable;
-import eu.stratosphere.nephele.template.AbstractOutputTask;
-import eu.stratosphere.nephele.template.IllegalConfigurationException;
-import eu.stratosphere.nephele.types.StringRecord;
-import eu.stratosphere.nephele.util.StringUtils;
-
-/**
- * A JobGenericOutputVertex is a specific subtype of a {@link JobOutputVertex} and is designed
- * for Nephele tasks which sink data in a not further specified way. As every job output vertex,
- * a JobGenericOutputVertex must not have any further output.
- *
- * @author warneke
- */
-public class JobGenericOutputVertex extends JobOutputVertex {
-
- /**
- * The class of the output task.
- */
- protected Class extends AbstractOutputTask> outputClass = null;
-
-
- /**
- * Creates a new job file output vertex with the specified name.
- *
- * @param name
- * the name of the new job file output vertex
- * @param id
- * the ID of this vertex
- * @param jobGraph
- * the job graph this vertex belongs to
- */
- public JobGenericOutputVertex(String name, JobVertexID id, JobGraph jobGraph) {
- super(name, id, jobGraph);
- }
-
- /**
- * Creates a new job file output vertex with the specified name.
- *
- * @param name
- * the name of the new job file output vertex
- * @param jobGraph
- * the job graph this vertex belongs to
- */
- public JobGenericOutputVertex(String name, JobGraph jobGraph) {
- super(name, null, jobGraph);
- }
-
- /**
- * Creates a new job file input vertex.
- *
- * @param jobGraph
- * the job graph this vertex belongs to
- */
- public JobGenericOutputVertex(JobGraph jobGraph) {
- super(null, null, jobGraph);
- }
-
- /**
- * Sets the class of the vertex's output task.
- *
- * @param outputClass The class of the vertex's output task.
- */
- public void setOutputClass(Class extends AbstractOutputTask> outputClass) {
- this.outputClass = outputClass;
- }
-
- /**
- * Returns the class of the vertex's output task.
- *
- * @return The class of the vertex's output task or null
if no task has yet been set.
- */
- public Class extends AbstractOutputTask> getOutputClass() {
- return this.outputClass;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void read(DataInput in) throws IOException {
- super.read(in);
-
- // Read class
- boolean isNotNull = in.readBoolean();
- if (isNotNull) {
-
- // Read the name of the class and try to instantiate the class object
- final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID());
- if (cl == null) {
- throw new IOException("Cannot find class loader for vertex " + getID());
- }
-
- // Read the name of the expected class
- final String className = StringRecord.readString(in);
-
- try {
- this.outputClass = Class.forName(className, true, cl).asSubclass(AbstractOutputTask.class);
- }
- catch (ClassNotFoundException cnfe) {
- throw new IOException("Class " + className + " not found in one of the supplied jar files: "
- + StringUtils.stringifyException(cnfe));
- }
- catch (ClassCastException ccex) {
- throw new IOException("Class " + className + " is not a subclass of "
- + AbstractOutputTask.class.getName() + ": " + StringUtils.stringifyException(ccex));
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
-
- // Write out the name of the class
- if (this.outputClass == null) {
- out.writeBoolean(false);
- }
- else {
- out.writeBoolean(true);
- StringRecord.writeString(out, this.outputClass.getName());
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException
- {
- // see if the task itself has a valid configuration
- // because this is user code running on the master, we embed it in a catch-all block
- try {
- invokable.checkConfiguration();
- }
- catch (IllegalConfigurationException icex) {
- throw icex; // simply forward
- }
- catch (Throwable t) {
- throw new IllegalConfigurationException("Checking the invokable's configuration caused an error: "
- + StringUtils.stringifyException(t));
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Class extends AbstractInvokable> getInvokableClass() {
-
- return this.outputClass;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public int getMaximumNumberOfSubtasks(AbstractInvokable invokable)
- {
- // Delegate call to invokable
- return invokable.getMaximumNumberOfSubtasks();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public int getMinimumNumberOfSubtasks(AbstractInvokable invokable)
- {
- // Delegate call to invokable
- return invokable.getMinimumNumberOfSubtasks();
- }
-}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java
index 889e6b8df7ed3..702197450c4c0 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java
@@ -52,12 +52,12 @@ public class JobGraph implements IOReadableWritable {
/**
* List of input vertices included in this job graph.
*/
- private Map inputVertices = new HashMap();
+ private Map inputVertices = new HashMap();
/**
* List of output vertices included in this job graph.
*/
- private Map outputVertices = new HashMap();
+ private Map outputVertices = new HashMap();
/**
* List of task vertices included in this job graph.
@@ -107,7 +107,7 @@ public JobGraph() {
* @param jobName
* the name for this job graph
*/
- public JobGraph(String jobName) {
+ public JobGraph(final String jobName) {
this();
this.jobName = jobName;
}
@@ -148,7 +148,7 @@ public Configuration getTaskmanagerConfiguration() {
* @param inputVertex
* the new input vertex to be added
*/
- public void addVertex(JobInputVertex inputVertex) {
+ public void addVertex(final AbstractJobInputVertex inputVertex) {
if (!inputVertices.containsKey(inputVertex.getID())) {
inputVertices.put(inputVertex.getID(), inputVertex);
@@ -161,7 +161,7 @@ public void addVertex(JobInputVertex inputVertex) {
* @param taskVertex
* the new task vertex to be added
*/
- public void addVertex(JobTaskVertex taskVertex) {
+ public void addVertex(final JobTaskVertex taskVertex) {
if (!taskVertices.containsKey(taskVertex.getID())) {
taskVertices.put(taskVertex.getID(), taskVertex);
@@ -174,7 +174,7 @@ public void addVertex(JobTaskVertex taskVertex) {
* @param outputVertex
* the new output vertex to be added
*/
- public void addVertex(JobOutputVertex outputVertex) {
+ public void addVertex(final AbstractJobOutputVertex outputVertex) {
if (!outputVertices.containsKey(outputVertex.getID())) {
outputVertices.put(outputVertex.getID(), outputVertex);
@@ -213,9 +213,9 @@ public int getNumberOfTaskVertices() {
*
* @return an iterator to iterate all input vertices registered with the job graph
*/
- public Iterator getInputVertices() {
+ public Iterator getInputVertices() {
- final Collection coll = this.inputVertices.values();
+ final Collection coll = this.inputVertices.values();
return coll.iterator();
}
@@ -225,9 +225,9 @@ public Iterator getInputVertices() {
*
* @return an iterator to iterate all output vertices registered with the job graph
*/
- public Iterator getOutputVertices() {
+ public Iterator getOutputVertices() {
- final Collection coll = this.outputVertices.values();
+ final Collection coll = this.outputVertices.values();
return coll.iterator();
}
@@ -278,12 +278,12 @@ public AbstractJobVertex[] getAllJobVertices() {
final AbstractJobVertex[] vertices = new AbstractJobVertex[inputVertices.size() + outputVertices.size()
+ taskVertices.size()];
- final Iterator iv = getInputVertices();
+ final Iterator iv = getInputVertices();
while (iv.hasNext()) {
vertices[i++] = iv.next();
}
- final Iterator ov = getOutputVertices();
+ final Iterator ov = getOutputVertices();
while (ov.hasNext()) {
vertices[i++] = ov.next();
}
@@ -304,10 +304,10 @@ public AbstractJobVertex[] getAllJobVertices() {
* @param collector
* a temporary list to store the vertices that have already been visisted
*/
- private void collectVertices(AbstractJobVertex jv, List collector) {
+ private void collectVertices(final AbstractJobVertex jv, final List collector) {
if (jv == null) {
- final Iterator iter = getInputVertices();
+ final Iterator iter = getInputVertices();
while (iter.hasNext()) {
collectVertices(iter.next(), collector);
}
@@ -341,18 +341,18 @@ public JobID getJobID() {
* the ID of the vertex to search for
* @return the vertex with the matching ID or null
if no vertex with such ID could be found
*/
- public AbstractJobVertex findVertexByID(JobVertexID id) {
+ public AbstractJobVertex findVertexByID(final JobVertexID id) {
- if (inputVertices.containsKey(id)) {
- return inputVertices.get(id);
+ if (this.inputVertices.containsKey(id)) {
+ return this.inputVertices.get(id);
}
- if (outputVertices.containsKey(id)) {
- return outputVertices.get(id);
+ if (this.outputVertices.containsKey(id)) {
+ return this.outputVertices.get(id);
}
- if (taskVertices.containsKey(id)) {
- return taskVertices.get(id);
+ if (this.taskVertices.containsKey(id)) {
+ return this.taskVertices.get(id);
}
return null;
@@ -366,17 +366,17 @@ public AbstractJobVertex findVertexByID(JobVertexID id) {
* @return true
if a vertex with the given ID is registered with the job graph, false
* otherwise.
*/
- private boolean includedInJobGraph(JobVertexID id) {
+ private boolean includedInJobGraph(final JobVertexID id) {
- if (inputVertices.containsKey(id)) {
+ if (this.inputVertices.containsKey(id)) {
return true;
}
- if (outputVertices.containsKey(id)) {
+ if (this.outputVertices.containsKey(id)) {
return true;
}
- if (taskVertices.containsKey(id)) {
+ if (this.taskVertices.containsKey(id)) {
return true;
}
@@ -449,8 +449,9 @@ public boolean isAcyclic() {
* Auxiliary method implementing Tarjan's algorithm for strongly-connected components to determine whether the job
* graph is acyclic.
*/
- private boolean tarjan(AbstractJobVertex jv, Integer index, HashMap indexMap,
- HashMap lowLinkMap, Stack stack) {
+ private boolean tarjan(final AbstractJobVertex jv, Integer index,
+ final HashMap indexMap, final HashMap lowLinkMap,
+ final Stack stack) {
indexMap.put(jv, Integer.valueOf(index));
lowLinkMap.put(jv, Integer.valueOf(index));
@@ -501,7 +502,7 @@ private boolean tarjan(AbstractJobVertex jv, Integer index, HashMap iter = getInputVertices();
+ final Iterator iter = getInputVertices();
while (iter.hasNext()) {
final AbstractJobVertex jv = iter.next();
@@ -523,7 +524,7 @@ public AbstractJobVertex areVertexDegreesCorrect() {
}
// Check output vertices
- final Iterator iter3 = getOutputVertices();
+ final Iterator iter3 = getOutputVertices();
while (iter3.hasNext()) {
final AbstractJobVertex jv = iter3.next();
@@ -540,7 +541,7 @@ public AbstractJobVertex areVertexDegreesCorrect() {
* {@inheritDoc}
*/
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
// Read job id
this.jobID.read(in);
@@ -625,7 +626,7 @@ public void read(DataInput in) throws IOException {
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
// Write job ID
this.jobID.write(out);
@@ -671,7 +672,7 @@ public void write(DataOutput out) throws IOException {
* @throws IOException
* thrown if an error occurs while writing to the stream
*/
- private void writeRequiredJarFiles(DataOutput out, AbstractJobVertex[] jobVertices) throws IOException {
+ private void writeRequiredJarFiles(final DataOutput out, final AbstractJobVertex[] jobVertices) throws IOException {
// Now check if all the collected jar files really exist
final FileSystem fs = FileSystem.getLocalFileSystem();
@@ -716,7 +717,7 @@ private void writeRequiredJarFiles(DataOutput out, AbstractJobVertex[] jobVertic
* @throws IOException
* thrown if an error occurs while reading the stream
*/
- private void readRequiredJarFiles(DataInput in) throws IOException {
+ private void readRequiredJarFiles(final DataInput in) throws IOException {
// Do jar files follow;
final int numJars = in.readInt();
@@ -748,7 +749,7 @@ private void readRequiredJarFiles(DataInput in) throws IOException {
* @param jar
* path of the JAR file required to run the job on a task manager
*/
- public void addJar(Path jar) {
+ public void addJar(final Path jar) {
if (jar == null) {
return;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java
index df1377557adb6..1a17fe66d43fb 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java
@@ -23,7 +23,7 @@
*
* @author warneke
*/
-public class JobID extends AbstractID {
+public final class JobID extends AbstractID {
/**
* Constructs a new random ID from a uniform distribution.
*/
@@ -33,8 +33,11 @@ public JobID() {
/**
* Constructs a new ID with a specific bytes value.
+ *
+ * @param bytes
+ * the ID in byte representation
*/
- public JobID(byte[] bytes) {
+ public JobID(final byte[] bytes) {
super(bytes);
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java
index f7066877b83f9..a610eb6ef1e4b 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java
@@ -15,46 +15,63 @@
package eu.stratosphere.nephele.jobgraph;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
+import eu.stratosphere.nephele.template.AbstractInputTask;
-/**
- * An abstract base class for input vertices in Nephele.
- *
- * @author warneke
- */
-public abstract class JobInputVertex extends AbstractJobVertex {
+public class JobInputVertex extends AbstractJobInputVertex {
/**
- * Constructs a new job input vertex with the given name.
+ * Creates a new job input vertex with the specified name.
*
* @param name
- * the name of the new job input vertex
+ * The name of the new job file input vertex.
* @param id
- * the ID of this vertex
+ * The ID of this vertex.
* @param jobGraph
- * the job graph this vertex belongs to
+ * The job graph this vertex belongs to.
*/
- protected JobInputVertex(String name, JobVertexID id, JobGraph jobGraph) {
+ public JobInputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
super(name, id, jobGraph);
+ }
+
+ /**
+ * Creates a new job file input vertex with the specified name.
+ *
+ * @param name
+ * The name of the new job file input vertex.
+ * @param jobGraph
+ * The job graph this vertex belongs to.
+ */
+ public JobInputVertex(final String name, final JobGraph jobGraph) {
+ super(name, null, jobGraph);
+ }
- jobGraph.addVertex(this);
+ /**
+ * Creates a new job file input vertex.
+ *
+ * @param jobGraph
+ * The job graph this vertex belongs to.
+ */
+ public JobInputVertex(final JobGraph jobGraph) {
+ super(null, null, jobGraph);
}
/**
- * {@inheritDoc}
+ * Sets the class of the vertex's input task.
+ *
+ * @param inputClass
+ * The class of the vertex's input task.
*/
- @Override
- public void read(DataInput in) throws IOException {
- super.read(in);
+ public void setInputClass(final Class extends AbstractInputTask>> inputClass) {
+ this.invokableClass = inputClass;
}
/**
- * {@inheritDoc}
+ * Returns the class of the vertex's input task.
+ *
+ * @return the class of the vertex's input task or null
if no task has yet been set
*/
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
+ @SuppressWarnings("unchecked")
+ public Class extends AbstractInputTask>> getInputClass() {
+ return (Class extends AbstractInputTask>>) this.invokableClass;
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java
index b97375351013c..1519ff466bbfb 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java
@@ -15,26 +15,70 @@
package eu.stratosphere.nephele.jobgraph;
+import eu.stratosphere.nephele.template.AbstractOutputTask;
+
/**
- * An abstract base class for output vertices in Nephele.
+ * A JobOutputVertex is a specific subtype of a {@link AbstractJobOutputVertex} and is designed
+ * for Nephele tasks which sink data in a not further specified way. As every job output vertex,
+ * a JobOutputVertex must not have any further output.
*
* @author warneke
*/
-public abstract class JobOutputVertex extends AbstractJobVertex {
+public class JobOutputVertex extends AbstractJobOutputVertex {
/**
- * Constructs a new job output vertex with the given name.
+ * Creates a new job file output vertex with the specified name.
*
* @param name
- * the name of the new job output vertex
+ * the name of the new job file output vertex
* @param id
* the ID of this vertex
* @param jobGraph
* the job graph this vertex belongs to
*/
- protected JobOutputVertex(String name, JobVertexID id, JobGraph jobGraph) {
+ public JobOutputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
super(name, id, jobGraph);
+ }
- jobGraph.addVertex(this);
+ /**
+ * Creates a new job file output vertex with the specified name.
+ *
+ * @param name
+ * the name of the new job file output vertex
+ * @param jobGraph
+ * the job graph this vertex belongs to
+ */
+ public JobOutputVertex(final String name, final JobGraph jobGraph) {
+ super(name, null, jobGraph);
+ }
+
+ /**
+ * Creates a new job file input vertex.
+ *
+ * @param jobGraph
+ * the job graph this vertex belongs to
+ */
+ public JobOutputVertex(final JobGraph jobGraph) {
+ super(null, null, jobGraph);
+ }
+
+ /**
+ * Sets the class of the vertex's output task.
+ *
+ * @param outputClass
+ * The class of the vertex's output task.
+ */
+ public void setOutputClass(final Class extends AbstractOutputTask> outputClass) {
+ this.invokableClass = outputClass;
+ }
+
+ /**
+ * Returns the class of the vertex's output task.
+ *
+ * @return The class of the vertex's output task or null
if no task has yet been set.
+ */
+ @SuppressWarnings("unchecked")
+ public Class extends AbstractOutputTask> getOutputClass() {
+ return (Class extends AbstractOutputTask>) this.invokableClass;
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java
index 6a46f757181ae..2e5fc70e7f40d 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java
@@ -31,7 +31,7 @@ public enum JobStatus {
CREATED,
/**
- * All tasks of the job have been accepted by the scheduler, resources have been requested
+ * All tasks of the job have been accepted by the scheduler, resources have been requested.
*/
SCHEDULED,
@@ -55,4 +55,4 @@ public enum JobStatus {
* All of the job's tasks have successfully finished.
*/
FINISHED
-};
\ No newline at end of file
+};
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java
index 437f2cf979760..ea777e2465854 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java
@@ -15,16 +15,8 @@
package eu.stratosphere.nephele.jobgraph;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager;
import eu.stratosphere.nephele.template.AbstractInvokable;
import eu.stratosphere.nephele.template.AbstractTask;
-import eu.stratosphere.nephele.template.IllegalConfigurationException;
-import eu.stratosphere.nephele.types.StringRecord;
-import eu.stratosphere.nephele.util.StringUtils;
/**
* A JobTaskVertex is the vertex type for regular tasks (with both input and output) in Nephele.
@@ -34,11 +26,6 @@
*/
public class JobTaskVertex extends AbstractJobVertex {
- /**
- * The task attached to this vertex.
- */
- private Class extends AbstractTask> taskClass = null;
-
/**
* Creates a new job task vertex with the specified name.
*
@@ -49,7 +36,7 @@ public class JobTaskVertex extends AbstractJobVertex {
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobTaskVertex(String name, JobVertexID id, JobGraph jobGraph) {
+ public JobTaskVertex(final String name, final JobVertexID id, final JobGraph jobGraph) {
super(name, id, jobGraph);
jobGraph.addVertex(this);
@@ -63,7 +50,7 @@ public JobTaskVertex(String name, JobVertexID id, JobGraph jobGraph) {
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobTaskVertex(String name, JobGraph jobGraph) {
+ public JobTaskVertex(final String name, final JobGraph jobGraph) {
super(name, null, jobGraph);
jobGraph.addVertex(this);
@@ -75,7 +62,7 @@ public JobTaskVertex(String name, JobGraph jobGraph) {
* @param jobGraph
* the job graph this vertex belongs to
*/
- public JobTaskVertex(JobGraph jobGraph) {
+ public JobTaskVertex(final JobGraph jobGraph) {
super(null, null, jobGraph);
jobGraph.addVertex(this);
@@ -87,8 +74,8 @@ public JobTaskVertex(JobGraph jobGraph) {
* @param taskClass
* the class of the vertex's task
*/
- public void setTaskClass(Class extends AbstractTask> taskClass) {
- this.taskClass = taskClass;
+ public void setTaskClass(final Class extends AbstractTask> taskClass) {
+ this.invokableClass = taskClass;
}
/**
@@ -96,75 +83,16 @@ public void setTaskClass(Class extends AbstractTask> taskClass) {
*
* @return the class of the vertex's task or null
if the class has not yet been set
*/
- public Class extends AbstractTask> getTaskClass() {
- return this.taskClass;
- }
-
- /**
- * {@inheritDoc}
- */
@SuppressWarnings("unchecked")
- @Override
- public void read(DataInput in) throws IOException {
- super.read(in);
-
- final boolean isNotNull = in.readBoolean();
- if (!isNotNull) {
- return;
- }
-
- // Read the name of the class and try to instantiate the class object
-
- final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID());
- if (cl == null) {
- throw new IOException("Cannot find class loader for vertex " + getID());
- }
-
- // Read the name of the expected class
- final String className = StringRecord.readString(in);
-
- try {
- this.taskClass = (Class extends AbstractTask>) Class.forName(className, true, cl);
- } catch (ClassNotFoundException cnfe) {
- throw new IOException("Class " + className + " not found in one of the supplied jar files: "
- + StringUtils.stringifyException(cnfe));
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
-
- if (this.taskClass == null) {
- out.writeBoolean(false);
- return;
- }
-
- out.writeBoolean(true);
-
- // Write out the name of the class
- StringRecord.writeString(out, this.taskClass.getName());
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException {
-
- // Delegate check to invokable
- invokable.checkConfiguration();
+ public Class extends AbstractTask> getTaskClass() {
+ return (Class extends AbstractTask>) this.invokableClass;
}
/**
* {@inheritDoc}
*/
@Override
- public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) {
+ public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) {
// Delegate call to invokable
return invokable.getMaximumNumberOfSubtasks();
@@ -174,18 +102,9 @@ public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) {
* {@inheritDoc}
*/
@Override
- public int getMinimumNumberOfSubtasks(AbstractInvokable invokable) {
+ public int getMinimumNumberOfSubtasks(final AbstractInvokable invokable) {
// Delegate call to invokable
return invokable.getMinimumNumberOfSubtasks();
}
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Class extends AbstractInvokable> getInvokableClass() {
-
- return this.taskClass;
- }
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java
index 6df7323caf154..4c74139b7c67f 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java
@@ -16,21 +16,22 @@
package eu.stratosphere.nephele.services;
public class ServiceException extends Exception {
+
private static final long serialVersionUID = 8751768275700167972L;
protected ServiceException() {
super();
}
-
- public ServiceException(String message) {
+
+ public ServiceException(final String message) {
super(message);
}
- public ServiceException(Throwable cause) {
+ public ServiceException(final Throwable cause) {
super(cause);
}
-
- public ServiceException(String message, Throwable cause) {
+
+ public ServiceException(final String message, final Throwable cause) {
super(message, cause);
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java
index c843d41489578..ae5d107ebc453 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java
@@ -22,16 +22,16 @@ public class EventDispatcher {
public final Set> handlers;
public EventDispatcher() {
- handlers = new HashSet>();
+ this.handlers = new HashSet>();
}
- public void register(EventHandler handler) {
+ public void register(final EventHandler handler) {
}
- public void unregister(EventHandler handler) {
- handlers.remove(handler);
+ public void unregister(final EventHandler handler) {
+ this.handlers.remove(handler);
}
- protected void dispatch(T event) {
+ protected void dispatch(final T event) {
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java
index 506e600287de2..a97f32f6f0bf1 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java
@@ -16,5 +16,5 @@
package eu.stratosphere.nephele.services.event;
public interface EventHandler {
- public void handleEvent(U event);
+ void handleEvent(U event);
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java
index b9b0f36f944fa..39a637f87152a 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java
@@ -26,8 +26,7 @@
*
* @author Alexander Alexandrov
*/
-public final class Channel
-{
+public final class Channel {
private static final int RANDOM_BYTES_LENGTH = 16;
/**
@@ -35,15 +34,14 @@ public final class Channel
*
* @author Alexander Alexandrov
*/
- public static class ID
- {
+ public static class ID {
private final String path;
- protected ID(String path) {
+ protected ID(final String path) {
this.path = path;
}
- protected ID(String basePath, Random random) {
+ protected ID(final String basePath, final Random random) {
this.path = basePath + File.separator + randomString(random) + ".channel";
}
@@ -61,21 +59,20 @@ public String toString() {
}
}
- public static final class Enumerator
- {
- private static final String format = "%s.%06d.channel";
+ public static final class Enumerator {
+ private static final String FORMAT = "%s.%06d.channel";
private final String pathPrefix;
private int counter;
- protected Enumerator(String basePath, Random random) {
+ protected Enumerator(final String basePath, final Random random) {
this.pathPrefix = basePath + File.separator + randomString(random);
this.counter = 0;
}
public ID next() {
- return new ID(String.format(format, pathPrefix, ++counter));
+ return new ID(String.format(FORMAT, pathPrefix, ++counter));
}
}
@@ -86,9 +83,8 @@ public ID next() {
* The random number generator to be used.
* @return A hex representation of the generated byte sequence
*/
- private static final String randomString(Random random)
- {
- byte[] bytes = new byte[RANDOM_BYTES_LENGTH];
+ private static final String randomString(final Random random) {
+ final byte[] bytes = new byte[RANDOM_BYTES_LENGTH];
random.nextBytes(bytes);
return StringUtils.byteToHexString(bytes);
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java
index 5a42815d23878..af3df12a2a392 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java
@@ -35,6 +35,7 @@
* @author warneke
*/
public abstract class AbstractFileInputTask extends AbstractInputTask {
+
public static final String INPUT_PATH_CONFIG_KEY = "input.path";
/**
@@ -59,8 +60,8 @@ public Iterator getFileInputSplits() {
* {@inheritDoc}
*/
@Override
- public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException {
-
+ public FileInputSplit[] computeInputSplits(final int minNumSplits) throws IOException {
+
final String pathURI = getRuntimeConfiguration().getString(INPUT_PATH_CONFIG_KEY, null);
if (pathURI == null) {
throw new IOException("The path to the file was not found in the runtime configuration.");
@@ -76,7 +77,7 @@ public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException
final List inputSplits = new ArrayList();
// get all the files that are involved in the splits
- List files = new ArrayList();
+ final List files = new ArrayList();
long totalLength = 0;
final FileSystem fs = path.getFileSystem();
@@ -128,7 +129,7 @@ public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException
// get the block containing the majority of the data
blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex);
// create a new split
- FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, splitSize,
+ final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, splitSize,
blocks[blockIndex]
.getHosts());
inputSplits.add(fis);
@@ -175,7 +176,9 @@ public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException
* The earliest index to look at.
* @return The index of the block containing the given position.
*/
- private final int getBlockIndexForPosition(BlockLocation[] blocks, long offset, long halfSplitSize, int startIndex) {
+ private final int getBlockIndexForPosition(final BlockLocation[] blocks, final long offset,
+ final long halfSplitSize, final int startIndex) {
+
// go over all indexes after the startIndex
for (int i = startIndex; i < blocks.length; i++) {
long blockStart = blocks[i].getOffset();
@@ -193,13 +196,13 @@ private final int getBlockIndexForPosition(BlockLocation[] blocks, long offset,
}
throw new IllegalArgumentException("The given offset is not contained in the any block.");
}
-
+
/**
* {@inheritDoc}
*/
@Override
public Class getInputSplitType() {
-
+
return FileInputSplit.class;
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputTask.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractGenericInputTask.java
similarity index 93%
rename from nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputTask.java
rename to nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractGenericInputTask.java
index 290806f41a516..2c588b6e93091 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputTask.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractGenericInputTask.java
@@ -18,7 +18,7 @@
/**
* An input task that processes generic input splits (partitions).
*/
-public abstract class GenericInputTask extends AbstractInputTask {
+public abstract class AbstractGenericInputTask extends AbstractInputTask {
/**
* {@inheritDoc}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java
index 9800ee0249f57..9dd8b34254c6b 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java
@@ -49,34 +49,38 @@ public abstract class AbstractInputTask extends AbstractIn
* @return the type of input splits that is generated by this input task
*/
public abstract Class getInputSplitType();
-
+
/**
* Returns an iterator to a (possible empty) list of input splits which is expected to be consumed by this
* instance of the {@link AbstractInputTask}.
*
* @return an iterator to a (possible empty) list of input splits.
*/
- public Iterator getInputSplits()
- {
+ public Iterator getInputSplits() {
+
final InputSplitProvider provider = getEnvironment().getInputSplitProvider();
+
return new Iterator() {
private T nextSplit;
-
+
@Override
public boolean hasNext() {
+
if (this.nextSplit == null) {
- InputSplit split = provider.getNextInputSplit();
- if (split != null)
- {
+
+ final InputSplit split = provider.getNextInputSplit();
+ if (split != null) {
@SuppressWarnings("unchecked")
- T tSplit = (T) split;
+ final T tSplit = (T) split;
this.nextSplit = tSplit;
return true;
+ } else {
+ return false;
}
- else return false;
+ } else {
+ return true;
}
- else return true;
}
@Override
@@ -84,16 +88,16 @@ public T next() {
if (this.nextSplit == null && !hasNext()) {
throw new NoSuchElementException();
}
-
- T tmp = this.nextSplit;
+
+ final T tmp = this.nextSplit;
this.nextSplit = null;
return tmp;
}
@Override
public void remove() {
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException();
}
};
}
-}
\ No newline at end of file
+}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java
index 9ea1e3d4187cd..3797a26bc473d 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java
@@ -19,67 +19,68 @@
import java.io.DataOutput;
import java.io.IOException;
-
/**
* A generic input split that has only a partition number.
*/
-public class GenericInputSplit implements InputSplit
-{
- protected int partitionNumber;
-
+public class GenericInputSplit implements InputSplit {
+
+ /**
+ * The number of this split.
+ */
+ protected int number;
+
// --------------------------------------------------------------------------------------------
-
+
/**
* Default constructor for instantiation during de-serialization.
*/
- public GenericInputSplit()
- {}
-
+ public GenericInputSplit() {
+ }
+
/**
- * Creates a generic input split with the given partition number.
+ * Creates a generic input split with the given split number.
*
- * @param partitionNumber The partition number of the split.
+ * @param number
+ * the number of the split
*/
- public GenericInputSplit(int partitionNumber)
- {
- this.partitionNumber = partitionNumber;
+ public GenericInputSplit(final int number) {
+ this.number = number;
}
-
-
- // --------------------------------------------------------------------------------------------
-
- /* (non-Javadoc)
+
+ // --------------------------------------------------------------------------------------------
+
+ /*
+ * (non-Javadoc)
* @see eu.stratosphere.nephele.io.IOReadableWritable#write(java.io.DataOutput)
*/
@Override
- public void write(DataOutput out) throws IOException
- {
- out.writeInt(this.partitionNumber);
+ public void write(final DataOutput out) throws IOException {
+ out.writeInt(this.number);
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
* @see eu.stratosphere.nephele.io.IOReadableWritable#read(java.io.DataInput)
*/
@Override
- public void read(DataInput in) throws IOException
- {
- this.partitionNumber = in.readInt();
+ public void read(final DataInput in) throws IOException {
+ this.number = in.readInt();
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
* @see eu.stratosphere.nephele.template.InputSplit#getPartitionNumber()
*/
@Override
- public int getPartitionNumber()
- {
- return this.partitionNumber;
+ public int getSplitNumber() {
+ return this.number;
}
- /* (non-Javadoc)
+ /*
+ * (non-Javadoc)
* @see java.lang.Object#toString()
*/
- public String toString()
- {
- return "[" + this.partitionNumber + "]";
+ public String toString() {
+ return "[" + this.number + "]";
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java
index e3a01f48d41fd..2e86f4b052c8c 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java
@@ -36,18 +36,20 @@ public class IllegalConfigurationException extends Exception {
* @param errorMsg
* the error message to be included in the exception
*/
- public IllegalConfigurationException(String errorMsg) {
+ public IllegalConfigurationException(final String errorMsg) {
super(errorMsg);
}
-
+
/**
* Constructs an new illegal configuration exception with the given error message
* and a given cause.
*
- * @param errorMsg The error message to be included in the exception.
- * @param cause The exception that caused this exception.
+ * @param errorMsg
+ * The error message to be included in the exception.
+ * @param cause
+ * The exception that caused this exception.
*/
- public IllegalConfigurationException(String errorMsg, Throwable cause) {
+ public IllegalConfigurationException(final String errorMsg, final Throwable cause) {
super(errorMsg, cause);
}
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java
index ed3463a3351a6..70fb7a19d6b56 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java
@@ -22,12 +22,11 @@
*
* @author warneke
*/
-public interface InputSplit extends IOReadableWritable
-{
+public interface InputSplit extends IOReadableWritable {
/**
- * Gets the partition number of this input split.
- *
- * @return The number of this input split.
+ * Returns the number of this input split.
+ *
+ * @return the number of this input split
*/
- public int getPartitionNumber();
+ int getSplitNumber();
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java
index 5e727bfb550dd..2003ff3d5bf42 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java
@@ -33,12 +33,12 @@ public FileRecord() {
fileName = "empty";
}
- public FileRecord(String fileName) {
+ public FileRecord(final String fileName) {
this.bytes = EMPTY_BYTES;
this.fileName = fileName;
}
- public void setFileName(String fileName) {
+ public void setFileName(final String fileName) {
this.fileName = fileName;
}
@@ -60,13 +60,14 @@ public byte[] getDataBuffer() {
* @param len
* the number of bytes to append
*/
- public void append(byte[] data, int start, int len) {
+ public void append(final byte[] data, final int start, final int len) {
final int oldLength = this.bytes.length;
setCapacity(this.bytes.length + len, true);
System.arraycopy(data, start, this.bytes, oldLength, len);
}
- private void setCapacity(int len, boolean keepData) {
+ private void setCapacity(final int len, final boolean keepData) {
+
if (this.bytes == null || this.bytes.length < len) {
final byte[] newBytes = new byte[len];
if (this.bytes != null && keepData) {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java
index 78237a4120649..bf5cbfa61f31f 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java
@@ -37,7 +37,7 @@ public class IntegerRecord implements Record {
* @param value
* the integer value this record should wrap up
*/
- public IntegerRecord(int value) {
+ public IntegerRecord(final int value) {
this.value = value;
}
@@ -63,7 +63,7 @@ public int getValue() {
* @param value
* the new value for this integer record
*/
- public void setValue(int value) {
+ public void setValue(final int value) {
this.value = value;
}
@@ -91,7 +91,7 @@ public void write(final DataOutput out) throws IOException {
* {@inheritDoc}
*/
@Override
- public boolean equals(Object obj) {
+ public boolean equals(final Object obj) {
if (!(obj instanceof IntegerRecord)) {
return false;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java
index 0703d4c861e95..05ce666c19bc4 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java
@@ -48,14 +48,14 @@
*/
public class StringRecord implements Record {
- private static ThreadLocal ENCODER_FACTORY = new ThreadLocal() {
+ private static final ThreadLocal ENCODER_FACTORY = new ThreadLocal() {
protected CharsetEncoder initialValue() {
return Charset.forName("UTF-8").newEncoder().onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT);
}
};
- private static ThreadLocal DECODER_FACTORY = new ThreadLocal() {
+ private static final ThreadLocal DECODER_FACTORY = new ThreadLocal() {
protected CharsetDecoder initialValue() {
return Charset.forName("UTF-8").newDecoder().onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT);
@@ -75,19 +75,19 @@ public StringRecord() {
/**
* Construct from a string.
*/
- public StringRecord(String string) {
+ public StringRecord(final String string) {
set(string);
}
/** Construct from another text. */
- public StringRecord(StringRecord utf8) {
+ public StringRecord(final StringRecord utf8) {
set(utf8);
}
/**
* Construct from a byte array.
*/
- public StringRecord(byte[] utf8) {
+ public StringRecord(final byte[] utf8) {
set(utf8);
}
@@ -119,7 +119,7 @@ public int charAt(final int position) {
return bytesToCodePoint(bb.slice());
}
- public int find(String what) {
+ public int find(final String what) {
return find(what, 0);
}
@@ -175,7 +175,7 @@ public int find(final String what, final int start) {
/**
* Set to contain the contents of a string.
*/
- public void set(String string) {
+ public void set(final String string) {
try {
final ByteBuffer bb = encode(string, true);
this.bytes = bb.array();
@@ -188,12 +188,12 @@ public void set(String string) {
/**
* Set to a utf8 byte array
*/
- public void set(byte[] utf8) {
+ public void set(final byte[] utf8) {
set(utf8, 0, utf8.length);
}
/** copy a text. */
- public void set(StringRecord other) {
+ public void set(final StringRecord other) {
set(other.getBytes(), 0, other.getLength());
}
@@ -212,7 +212,7 @@ public int getLength() {
* @param len
* the number of bytes of the new string
*/
- public void set(byte[] utf8, int start, int len) {
+ public void set(final byte[] utf8, final int start, final int len) {
setCapacity(len, false);
System.arraycopy(utf8, start, bytes, 0, len);
this.length = len;
@@ -228,7 +228,7 @@ public void set(byte[] utf8, int start, int len) {
* @param len
* the number of bytes to append
*/
- public void append(byte[] utf8, int start, int len) {
+ public void append(final byte[] utf8, final int start, final int len) {
setCapacity(length + len, true);
System.arraycopy(utf8, start, bytes, length, len);
this.length += len;
@@ -250,7 +250,7 @@ public void clear() {
* @param len the number of bytes we need
* @param keepData should the old data be kept
*/
- private void setCapacity(int len, boolean keepData) {
+ private void setCapacity(final int len, final boolean keepData) {
if (this.bytes == null || this.bytes.length < len) {
final byte[] newBytes = new byte[len];
if (this.bytes != null && keepData) {
@@ -276,7 +276,7 @@ public String toString() {
/**
* deserialize
*/
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
final int newLength = in.readInt();
setCapacity(newLength, false);
in.readFully(this.bytes, 0, newLength);
@@ -284,12 +284,12 @@ public void read(DataInput in) throws IOException {
}
/** Skips over one Text in the input. */
- public static void skip(DataInput in) throws IOException {
+ public static void skip(final DataInput in) throws IOException {
final int length = in.readInt();
skipFully(in, length);
}
- public static void skipFully(DataInput in, int len) throws IOException {
+ public static void skipFully(final DataInput in, final int len) throws IOException {
int total = 0;
int cur = 0;
@@ -307,7 +307,7 @@ public static void skipFully(DataInput in, int len) throws IOException {
*
* @see Writable#read(DataOutput)
*/
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
out.writeInt(this.length);
out.write(this.bytes, 0, this.length);
}
@@ -357,11 +357,11 @@ public int hashCode() {
* Converts the provided byte array to a String using the UTF-8 encoding. If
* the input is malformed, replace by a default value.
*/
- public static String decode(byte[] utf8) throws CharacterCodingException {
+ public static String decode(final byte[] utf8) throws CharacterCodingException {
return decode(ByteBuffer.wrap(utf8), true);
}
- public static String decode(byte[] utf8, int start, int length) throws CharacterCodingException {
+ public static String decode(final byte[] utf8, final int start, final int length) throws CharacterCodingException {
return decode(ByteBuffer.wrap(utf8, start, length), true);
}
@@ -371,11 +371,12 @@ public static String decode(byte[] utf8, int start, int length) throws Character
* substitution character, which is U+FFFD. Otherwise the method throws a
* MalformedInputException.
*/
- public static String decode(byte[] utf8, int start, int length, boolean replace) throws CharacterCodingException {
+ public static String decode(final byte[] utf8, final int start, final int length, final boolean replace)
+ throws CharacterCodingException {
return decode(ByteBuffer.wrap(utf8, start, length), replace);
}
- private static String decode(ByteBuffer utf8, boolean replace) throws CharacterCodingException {
+ private static String decode(final ByteBuffer utf8, final boolean replace) throws CharacterCodingException {
final CharsetDecoder decoder = DECODER_FACTORY.get();
if (replace) {
decoder.onMalformedInput(java.nio.charset.CodingErrorAction.REPLACE);
@@ -398,7 +399,7 @@ private static String decode(ByteBuffer utf8, boolean replace) throws CharacterC
* ByteBuffer.limit()
*/
- public static ByteBuffer encode(String string) throws CharacterCodingException {
+ public static ByteBuffer encode(final String string) throws CharacterCodingException {
return encode(string, true);
}
@@ -411,7 +412,8 @@ public static ByteBuffer encode(String string) throws CharacterCodingException {
* @return ByteBuffer: bytes stores at ByteBuffer.array() and length is
* ByteBuffer.limit()
*/
- public static ByteBuffer encode(String string, boolean replace) throws CharacterCodingException {
+ public static ByteBuffer encode(final String string, final boolean replace) throws CharacterCodingException {
+
final CharsetEncoder encoder = ENCODER_FACTORY.get();
if (replace) {
encoder.onMalformedInput(CodingErrorAction.REPLACE);
@@ -428,7 +430,7 @@ public static ByteBuffer encode(String string, boolean replace) throws Character
/**
* Read a UTF8 encoded string from in
*/
- public static String readString(DataInput in) throws IOException {
+ public static String readString(final DataInput in) throws IOException {
if (in.readBoolean()) {
final int length = in.readInt();
@@ -447,7 +449,7 @@ public static String readString(DataInput in) throws IOException {
/**
* Write a UTF8 encoded string to out
*/
- public static int writeString(DataOutput out, String s) throws IOException {
+ public static int writeString(final DataOutput out, final String s) throws IOException {
int length = 0;
@@ -479,7 +481,7 @@ public static int writeString(DataOutput out, String s) throws IOException {
* @throws MalformedInputException
* if the byte array contains invalid utf-8
*/
- public static void validateUTF8(byte[] utf8) throws MalformedInputException {
+ public static void validateUTF8(final byte[] utf8) throws MalformedInputException {
validateUTF8(utf8, 0, utf8.length);
}
@@ -495,7 +497,7 @@ public static void validateUTF8(byte[] utf8) throws MalformedInputException {
* @throws MalformedInputException
* if the byte array contains invalid bytes
*/
- public static void validateUTF8(byte[] utf8, int start, int len) throws MalformedInputException {
+ public static void validateUTF8(final byte[] utf8, final int start, final int len) throws MalformedInputException {
int count = start;
int leadByte = 0;
int length = 0;
@@ -593,7 +595,7 @@ public static void validateUTF8(byte[] utf8, int start, int len) throws Malforme
* buffer's position will be incremented. Any mark set on this buffer will
* be changed by this method!
*/
- public static int bytesToCodePoint(ByteBuffer bytes) {
+ public static int bytesToCodePoint(final ByteBuffer bytes) {
bytes.mark();
final byte b = bytes.get();
bytes.reset();
@@ -637,7 +639,7 @@ public static int bytesToCodePoint(ByteBuffer bytes) {
* text to encode
* @return number of UTF-8 bytes required to encode
*/
- public static int utf8Length(String string) {
+ public static int utf8Length(final String string) {
final CharacterIterator iter = new StringCharacterIterator(string);
char ch = iter.first();
int size = 0;
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java
index 690e282506d43..d94c1f69cdc75 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java
@@ -27,7 +27,7 @@
*
* @author warneke
*/
-public class ClassUtils {
+public final class ClassUtils {
/**
* The log object used for debugging.
@@ -51,7 +51,8 @@ private ClassUtils() {
*/
// TODO: See if we can improve type safety here
@SuppressWarnings("unchecked")
- public static Class extends VersionedProtocol> getProtocolByName(String className) throws ClassNotFoundException {
+ public static Class extends VersionedProtocol> getProtocolByName(final String className)
+ throws ClassNotFoundException {
if (!className.contains("Protocol")) {
throw new ClassNotFoundException("Only use this method for protocols!");
@@ -71,7 +72,7 @@ public static Class extends VersionedProtocol> getProtocolByName(String classN
*/
// TODO: See if we can improve type safety here
@SuppressWarnings("unchecked")
- public static Class extends Record> getRecordByName(String className) throws ClassNotFoundException {
+ public static Class extends Record> getRecordByName(final String className) throws ClassNotFoundException {
return (Class extends Record>) Class.forName(className, true, getClassLoader());
@@ -88,7 +89,7 @@ public static Class extends Record> getRecordByName(String className) throws C
*/
// TODO: See if we can improve type safety here
@SuppressWarnings("unchecked")
- public static Class extends FileSystem> getFileSystemByName(String className) throws ClassNotFoundException {
+ public static Class extends FileSystem> getFileSystemByName(final String className) throws ClassNotFoundException {
return (Class extends FileSystem>) Class.forName(className, true, getClassLoader());
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java
index a83204760532a..eea2a8c4b006e 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java
@@ -26,7 +26,7 @@
*
* @author warneke
*/
-public class EnumUtils {
+public final class EnumUtils {
/**
* Private constructor to overwrite public one.
@@ -47,7 +47,7 @@ private EnumUtils() {
* @throws IOException
* thrown if any error occurred while reading data from the stream
*/
- public static > T readEnum(DataInput in, Class enumType) throws IOException {
+ public static > T readEnum(final DataInput in, final Class enumType) throws IOException {
if (!in.readBoolean()) {
return null;
@@ -66,7 +66,7 @@ public static > T readEnum(DataInput in, Class enumType) th
* @throws IOException
* thrown if any error occurred while writing data to the stream
*/
- public static void writeEnum(DataOutput out, Enum> enumVal) throws IOException {
+ public static void writeEnum(final DataOutput out, final Enum> enumVal) throws IOException {
if (enumVal == null) {
out.writeBoolean(false);
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java
index bc88ceef4f299..da209b8ef7bc3 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java
@@ -20,18 +20,24 @@
*
* @author warneke
*/
-public class FileUtils {
+public final class FileUtils {
/**
* The alphabet to construct the random part of the filename from.
*/
- private static char[] ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd', 'e',
- 'f' };
+ private static final char[] ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd',
+ 'e', 'f' };
/**
- * The length of the random part of the filename
+ * The length of the random part of the filename.
*/
- private static int LENGTH = 12;
+ private static final int LENGTH = 12;
+
+ /**
+ * Empty private constructor to avoid instantiation.
+ */
+ private FileUtils() {
+ }
/**
* Constructs a random filename with the given prefix and
@@ -41,12 +47,12 @@ public class FileUtils {
* the prefix to the filename to be constructed
* @return the generated random filename with the given prefix
*/
- public static String getRandomFilename(String prefix) {
+ public static String getRandomFilename(final String prefix) {
final StringBuilder stringBuilder = new StringBuilder(prefix);
for (int i = 0; i < LENGTH; i++) {
- stringBuilder.append(ALPHABET[(int) Math.floor(Math.random() * (double)ALPHABET.length)]);
+ stringBuilder.append(ALPHABET[(int) Math.floor(Math.random() * (double) ALPHABET.length)]);
}
return stringBuilder.toString();
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java
index 69800bacd4175..31c2ceff892e5 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java
@@ -28,7 +28,7 @@
*
* @author warneke
*/
-public class IOUtils {
+public final class IOUtils {
/**
* The block size for byte operations in byte.
@@ -56,10 +56,11 @@ private IOUtils() {
* @throws IOException
* thrown if an error occurred while writing to the output stream
*/
- public static void copyBytes(InputStream in, OutputStream out, int buffSize, boolean close) throws IOException {
+ public static void copyBytes(final InputStream in, final OutputStream out, final int buffSize, final boolean close)
+ throws IOException {
final PrintStream ps = out instanceof PrintStream ? (PrintStream) out : null;
- final byte buf[] = new byte[buffSize];
+ final byte[] buf = new byte[buffSize];
try {
int bytesRead = in.read(buf);
while (bytesRead >= 0) {
@@ -88,7 +89,7 @@ public static void copyBytes(InputStream in, OutputStream out, int buffSize, boo
* @throws IOException
* thrown if an I/O error occurs while copying
*/
- public static void copyBytes(InputStream in, OutputStream out) throws IOException {
+ public static void copyBytes(final InputStream in, final OutputStream out) throws IOException {
copyBytes(in, out, BLOCKSIZE, true);
}
@@ -105,7 +106,7 @@ public static void copyBytes(InputStream in, OutputStream out) throws IOExceptio
* @throws IOException
* thrown if an I/O error occurs while copying
*/
- public static void copyBytes(InputStream in, OutputStream out, boolean close) throws IOException {
+ public static void copyBytes(final InputStream in, final OutputStream out, final boolean close) throws IOException {
copyBytes(in, out, BLOCKSIZE, close);
}
@@ -123,7 +124,8 @@ public static void copyBytes(InputStream in, OutputStream out, boolean close) th
* @throws IOException
* if it could not read requested number of bytes for any reason (including EOF)
*/
- public static void readFully(InputStream in, byte buf[], int off, int len) throws IOException {
+ public static void readFully(final InputStream in, final byte[] buf, int off, final int len)
+ throws IOException {
int toRead = len;
while (toRead > 0) {
final int ret = in.read(buf, off, toRead);
@@ -145,7 +147,7 @@ public static void readFully(InputStream in, byte buf[], int off, int len) throw
* @throws IOException
* if it could not skip requested number of bytes for any reason (including EOF)
*/
- public static void skipFully(InputStream in, long len) throws IOException {
+ public static void skipFully(final InputStream in, long len) throws IOException {
while (len > 0) {
final long ret = in.skip(len);
if (ret < 0) {
@@ -164,7 +166,7 @@ public static void skipFully(InputStream in, long len) throws IOException {
* @param closeables
* the objects to close
*/
- public static void cleanup(Log log, java.io.Closeable... closeables) {
+ public static void cleanup(final Log log, final java.io.Closeable... closeables) {
for (java.io.Closeable c : closeables) {
if (c != null) {
try {
@@ -185,7 +187,7 @@ public static void cleanup(Log log, java.io.Closeable... closeables) {
* @param stream
* the stream to close
*/
- public static void closeStream(java.io.Closeable stream) {
+ public static void closeStream(final java.io.Closeable stream) {
cleanup(null, stream);
}
@@ -195,7 +197,7 @@ public static void closeStream(java.io.Closeable stream) {
* @param sock
* the socket to close
*/
- public static void closeSocket(Socket sock) {
+ public static void closeSocket(final Socket sock) {
// avoids try { close() } dance
if (sock != null) {
try {
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java
index e88abe522ec63..599e649580ffe 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java
@@ -38,7 +38,7 @@ public class JarFileCreator {
/**
* The file extension of java classes.
*/
- private final static String CLASS_EXTENSION = ".class";
+ private static final String CLASS_EXTENSION = ".class";
/**
* A set of classes which shall be included in the final jar file.
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java
index db4ea676c361c..9252e1a21a803 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java
@@ -28,7 +28,7 @@
*
* @author warneke
*/
-public class NativeCodeLoader {
+public final class NativeCodeLoader {
/**
* Set of the native libraries which are already loaded.
@@ -36,7 +36,7 @@ public class NativeCodeLoader {
private static Set loadedLibrarySet = new HashSet();
/**
- * Directory prefix for native libraries inside JAR files
+ * Directory prefix for native libraries inside JAR files.
*/
private static final String JAR_PREFIX = "META-INF/lib/";
@@ -58,15 +58,10 @@ private NativeCodeLoader() {
* the directory in which the library is supposed to be located
* @param filename
* filename of the library to be loaded
- * @throws SecurityException
- * thrown if a security exception occurs.
- * @throws UnsatisfiedLinkError
- * thrown if the library contains unsatisfied links
* @throws IOException
* thrown if an internal native library cannot be extracted
*/
- public static void loadLibraryFromFile(String directory, String filename) throws SecurityException,
- UnsatisfiedLinkError, IOException {
+ public static void loadLibraryFromFile(final String directory, final String filename) throws IOException {
final String libraryPath = directory + File.separator + filename;
@@ -101,7 +96,7 @@ public static void loadLibraryFromFile(String directory, String filename) throws
* @throws IOException
* thrown if an I/O error occurs while copying the data
*/
- private static void copy(InputStream in, OutputStream out) throws IOException {
+ private static void copy(final InputStream in, final OutputStream out) throws IOException {
final byte[] buf = new byte[BUFSIZE];
int len = 0;
@@ -121,7 +116,7 @@ private static void copy(InputStream in, OutputStream out) throws IOException {
* the filename of the library to check
* @return true
if the library is already loaded, false
otherwise
*/
- public static boolean isLibraryLoaded(String libraryName) {
+ public static boolean isLibraryLoaded(final String libraryName) {
synchronized (loadedLibrarySet) {
return loadedLibrarySet.contains(libraryName);
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java
index fae8b8f3a47a4..79cf56ea0c269 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java
@@ -46,7 +46,7 @@ public class SerializableArrayList extends ArrayLi
* {@inheritDoc}
*/
@Override
- public void write(DataOutput out) throws IOException {
+ public void write(final DataOutput out) throws IOException {
out.writeInt(size());
final Iterator it = iterator();
@@ -64,9 +64,8 @@ public void write(DataOutput out) throws IOException {
* {@inheritDoc}
*/
@SuppressWarnings("unchecked")
- // TODO: See if type safety can be improved here
@Override
- public void read(DataInput in) throws IOException {
+ public void read(final DataInput in) throws IOException {
// Make sure the list is empty
clear();
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java
index 7438b984c6576..aad1491b93e5c 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java
@@ -53,11 +53,11 @@ public class SerializableHashMap> it = entrySet().iterator();
-
- while(it.hasNext()) {
-
+
+ while (it.hasNext()) {
+
final Map.Entry entry = it.next();
final K key = entry.getKey();
final V value = entry.getValue();
@@ -72,14 +72,13 @@ public void write(final DataOutput out) throws IOException {
* {@inheritDoc}
*/
@SuppressWarnings("unchecked")
- // TODO: See if type safety can be improved here
@Override
public void read(final DataInput in) throws IOException {
final int numberOfMapEntries = in.readInt();
-
- for(int i = 0; i < numberOfMapEntries; i++) {
-
+
+ for (int i = 0; i < numberOfMapEntries; i++) {
+
final String keyType = StringRecord.readString(in);
Class keyClass = null;
try {
@@ -94,9 +93,9 @@ public void read(final DataInput in) throws IOException {
} catch (Exception e) {
throw new IOException(StringUtils.stringifyException(e));
}
-
+
key.read(in);
-
+
final String valueType = StringRecord.readString(in);
Class valueClass = null;
try {
@@ -112,10 +111,9 @@ public void read(final DataInput in) throws IOException {
throw new IOException(StringUtils.stringifyException(e));
}
value.read(in);
-
+
put(key, value);
}
-
- }
+ }
}
diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java
index 57d781548f4bc..0de923f09904b 100644
--- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java
+++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java
@@ -29,7 +29,7 @@
*
* @author warneke
*/
-public class StringUtils {
+public final class StringUtils {
/**
* Empty private constructor to overwrite public one.
@@ -44,7 +44,7 @@ private StringUtils() {
* the exception to stringify
* @return A string with exception name and call stack.
*/
- public static String stringifyException(Throwable e) {
+ public static String stringifyException(final Throwable e) {
final StringWriter stm = new StringWriter();
final PrintWriter wrt = new PrintWriter(stm);
e.printStackTrace(wrt);
@@ -64,7 +64,7 @@ public static String stringifyException(Throwable e) {
* end index, exclusively
* @return hex string representation of the byte array
*/
- public static String byteToHexString(byte[] bytes, int start, int end) {
+ public static String byteToHexString(final byte[] bytes, final int start, final int end) {
if (bytes == null) {
throw new IllegalArgumentException("bytes == null");
}
@@ -83,7 +83,7 @@ public static String byteToHexString(byte[] bytes, int start, int end) {
* the bytes to convert in a hex string
* @return hex string representation of the byte array
*/
- public static String byteToHexString(byte bytes[]) {
+ public static String byteToHexString(final byte[] bytes) {
return byteToHexString(bytes, 0, bytes.length);
}
@@ -96,7 +96,7 @@ public static String byteToHexString(byte bytes[]) {
* @return a byte array that is a hex string representation of the given
* string. The size of the byte array is therefore hex.length/2
*/
- public static byte[] hexStringToByte(String hex) {
+ public static byte[] hexStringToByte(final String hex) {
final byte[] bts = new byte[hex.length() / 2];
for (int i = 0; i < bts.length; i++) {
bts[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16);
diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2ClientFactory.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2ClientFactory.java
similarity index 85%
rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2ClientFactory.java
rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2ClientFactory.java
index 0087e3fe06468..0ade972d70871 100644
--- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2ClientFactory.java
+++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2ClientFactory.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import java.util.Hashtable;
@@ -42,6 +42,14 @@ final class EC2ClientFactory {
*/
static synchronized AmazonEC2Client getEC2Client(final String awsAccessId, final String awsSecretKey) {
+ if(awsAccessId == null) {
+ throw new IllegalArgumentException("AWS access ID is null");
+ }
+
+ if(awsSecretKey == null) {
+ throw new IllegalArgumentException("AWS secret key is null");
+ }
+
// Check if a client-object was already generated
if (ec2clients.containsKey(awsAccessId)) {
return ec2clients.get(awsAccessId);
@@ -52,7 +60,7 @@ static synchronized AmazonEC2Client getEC2Client(final String awsAccessId, final
final BasicAWSCredentials credentials = new BasicAWSCredentials(awsAccessId, awsSecretKey);
final AmazonEC2Client client = new AmazonEC2Client(credentials);
- final String endpoint = GlobalConfiguration.getString("ec2.webservice.endpoint", "ec2.eu-west-1.amazonaws.com");
+ final String endpoint = GlobalConfiguration.getString("instancemanager.ec2.endpoint", "ec2.eu-west-1.amazonaws.com");
client.setEndpoint(endpoint);
diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstance.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstance.java
similarity index 82%
rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstance.java
rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstance.java
index 5d84d568867d1..63fe77cc9bc25 100644
--- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstance.java
+++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstance.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import eu.stratosphere.nephele.instance.AbstractInstance;
import eu.stratosphere.nephele.instance.AllocatedResource;
@@ -31,7 +31,7 @@
*
* @author warneke
*/
-public class CloudInstance extends AbstractInstance {
+public class EC2CloudInstance extends AbstractInstance {
/**
* The cached allocated resource object.
@@ -40,16 +40,21 @@ public class CloudInstance extends AbstractInstance {
/** The instance ID. */
private final String instanceID;
-
+
/** The AWS Access Key to access this machine */
private String awsAccessKey;
-
+
/** The AWS Secret Key to access this machine */
private String awsSecretKey;
-
+
/** The time the instance was allocated. */
private final long launchTime;
+ /**
+ * The period of time for which the instance is leased from EC2.
+ */
+ private final long leasePeriod;
+
/** The last received heart beat. */
private long lastReceivedHeartBeat = System.currentTimeMillis();
@@ -64,20 +69,23 @@ public class CloudInstance extends AbstractInstance {
* the instance type
* @param instanceConnectionInfo
* the information required to connect to the instance's task manager
- * @param allocationTime
+ * @param launchTime
* the time the instance was allocated
+ * @param leasePeriod
+ * the period of time for which the instance is leased from EC2
* @param parentNode
* the parent node in the network topology
* @param hardwareDescription
* the hardware description reported by the instance itself
* @param awsAccessKey
- * The AWS Access Key to access this machine
+ * The AWS Access Key to access this machine
* @param awsSecretKey
- * The AWS Secret Key to access this machine
+ * The AWS Secret Key to access this machine
*/
- public CloudInstance(String instanceID, InstanceType type,
- InstanceConnectionInfo instanceConnectionInfo, long launchTime, NetworkNode parentNode,
- NetworkTopology networkTopology, HardwareDescription hardwareDescription, String awsAccessKey, String awsSecretKey) {
+ public EC2CloudInstance(String instanceID, InstanceType type,
+ InstanceConnectionInfo instanceConnectionInfo, long launchTime, long leasePeriod, NetworkNode parentNode,
+ NetworkTopology networkTopology, HardwareDescription hardwareDescription, String awsAccessKey,
+ String awsSecretKey) {
super(type, instanceConnectionInfo, parentNode, networkTopology, hardwareDescription);
this.allocatedResource = new AllocatedResource(this, type, new AllocationID());
@@ -85,9 +93,11 @@ public CloudInstance(String instanceID, InstanceType type,
this.instanceID = instanceID;
this.launchTime = launchTime;
-
+
+ this.leasePeriod = leasePeriod;
+
this.awsAccessKey = awsAccessKey;
-
+
this.awsSecretKey = awsSecretKey;
}
@@ -131,9 +141,11 @@ public AllocatedResource asAllocatedResource() {
/**
* Returns this Instance as a FloatingInstance object
+ *
* @return
*/
- public FloatingInstance asFloatingInstance(){
- return new FloatingInstance(this.instanceID, this.getInstanceConnectionInfo(), this.launchTime, this.getType(), this.awsAccessKey, this.awsSecretKey);
+ public FloatingInstance asFloatingInstance() {
+ return new FloatingInstance(this.instanceID, this.getInstanceConnectionInfo(), this.launchTime,
+ this.leasePeriod, this.getType(), this.getHardwareDescription(), this.awsAccessKey, this.awsSecretKey);
}
}
diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceNotifier.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceNotifier.java
similarity index 92%
rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceNotifier.java
rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceNotifier.java
index f2290b1ae5c20..052c181cf2bf3 100644
--- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceNotifier.java
+++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceNotifier.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import java.util.List;
@@ -32,7 +32,7 @@
*
* @author warneke
*/
-final class CloudInstanceNotifier extends Thread {
+final class EC2CloudInstanceNotifier extends Thread {
/**
* The {@link InstanceListener} object to send the notification to.
@@ -53,7 +53,7 @@ final class CloudInstanceNotifier extends Thread {
* @param allocatedResources
* the resources which have been allocated for the job with the given ID
*/
- public CloudInstanceNotifier(final InstanceListener instanceListener, final JobID jobID,
+ public EC2CloudInstanceNotifier(final InstanceListener instanceListener, final JobID jobID,
final List allocatedResources) {
this.instanceListener = instanceListener;
this.jobID = jobID;
diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudManager.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManager.java
similarity index 69%
rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudManager.java
rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManager.java
index 76d22f1bf62af..531cc4124d84d 100644
--- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudManager.java
+++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManager.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import java.net.InetAddress;
import java.net.UnknownHostException;
@@ -32,11 +32,13 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import com.amazonaws.AmazonClientException;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.amazonaws.services.ec2.model.BlockDeviceMapping;
import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
import com.amazonaws.services.ec2.model.DescribeInstancesResult;
import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.Placement;
import com.amazonaws.services.ec2.model.Reservation;
import com.amazonaws.services.ec2.model.RunInstancesRequest;
import com.amazonaws.services.ec2.model.RunInstancesResult;
@@ -51,12 +53,12 @@
import eu.stratosphere.nephele.instance.InstanceException;
import eu.stratosphere.nephele.instance.InstanceListener;
import eu.stratosphere.nephele.instance.InstanceManager;
-import eu.stratosphere.nephele.instance.InstanceRequestMap;
import eu.stratosphere.nephele.instance.InstanceType;
import eu.stratosphere.nephele.instance.InstanceTypeDescription;
import eu.stratosphere.nephele.instance.InstanceTypeDescriptionFactory;
import eu.stratosphere.nephele.instance.InstanceTypeFactory;
import eu.stratosphere.nephele.jobgraph.JobID;
+import eu.stratosphere.nephele.topology.NetworkNode;
import eu.stratosphere.nephele.topology.NetworkTopology;
import eu.stratosphere.nephele.util.SerializableHashMap;
import eu.stratosphere.nephele.util.StringUtils;
@@ -70,10 +72,37 @@
* user pays fee for allocated instances hourly. The cloud manager checks the floating instances every 60 seconds in
* order to terminate the floating instances which expire.
*/
-public final class CloudManager extends TimerTask implements InstanceManager {
+public final class EC2CloudManager extends TimerTask implements InstanceManager {
- /** The log for the cloud manager. */
- private static final Log LOG = LogFactory.getLog(CloudManager.class);
+ /**
+ * The log for the EC2 cloud manager.
+ **/
+ private static final Log LOG = LogFactory.getLog(EC2CloudManager.class);
+
+ /**
+ * The key to access the lease period from the configuration.
+ */
+ private static String LEASE_PERIOD_KEY = "instancemanager.ec2.leaseperiod";
+
+ /**
+ * The default lease period in milliseconds for instances on Amazon EC2.
+ */
+ static final long DEFAULT_LEASE_PERIOD = 60 * 60 * 1000; // 1 hour in ms.
+
+ /**
+ * The configuration key to access the AWS access ID of a job.
+ */
+ static final String AWS_ACCESS_ID_KEY = "job.ec2.awsaccessid";
+
+ /**
+ * The configuration key to access the AWS secret key of a job.
+ */
+ static final String AWS_SECRET_KEY_KEY = "job.ec2.awssecretkey";
+
+ /**
+ * The configuration key to access the AMI to run the task managers on.
+ */
+ static final String AWS_AMI_KEY = "job.ec2.ami";
/**
* The cloud manager checks the floating instances every base interval to terminate the floating instances which
@@ -87,12 +116,26 @@ public final class CloudManager extends TimerTask implements InstanceManager {
*/
private static final int DEFAULTCLEANUPINTERVAL = 2 * 60 * 1000; // 2 min
- /** TMs that send HeartBeats but do not belong to any job are kept in this set. */
- private final HashSet orphanedTMs = new HashSet();
+ /**
+ * Instances that send heart beats but do not belong to any job are kept in this map.
+ **/
+ private final Map orphanedInstances = new HashMap();
- /** The array of all available instance types in the cloud. */
+ /**
+ * The array of all available instance types in the cloud.
+ **/
private final InstanceType[] availableInstanceTypes;
+ /**
+ * The lease period for instances on Amazon EC2 in milliseconds, potentially configured by the user.
+ */
+ private final long leasePeriod;
+
+ /**
+ * The default instance type.
+ */
+ private InstanceType defaultInstanceType = null;
+
/** Mapping jobs to instances. */
private final Map jobToInstancesAssignmentMap = new HashMap();
@@ -115,19 +158,34 @@ public final class CloudManager extends TimerTask implements InstanceManager {
private long cleanUpInterval = DEFAULTCLEANUPINTERVAL;
/**
- * The network topology inside the cloud (currently only a fake).
+ * The network topology for each job.
+ */
+ private final Map networkTopologies = new HashMap();
+
+ /**
+ * The preferred availability for instances on EC2.
*/
- private final NetworkTopology networkTopology;
+ private final String availabilityZone;
/**
* Creates the cloud manager.
*/
- public CloudManager() {
+ public EC2CloudManager() {
// Load the instance type this cloud can offer
this.availableInstanceTypes = populateInstanceTypeArray();
- this.cleanUpInterval = (long) GlobalConfiguration.getInteger("cloud.ec2.cleanupinterval",
+ // Calculate lease period
+ long lp = GlobalConfiguration.getInteger(LEASE_PERIOD_KEY, -1);
+ if (lp > 0) {
+ LOG.info("Found user-defined lease period of " + lp + " minutes for instances");
+ lp = lp * 1000L * 60L; // Convert to milliseconds
+ } else {
+ lp = DEFAULT_LEASE_PERIOD;
+ }
+ this.leasePeriod = lp;
+
+ this.cleanUpInterval = (long) GlobalConfiguration.getInteger("instancemanager.ec2.cleanupinterval",
DEFAULTCLEANUPINTERVAL);
if ((this.cleanUpInterval % BASEINTERVAL) != 0) {
@@ -135,7 +193,12 @@ public CloudManager() {
this.cleanUpInterval = DEFAULTCLEANUPINTERVAL;
}
- this.networkTopology = NetworkTopology.createEmptyTopology();
+ this.availabilityZone = GlobalConfiguration.getString("instancemanager.ec2.availabilityzone", null);
+ if (this.availabilityZone == null) {
+ LOG.info("No preferred availability zone configured");
+ } else {
+ LOG.info("Found " + this.availabilityZone + " as preferred availability zone in configuration");
+ }
this.timer.schedule(this, (long) BASEINTERVAL, (long) BASEINTERVAL);
}
@@ -153,23 +216,36 @@ private InstanceType[] populateInstanceTypeArray() {
final List instanceTypes = new ArrayList();
- // read the number of instance types
- final int num = GlobalConfiguration.getInteger("cloudmgr.nrtypes", -1);
- if (num <= 0) {
- throw new RuntimeException("Illegal configuration, cloudmgr.nrtypes is not configured");
- }
+ int count = 1;
+ while (true) {
- for (int i = 0; i < num; ++i) {
-
- final String key = "cloudmgr.instancetype." + (i + 1);
+ final String key = "instancemanager.ec2.type." + count;
final String type = GlobalConfiguration.getString(key, null);
if (type == null) {
- throw new RuntimeException("Illegal configuration for " + key);
+ break;
}
- instanceTypes.add(InstanceTypeFactory.constructFromDescription(type));
+ final InstanceType instanceType = InstanceTypeFactory.constructFromDescription(type);
+ LOG.info("Found instance type " + count + ": " + instanceType);
+
+ instanceTypes.add(instanceType);
+ ++count;
+ }
+
+ if (instanceTypes.isEmpty()) {
+ LOG.error("No instance types found in configuration");
+ return new InstanceType[0];
+ }
+
+ int defaultIndex = GlobalConfiguration.getInteger("instancemanager.ec2.defaulttype", -1);
+ if (defaultIndex < 1 || defaultIndex >= (instanceTypes.size() + 1)) {
+ LOG.warn("Invalid index to default instance " + defaultIndex);
+ defaultIndex = 1;
}
+ this.defaultInstanceType = instanceTypes.get(defaultIndex - 1);
+ LOG.info("Default instance type is " + this.defaultInstanceType);
+
// sort by price
Collections.sort(instanceTypes, new Comparator() {
@Override
@@ -242,13 +318,15 @@ public synchronized void releaseAllocatedResource(final JobID jobID, final Confi
if (jobToInstanceMapping.getAssignedInstances().contains(instance)) {
// Unassigned Instance
- jobToInstanceMapping.unassignInstanceFromJob((CloudInstance) instance);
+ jobToInstanceMapping.unassignInstanceFromJob((EC2CloudInstance) instance);
// Make it a floating Instance
this.floatingInstances.put(instance.getInstanceConnectionInfo(),
- ((CloudInstance) instance).asFloatingInstance());
+ ((EC2CloudInstance) instance).asFloatingInstance());
+
+ // TODO: Clean up job to instance mapping and network topology
- LOG.info("Convert " + ((CloudInstance) instance).getInstanceID()
+ LOG.info("Converting " + ((EC2CloudInstance) instance).getInstanceID()
+ " from allocated instance to floating instance");
} else {
@@ -265,8 +343,10 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo
final HardwareDescription hardwareDescription) {
// Check if this TM is orphaned
- if (this.orphanedTMs.contains(instanceConnectionInfo)) {
- LOG.debug("Received HeartBeat from orphaned TM " + instanceConnectionInfo);
+ if (this.orphanedInstances.containsKey(instanceConnectionInfo)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received heart beat from orphaned instance " + instanceConnectionInfo);
+ }
return;
}
@@ -278,7 +358,7 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo
}
// Check if heart beat belongs to an assigned instance
- CloudInstance instance = isAssignedInstance(instanceConnectionInfo);
+ EC2CloudInstance instance = isAssignedInstance(instanceConnectionInfo);
if (instance != null) {
// If it's an assigned instance, just update the heart beat time stamp and leave
instance.updateLastReceivedHeartBeat();
@@ -287,7 +367,7 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo
// Check if heart beat belongs to a reserved instance
try {
- instance = isReservedInstance(instanceConnectionInfo);
+ instance = isReservedInstance(instanceConnectionInfo, hardwareDescription);
} catch (InstanceException e) {
LOG.error(e);
}
@@ -310,16 +390,15 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo
mapping.assignInstanceToJob(instance);
// Trigger notification that instance is available (outside synchronized section)
- final List allocatedResources = new ArrayList(1);
- allocatedResources.add(instance.asAllocatedResource());
- this.instanceListener.resourcesAllocated(jobID, allocatedResources);
+ this.instanceListener.resourceAllocated(jobID, instance.asAllocatedResource());
return;
}
// This TM seems to be unknown to the JobManager.. blacklist
- LOG.info("Received HeartBeat from unknown TM. Put into orphaned TM set. Address is: " + instanceConnectionInfo);
- this.orphanedTMs.add(instanceConnectionInfo);
+ LOG.info("Received heart beat from unknown instance " + instanceConnectionInfo
+ + ", converting it into orphaned instance");
+ this.orphanedInstances.put(instanceConnectionInfo, hardwareDescription);
}
@@ -330,7 +409,7 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo
* the connection information object identifying the instance
* @return a cloud instance
*/
- private CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceConnectionInfo) {
+ private EC2CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceConnectionInfo) {
synchronized (this.jobToInstancesAssignmentMap) {
@@ -340,7 +419,7 @@ private CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceCo
final JobID jobID = it.next();
final JobToInstancesMapping m = this.jobToInstancesAssignmentMap.get(jobID);
- final CloudInstance instance = m.getInstanceByConnectionInfo(instanceConnectionInfo);
+ final EC2CloudInstance instance = m.getInstanceByConnectionInfo(instanceConnectionInfo);
if (instance != null) {
return instance;
}
@@ -355,11 +434,14 @@ private CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceCo
*
* @param instanceConnectionInfo
* the {@link InstanceConnectionInfo} object identifying the instance
+ * @param hardwareDescription
+ * the actual hardware description of the instance
* @return a cloud instance
* @throws InstanceException
* something wrong happens to the global configuration
*/
- private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceConnectionInfo)
+ private EC2CloudInstance isReservedInstance(final InstanceConnectionInfo instanceConnectionInfo,
+ final HardwareDescription hardwareDescription)
throws InstanceException {
if (instanceConnectionInfo == null) {
@@ -380,18 +462,18 @@ private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceCo
jobsWithReservedInstances.add(id);
}
- // Now we call the webservice for each job..
+ // Now we call the web service for each job..
- for (JobID id : jobsWithReservedInstances) {
+ for (final JobID jobID : jobsWithReservedInstances) {
JobToInstancesMapping mapping = null;
synchronized (this.jobToInstancesAssignmentMap) {
- mapping = this.jobToInstancesAssignmentMap.get(id);
+ mapping = this.jobToInstancesAssignmentMap.get(jobID);
}
if (mapping == null) {
- LOG.error("Unknown mapping for job ID " + id);
+ LOG.error("Unknown mapping for job ID " + jobID);
continue;
}
@@ -414,8 +496,18 @@ private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceCo
}
if (instanceConnectionInfo.getAddress().equals(candidateAddress)) {
+
+ NetworkTopology networkTopology;
+ synchronized (this.networkTopologies) {
+ networkTopology = this.networkTopologies.get(jobID);
+ }
+
+ if (networkTopology == null) {
+ throw new InstanceException("Cannot find network topology for job " + jobID);
+ }
+
return convertIntoCloudInstance(t, instanceConnectionInfo, mapping.getAwsAccessId(),
- mapping.getAwsSecretKey());
+ mapping.getAwsSecretKey(), networkTopology.getRootNode(), hardwareDescription);
}
}
}
@@ -436,8 +528,9 @@ private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceCo
* the information required to connect to the instance's task manager later on
* @return a cloud instance
*/
- private CloudInstance convertIntoCloudInstance(final com.amazonaws.services.ec2.model.Instance instance,
- final InstanceConnectionInfo instanceConnectionInfo, final String awsAccessKey, final String awsSecretKey) {
+ private EC2CloudInstance convertIntoCloudInstance(final Instance instance,
+ final InstanceConnectionInfo instanceConnectionInfo, final String awsAccessKey, final String awsSecretKey,
+ final NetworkNode parentNode, final HardwareDescription hardwareDescription) {
InstanceType type = null;
@@ -455,9 +548,9 @@ private CloudInstance convertIntoCloudInstance(final com.amazonaws.services.ec2.
return null;
}
- final CloudInstance cloudInstance = new CloudInstance(instance.getInstanceId(), type, instanceConnectionInfo,
- instance.getLaunchTime().getTime(), this.networkTopology.getRootNode(), this.networkTopology, null,
- awsAccessKey, awsSecretKey);
+ final EC2CloudInstance cloudInstance = new EC2CloudInstance(instance.getInstanceId(), type,
+ instanceConnectionInfo, instance.getLaunchTime().getTime(), this.leasePeriod, parentNode,
+ parentNode.getNetworkTopology(), hardwareDescription, awsAccessKey, awsSecretKey);
// TODO: Define hardware descriptions for cloud instance types
@@ -468,8 +561,9 @@ private CloudInstance convertIntoCloudInstance(final com.amazonaws.services.ec2.
* {@inheritDoc}
*/
@Override
- public void requestInstance(final JobID jobID, Configuration conf, final InstanceRequestMap instanceRequestMap,
- final List splitAffinityList) throws InstanceException {
+ public void requestInstance(final JobID jobID, Configuration conf,
+ final Map instanceRequestMap, final List splitAffinityList)
+ throws InstanceException {
if (conf == null) {
throw new IllegalArgumentException("No job configuration provided, unable to acquire credentials");
@@ -477,17 +571,21 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc
// First check, if all required configuration entries are available
- final String awsAccessId = conf.getString("job.cloud.awsaccessid", null);
- LOG.info("found AWS access ID from Job Conf: " + awsAccessId);
+ final String awsAccessId = conf.getString(AWS_ACCESS_ID_KEY, null);
if (awsAccessId == null) {
throw new InstanceException("Unable to allocate cloud instance: Cannot find AWS access ID");
}
- final String awsSecretKey = conf.getString("job.cloud.awssecretkey", null);
+
+ final String awsSecretKey = conf.getString(AWS_SECRET_KEY_KEY, null);
if (awsSecretKey == null) {
throw new InstanceException("Unable to allocate cloud instance: Cannot find AWS secret key");
}
- // First we check, if there are any orphaned TMs that are accessible with the provided configuration
+ if (conf.getString(AWS_AMI_KEY, null) == null) {
+ throw new InstanceException("Unable to allocate cloud instance: Cannot find AMI image ID");
+ }
+
+ // First we check, if there are any orphaned instances that are accessible with the provided configuration
checkAndConvertOrphanedInstances(conf);
// Check if there already exist a mapping for this job
@@ -503,12 +601,22 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc
}
}
+ // Check if there already exists a network topology for this job
+ NetworkTopology networkTopology = null;
+ synchronized (this.networkTopologies) {
+ networkTopology = this.networkTopologies.get(jobID);
+ if (networkTopology == null) {
+ networkTopology = NetworkTopology.createEmptyTopology();
+ this.networkTopologies.put(jobID, networkTopology);
+ }
+ }
+
// Our bill with all instances that we will provide...
final LinkedList floatingInstances = new LinkedList();
final LinkedList requestedInstances = new LinkedList();
// We iterate over the maximum of requested Instances...
- final Iterator> it = instanceRequestMap.getMaximumIterator();
+ final Iterator> it = instanceRequestMap.entrySet().iterator();
while (it.hasNext()) {
@@ -517,11 +625,12 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc
// This is our actual type...
final InstanceType actualtype = e.getKey();
final int maxcount = e.getValue();
- final int mincount = instanceRequestMap.getMinimumNumberOfInstances(actualtype);
+ final int mincount = maxcount;
+ LOG.info("Requesting " + maxcount + " instances of type " + actualtype + " for job " + jobID);
// And this is the list of instances we will have...
- LinkedList actualFloatingInstances = new LinkedList();
- LinkedList actualRequestedInstances = new LinkedList();
+ LinkedList actualFloatingInstances = null;
+ LinkedList actualRequestedInstances = null;
// Check if floating instances available...
actualFloatingInstances = anyFloatingInstancesAvailable(awsAccessId, awsSecretKey, actualtype, maxcount);
@@ -533,6 +642,8 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc
actualRequestedInstances = allocateCloudInstance(conf, actualtype, minimumrequestcount,
maximumrequestcount);
+ } else {
+ actualRequestedInstances = new LinkedList();
}
// Add provided Instances to overall bill...
@@ -541,8 +652,8 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc
// Are we outer limits?
if (actualRequestedInstances.size() + actualFloatingInstances.size() < mincount) {
- LOG.error("Requested: " + mincount + " to " + maxcount + " instanes of type "
- + actualtype.getIdentifier() + ". Could only provide "
+ LOG.error("Requested: " + mincount + " to " + maxcount + " instances of type "
+ + actualtype.getIdentifier() + ", but could only provide "
+ (actualRequestedInstances.size() + actualFloatingInstances.size()) + ".");
// something went wrong.. give the floating instances back!
@@ -551,25 +662,21 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc
this.floatingInstances.put(i.getInstanceConnectionInfo(), i);
}
}
- throw new InstanceException("Could not allocate enough cloud instances");
+ throw new InstanceException("Could not allocate enough cloud instances. See logs for details.");
} // End outer limits
} // End iterating over instance types..
- // If we reached this point, everything went well
- final List allocatedResources = new ArrayList();
-
// Convert and allocate Floating Instances...
for (final FloatingInstance fi : floatingInstances) {
- final CloudInstance ci = fi.asCloudInstance();
+ final EC2CloudInstance ci = fi.asCloudInstance(networkTopology.getRootNode());
jobToInstanceMapping.assignInstanceToJob(ci);
- allocatedResources.add(ci.asAllocatedResource());
+ final EC2CloudInstanceNotifier notifier = new EC2CloudInstanceNotifier(this.instanceListener, jobID,
+ ci.asAllocatedResource());
+ notifier.start();
}
// Finally, inform the scheduler about the instances which have been floating before
- final CloudInstanceNotifier notifier = new CloudInstanceNotifier(this.instanceListener, jobID,
- allocatedResources);
- notifier.start();
// Add reserved Instances to Job Mapping...
for (final String i : requestedInstances) {
@@ -594,26 +701,18 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc
private LinkedList allocateCloudInstance(final Configuration conf, final InstanceType type,
final int mincount, final int maxcount) {
- final String awsAccessId = conf.getString("job.cloud.awsaccessid", null);
- final String awsSecretKey = conf.getString("job.cloud.awssecretkey", null);
-
- String imageID = conf.getString("job.ec2.image.id", null);
- LOG.info("EC2 Image ID from job conf: " + imageID);
- if (imageID == null) {
+ final String awsAccessId = conf.getString(AWS_ACCESS_ID_KEY, null);
+ final String awsSecretKey = conf.getString(AWS_SECRET_KEY_KEY, null);
- imageID = GlobalConfiguration.getString("ec2.image.id", null);
- if (imageID == null) {
- LOG.error("Unable to allocate instance: Image ID is unknown");
- return null;
- }
- }
+ final String imageID = conf.getString(AWS_AMI_KEY, null);
+ LOG.info("Read Amazon Machine Image from job configuration: " + imageID);
final String jobManagerIPAddress = GlobalConfiguration.getString("jobmanager.rpc.address", null);
if (jobManagerIPAddress == null) {
LOG.error("JobManager IP address is not set (jobmanager.rpc.address)");
return null;
}
- final String sshKeyPair = conf.getString("job.cloud.sshkeypair", null);
+ final String sshKeyPair = conf.getString("job.ec2.sshkeypair", null);
final AmazonEC2Client ec2client = EC2ClientFactory.getEC2Client(awsAccessId, awsSecretKey);
final LinkedList instanceIDs = new LinkedList();
@@ -623,6 +722,21 @@ private LinkedList allocateCloudInstance(final Configuration conf, final
final RunInstancesRequest request = new RunInstancesRequest(imageID, mincount, maxcount);
request.setInstanceType(type.getIdentifier());
+ // Set availability zone if configured
+ String av = null;
+ if (this.availabilityZone != null) {
+ av = this.availabilityZone;
+ }
+ final String jobAV = conf.getString("job.ec2.availabilityzone", null);
+ if (jobAV != null) {
+ LOG.info("Found " + jobAV + " as job-specific preference for availability zone");
+ av = jobAV;
+ }
+
+ if (av != null) {
+ request.setPlacement(new Placement(av));
+ }
+
// TODO: Make this configurable!
final BlockDeviceMapping bdm = new BlockDeviceMapping();
bdm.setVirtualName("ephemeral0");
@@ -640,10 +754,15 @@ private LinkedList allocateCloudInstance(final Configuration conf, final
request.setUserData(EC2Utilities.createTaskManagerUserData(jobManagerIPAddress));
// Request instances!
- final RunInstancesResult result = ec2client.runInstances(request);
+ try {
+ final RunInstancesResult result = ec2client.runInstances(request);
- for (Instance i : result.getReservation().getInstances()) {
- instanceIDs.add(i.getInstanceId());
+ for (Instance i : result.getReservation().getInstances()) {
+ instanceIDs.add(i.getInstanceId());
+ }
+ } catch (AmazonClientException e) {
+ // Only log the error here
+ LOG.error(StringUtils.stringifyException(e));
}
return instanceIDs;
@@ -655,20 +774,31 @@ private LinkedList allocateCloudInstance(final Configuration conf, final
*
* @param conf
* The configuration provided upon instances request
+ * @throws InstanceException
+ * thrown if an error occurs while communicating with Amazon EC2
*/
- private void checkAndConvertOrphanedInstances(final Configuration conf) {
- if (this.orphanedTMs.size() == 0) {
+ private void checkAndConvertOrphanedInstances(final Configuration conf) throws InstanceException {
+
+ if (this.orphanedInstances.size() == 0) {
return;
}
- final String awsAccessId = conf.getString("job.cloud.awsaccessid", null);
- final String awsSecretKey = conf.getString("job.cloud.awssecretkey", null);
+ final String awsAccessId = conf.getString(AWS_ACCESS_ID_KEY, null);
+ final String awsSecretKey = conf.getString(AWS_SECRET_KEY_KEY, null);
- LOG.debug("Checking orphaned Instances... " + this.orphanedTMs.size() + " orphaned instances listed.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking orphaned instances, " + this.orphanedInstances.size() + " orphaned instances listed.");
+ }
final AmazonEC2Client ec2client = EC2ClientFactory.getEC2Client(awsAccessId, awsSecretKey);
- final DescribeInstancesRequest request = new DescribeInstancesRequest();
- final DescribeInstancesResult result = ec2client.describeInstances(request);
+ DescribeInstancesResult result = null;
+
+ try {
+ final DescribeInstancesRequest request = new DescribeInstancesRequest();
+ result = ec2client.describeInstances(request);
+ } catch (AmazonClientException e) {
+ throw new InstanceException(StringUtils.stringifyException(e));
+ }
// Iterate over all Instances
for (final Reservation r : result.getReservations()) {
@@ -694,23 +824,27 @@ private void checkAndConvertOrphanedInstances(final Configuration conf) {
continue;
}
- final Iterator it = this.orphanedTMs.iterator();
+ final Iterator> it = this.orphanedInstances
+ .entrySet().iterator();
while (it.hasNext()) {
- final InstanceConnectionInfo oi = it.next();
+ final Map.Entry entry = it.next();
+
+ final InstanceConnectionInfo oi = entry.getKey();
+ final HardwareDescription hd = entry.getValue();
+
if (oi.getAddress().equals(inetAddress) && type != null) {
- LOG.info("Orphaned Instance " + oi + " converted into floating instance.");
+ LOG.info("Orphaned instance " + oi + " converted into floating instance.");
- // We have found the corresponding orphaned TM.. take the poor lamb back to its nest.
- FloatingInstance floatinginstance = new FloatingInstance(t.getInstanceId(), oi, t
- .getLaunchTime().getTime(), type, awsAccessId, awsSecretKey);
+ // We have found the corresponding orphaned TM.. convert it back to a floating instance.
+ final FloatingInstance floatinginstance = new FloatingInstance(t.getInstanceId(), oi, t
+ .getLaunchTime().getTime(), this.leasePeriod, type, hd, awsAccessId, awsSecretKey);
this.floatingInstances.put(oi, floatinginstance);
it.remove();
break;
}
}
-
}
}
}
@@ -732,7 +866,7 @@ private void checkAndConvertOrphanedInstances(final Configuration conf) {
private LinkedList anyFloatingInstancesAvailable(final String awsAccessId,
final String awsSecretKey, final InstanceType type, final int count) throws InstanceException {
- LOG.info("Check for floating instance of type" + type.getIdentifier() + " requested count: " + count + ".");
+ LOG.info("Checking for up to " + count + " floating instance of type " + type.getIdentifier());
final LinkedList foundfloatinginstances = new LinkedList();
@@ -774,14 +908,13 @@ public void run() {
while (it.hasNext()) {
final Map.Entry entry = it.next();
- // Call lifecycle method for each floating instance. If true, remove from floatinginstances list.
- if (entry.getValue().checkIfLifeCycleEnded()) {
+ // Call life cycle method for each floating instance. If true, remove from floating instances list.
+ if (entry.getValue().hasLifeCycleEnded()) {
it.remove();
- LOG.info("Floating Instance " + entry.getValue().getInstanceID()
- + " ended its lifecycle. Terminated");
+ LOG.info("Lifecycle of floating instance " + entry.getValue().getInstanceID()
+ + " has ended, terminating instance...");
}
}
-
}
}
@@ -793,12 +926,7 @@ public void run() {
@Override
public InstanceType getDefaultInstanceType() {
- final String instanceIdentifier = GlobalConfiguration.getString("cloudmgr.instancetype.defaultInstance", null);
- if (instanceIdentifier == null) {
- return null;
- }
-
- return getInstanceTypeByName(instanceIdentifier);
+ return this.defaultInstanceType;
}
/**
@@ -832,17 +960,19 @@ public void shutdown() {
}
@Override
- public NetworkTopology getNetworkTopology(JobID jobID) {
+ public NetworkTopology getNetworkTopology(final JobID jobID) {
- // TODO: Make topology job specific
- return this.networkTopology;
+ synchronized (this.networkTopologies) {
+
+ return this.networkTopologies.get(jobID);
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public void setInstanceListener(InstanceListener instanceListener) {
+ public void setInstanceListener(final InstanceListener instanceListener) {
this.instanceListener = instanceListener;
}
@@ -856,7 +986,10 @@ public Map getMapOfAvailableInstanceTypes
final Map availableinstances = new SerializableHashMap();
for (final InstanceType t : this.availableInstanceTypes) {
- availableinstances.put(t, InstanceTypeDescriptionFactory.construct(t, estimateHardwareDescription(t), -1));
+ // TODO: Number of available instances is set to 1000 to improve interaction with PACT layer, must be -1
+ // actually according to API
+ availableinstances
+ .put(t, InstanceTypeDescriptionFactory.construct(t, estimateHardwareDescription(t), 1000));
}
return availableinstances;
diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2Utilities.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2Utilities.java
similarity index 97%
rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2Utilities.java
rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2Utilities.java
index 6a305f4d6b8c1..c3ea5dd8776ff 100644
--- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2Utilities.java
+++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2Utilities.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
/**
* This class provides auxiliary methods needed to set up the custom EC2-image.
diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/FloatingInstance.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/FloatingInstance.java
similarity index 67%
rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/FloatingInstance.java
rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/FloatingInstance.java
index 6e50256d62158..1abe00ac010a0 100644
--- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/FloatingInstance.java
+++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/FloatingInstance.java
@@ -13,15 +13,17 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import java.util.LinkedList;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
+import eu.stratosphere.nephele.instance.HardwareDescription;
import eu.stratosphere.nephele.instance.InstanceConnectionInfo;
import eu.stratosphere.nephele.instance.InstanceType;
+import eu.stratosphere.nephele.topology.NetworkNode;
/**
* A FloatingInstance is an instance in the cloud allocated for a user. It is idle and carries out no task.
@@ -29,11 +31,15 @@
*/
class FloatingInstance {
- /** The user pays fee for his instances every time unit. */
- private static final long TIMEUNIT = 60 * 60 * 1000; // 1 hour in ms.
+ /**
+ * Time limit to full next hour when instance is terminate.
+ **/
+ private static final long TIME_THRESHOLD = 2 * 60 * 1000; // 2 mins in ms.
- /** Timelimit to full next hour when instance is kicked. */
- private static final long TIMETHRESHOLD = 2 * 60 * 1000; // 2 mins in ms.
+ /**
+ * The lease period for this instance of Amazon EC2 in milliseconds.
+ */
+ private final long leasePeriod;
/** The instance ID. */
private final String instanceID;
@@ -50,9 +56,14 @@ class FloatingInstance {
/** The AWS Secret Key to access this machine */
private String awsSecretKey;
- /** The instance Type */
+ /** The instance type */
private InstanceType type;
+ /**
+ * The instance's hardware description.
+ */
+ private final HardwareDescription hardwareDescription;
+
/** The last received heart beat. */
private long lastHeartBeat;
@@ -64,7 +75,9 @@ class FloatingInstance {
* @param instanceConnectionInfo
* the information required to connect to the instance's task manager
* @param launchTime
- * the time the instance was allocated
+ * the time the instance was allocated in milliseconds since January 1st, 1970
+ * @param leasePeriod
+ * the lease period for this floating instances in milliseconds
* @param type
* The type of this instance.
* @param awsAccessKey
@@ -72,15 +85,29 @@ class FloatingInstance {
* @param awsSecretKey
* The AWS Secret Key to access this machine
*/
- public FloatingInstance(String instanceID, InstanceConnectionInfo instanceConnectionInfo, long launchTime,
- InstanceType type, String awsAccessKey, String awsSecretKey) {
+ public FloatingInstance(final String instanceID, final InstanceConnectionInfo instanceConnectionInfo,
+ final long launchTime,
+ final long leasePeriod, final InstanceType type, final HardwareDescription hardwareDescription,
+ final String awsAccessKey, final String awsSecretKey) {
+
+ if (launchTime < 0) {
+ throw new IllegalArgumentException("Argument launchTime must be greater than 0");
+ }
+
+ if (leasePeriod <= 0) {
+ throw new IllegalArgumentException("Argument leasePeriod be greater than 0");
+ }
+
this.instanceID = instanceID;
this.instanceConnectionInfo = instanceConnectionInfo;
this.launchTime = launchTime;
+ this.leasePeriod = leasePeriod;
this.lastHeartBeat = System.currentTimeMillis();
this.awsAccessKey = awsAccessKey;
this.awsSecretKey = awsSecretKey;
this.type = type;
+ this.hardwareDescription = hardwareDescription;
+
}
/**
@@ -155,22 +182,24 @@ public long getLaunchTime() {
*
* @return
*/
- public CloudInstance asCloudInstance() {
- return new CloudInstance(this.instanceID, this.type, this.getInstanceConnectionInfo(), this.launchTime, null,
- null, null, this.awsAccessKey, this.awsSecretKey);
+ public EC2CloudInstance asCloudInstance(final NetworkNode parentNode) {
+
+ return new EC2CloudInstance(this.instanceID, this.type, this.getInstanceConnectionInfo(), this.launchTime,
+ this.leasePeriod, parentNode, parentNode.getNetworkTopology(), this.hardwareDescription, this.awsAccessKey,
+ this.awsSecretKey);
}
/**
- * This method checks, if this floating instance has reached the end of its lifecycle and - if so - terminates
+ * This method checks if this floating instance has reached the end of its life cycle and, if so, terminates
* itself.
*/
- public boolean checkIfLifeCycleEnded() {
+ public boolean hasLifeCycleEnded() {
final long currentTime = System.currentTimeMillis();
- final long msremaining = TIMEUNIT - ((currentTime - this.launchTime) % TIMEUNIT);
+ final long msremaining = this.leasePeriod - ((currentTime - this.launchTime) % this.leasePeriod);
- if (msremaining < TIMETHRESHOLD) {
- // Destroy this instance.
+ if (msremaining < TIME_THRESHOLD) {
+ // Destroy this instance
final AmazonEC2Client client = EC2ClientFactory.getEC2Client(this.awsAccessKey, this.awsSecretKey);
final TerminateInstancesRequest tr = new TerminateInstancesRequest();
final LinkedList instanceIDlist = new LinkedList();
@@ -178,9 +207,18 @@ public boolean checkIfLifeCycleEnded() {
tr.setInstanceIds(instanceIDlist);
client.terminateInstances(tr);
return true;
- } else {
- return false;
}
+
+ return false;
}
+ /**
+ * Returns the hardware description of the floating instance.
+ *
+ * @return the hardware description of the floating instance
+ */
+ public HardwareDescription getHardwareDescription() {
+
+ return this.hardwareDescription;
+ }
}
diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMapping.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMapping.java
similarity index 87%
rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMapping.java
rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMapping.java
index 4e4d2a8e96ce9..272a24f6aacbd 100644
--- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMapping.java
+++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMapping.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import java.util.ArrayList;
import java.util.Iterator;
@@ -28,7 +28,7 @@
final class JobToInstancesMapping {
/** The list of assigned cloud instances for the job. */
- private final List assignedInstances = new ArrayList();
+ private final List assignedInstances = new ArrayList();
/** The access ID into Amazon Web Services. */
private final String awsAccessId;
@@ -56,7 +56,7 @@ public JobToInstancesMapping(final String awsAccessId, final String awsSecretKey
* @param instance
* the cloud instance which will be assigned
*/
- public void assignInstanceToJob(final CloudInstance instance) {
+ public void assignInstanceToJob(final EC2CloudInstance instance) {
synchronized (this.assignedInstances) {
this.assignedInstances.add(instance);
@@ -70,7 +70,7 @@ public void assignInstanceToJob(final CloudInstance instance) {
* the cloud instance which will be unassigned
* @return the unassigned cloud instance
*/
- public boolean unassignInstanceFromJob(CloudInstance instance) {
+ public boolean unassignInstanceFromJob(EC2CloudInstance instance) {
synchronized (this.assignedInstances) {
return this.assignedInstances.remove(instance);
@@ -91,7 +91,7 @@ public int getNumberOfAssignedInstances() {
*
* @return the list of assigned cloud instances for the job
*/
- public List getAssignedInstances() {
+ public List getAssignedInstances() {
return this.assignedInstances;
}
@@ -103,7 +103,7 @@ public List getAssignedInstances() {
* @return the cloud instance matching the given connection information or null
if no matching instance
* exists
*/
- public CloudInstance getInstanceByConnectionInfo(final InstanceConnectionInfo instanceConnectionInfo) {
+ public EC2CloudInstance getInstanceByConnectionInfo(final InstanceConnectionInfo instanceConnectionInfo) {
if (instanceConnectionInfo == null) {
return null;
@@ -111,11 +111,11 @@ public CloudInstance getInstanceByConnectionInfo(final InstanceConnectionInfo in
synchronized (this.assignedInstances) {
- final Iterator it = this.assignedInstances.iterator();
+ final Iterator it = this.assignedInstances.iterator();
while (it.hasNext()) {
- final CloudInstance ci = it.next();
+ final EC2CloudInstance ci = it.next();
if (instanceConnectionInfo.equals(ci.getInstanceConnectionInfo())) {
return ci;
}
diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceTest.java
similarity index 77%
rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceTest.java
rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceTest.java
index e67f0e36a517b..84712622c81e5 100644
--- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceTest.java
+++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceTest.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import static org.junit.Assert.*;
@@ -25,28 +25,30 @@
import eu.stratosphere.nephele.instance.HardwareDescriptionFactory;
import eu.stratosphere.nephele.instance.InstanceConnectionInfo;
import eu.stratosphere.nephele.instance.InstanceTypeFactory;
-import eu.stratosphere.nephele.instance.cloud.CloudInstance;
+import eu.stratosphere.nephele.instance.ec2.EC2CloudInstance;
import eu.stratosphere.nephele.topology.NetworkTopology;
-public class CloudInstanceTest {
+public class EC2CloudInstanceTest {
+
+ private EC2CloudInstance constructSmallCloudInstance() {
- private CloudInstance constructSmallCloudInstance() {
-
final NetworkTopology networkTopology = NetworkTopology.createEmptyTopology();
- final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L*1024L*1024L, 2048L*1024L*1024L);
-
- final CloudInstance cloudInstance = new CloudInstance("i-1234ABCD",
+ final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L * 1024L * 1024L,
+ 2048L * 1024L * 1024L);
+
+ final EC2CloudInstance cloudInstance = new EC2CloudInstance("i-1234ABCD",
InstanceTypeFactory.constructFromDescription("m1.small,1,1,2048,40,10"),
new InstanceConnectionInfo(new InetSocketAddress("localhost", 6122).getAddress(), 6122, 6121),
- 1234567890, networkTopology.getRootNode(), networkTopology, hardwareDescription, null, null);
-
+ 1234567890, EC2CloudManager.DEFAULT_LEASE_PERIOD, networkTopology.getRootNode(), networkTopology,
+ hardwareDescription, null, null);
+
return cloudInstance;
}
-
+
@Test
public void testHeartBeat() {
- final CloudInstance ci = constructSmallCloudInstance();
+ final EC2CloudInstance ci = constructSmallCloudInstance();
long lastHeartBeat = ci.getLastReceivedHeartBeat();
try {
@@ -58,4 +60,4 @@ public void testHeartBeat() {
assertTrue(ci.getLastReceivedHeartBeat() - lastHeartBeat > 0);
}
-}
+}
\ No newline at end of file
diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudManagerTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManagerTest.java
similarity index 82%
rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudManagerTest.java
rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManagerTest.java
index c1573c3139fb7..a253f0e528847 100644
--- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudManagerTest.java
+++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManagerTest.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import static org.junit.Assert.*;
@@ -49,13 +49,13 @@
import eu.stratosphere.nephele.instance.InstanceRequestMap;
import eu.stratosphere.nephele.instance.InstanceType;
import eu.stratosphere.nephele.instance.InstanceTypeFactory;
-import eu.stratosphere.nephele.instance.cloud.CloudInstance;
-import eu.stratosphere.nephele.instance.cloud.CloudManager;
-import eu.stratosphere.nephele.instance.cloud.FloatingInstance;
-import eu.stratosphere.nephele.instance.cloud.JobToInstancesMapping;
+import eu.stratosphere.nephele.instance.ec2.EC2CloudInstance;
+import eu.stratosphere.nephele.instance.ec2.EC2CloudManager;
+import eu.stratosphere.nephele.instance.ec2.FloatingInstance;
+import eu.stratosphere.nephele.instance.ec2.JobToInstancesMapping;
import eu.stratosphere.nephele.jobgraph.JobID;
-public class CloudManagerTest {
+public class EC2CloudManagerTest {
private static final class MyInstanceListener implements InstanceListener {
@@ -68,12 +68,8 @@ public void allocatedResourcesDied(final JobID jobID, final List resourcesOfJob = this.resourcesOfJobs.get(jobID);
assertTrue(resourcesOfJob != null);
-
- for (final AllocatedResource allocatedResource : allocatedResources) {
- assertTrue(resourcesOfJob.contains(allocatedResource));
- resourcesOfJob.remove(allocatedResource);
- }
-
+ assertTrue(resourcesOfJob.contains(allocatedResource));
+ resourcesOfJob.remove(allocatedResource);
if (resourcesOfJob.isEmpty()) {
this.resourcesOfJobs.remove(jobID);
}
@@ -102,13 +98,12 @@ public void testLoadConf() {
GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf");
- assertEquals(5, GlobalConfiguration.getInteger("cloudmgr.nrtypes", -1));
- assertEquals("m1.small,1,1,2048,40,10", GlobalConfiguration.getString("cloudmgr.instancetype.1", null));
- assertEquals("c1.medium,2,2,4096,80,20", GlobalConfiguration.getString("cloudmgr.instancetype.2", null));
- assertEquals("m1.large,4,4,6144,160,40", GlobalConfiguration.getString("cloudmgr.instancetype.3", null));
- assertEquals("m1.xlarge,4,4,12288,160,60", GlobalConfiguration.getString("cloudmgr.instancetype.4", null));
- assertEquals("c1.xlarge,8,8,28672,280,80", GlobalConfiguration.getString("cloudmgr.instancetype.5", null));
- assertEquals("m1.small", GlobalConfiguration.getString("cloudmgr.instancetype.defaultInstance", null));
+ assertEquals("m1.small,1,1,2048,40,10", GlobalConfiguration.getString("instancemanager.ec2.type.1", null));
+ assertEquals("c1.medium,2,2,4096,80,20", GlobalConfiguration.getString("instancemanager.ec2.type.2", null));
+ assertEquals("m1.large,4,4,6144,160,40", GlobalConfiguration.getString("instancemanager.ec2.type.3", null));
+ assertEquals("m1.xlarge,4,4,12288,160,60", GlobalConfiguration.getString("instancemanager.ec2.type.4", null));
+ assertEquals("c1.xlarge,8,8,28672,280,80", GlobalConfiguration.getString("instancemanager.ec2.type.5", null));
+ assertEquals(1, GlobalConfiguration.getInteger("instancemanager.ec2.defaulttype", -1));
}
@Test
@@ -133,7 +128,7 @@ public void testGetDefaultInstance() {
GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf");
MyInstanceListener myInstanceListener = new MyInstanceListener();
- CloudManager cm = new CloudManager();
+ EC2CloudManager cm = new EC2CloudManager();
cm.setInstanceListener(myInstanceListener);
InstanceType defaultIT = cm.getDefaultInstanceType();
@@ -153,7 +148,7 @@ public void testGetSuitableInstanceType() {
GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf");
MyInstanceListener myInstanceListener = new MyInstanceListener();
- CloudManager cm = new CloudManager();
+ EC2CloudManager cm = new EC2CloudManager();
cm.setInstanceListener(myInstanceListener);
InstanceType type1 = cm.getSuitableInstanceType(16, 16, 2048, 40, 80);
@@ -185,7 +180,7 @@ public void testGetInstanceTypeByName() {
GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf");
MyInstanceListener myInstanceListener = new MyInstanceListener();
- CloudManager cm = new CloudManager();
+ EC2CloudManager cm = new EC2CloudManager();
cm.setInstanceListener(myInstanceListener);
InstanceType type = cm.getInstanceTypeByName("m1.small");
@@ -205,7 +200,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf");
MyInstanceListener myInstanceListener = new MyInstanceListener();
- CloudManager cm = new CloudManager();
+ EC2CloudManager cm = new EC2CloudManager();
cm.setInstanceListener(myInstanceListener);
JobID jobID = new JobID();
@@ -216,7 +211,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
if (!f.exists()) {
System.err.println("Please create an XML file \"ec2-account.xml\" for EC2 account in the folder "
+ System.getProperty("user.dir") + "/correct-conf\n"
- + "Three keys must be included: job.cloud.username, job.cloud.awsaccessid, job.cloud.awssecretkey\n"
+ + "Three keys must be included: , job.ec2.awsaccessid, job.ec2.awssecretkey\n"
+ "The format is:\n" + "\n" + " ...\n" + " ...\n" + "");
return;
}
@@ -245,21 +240,16 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
e.printStackTrace();
}
- // check whether all three keys are included
- if (conf.getString("job.cloud.username", null) == null) {
- System.err.println("Please set the key job.cloud.username in " + System.getProperty("user.dir")
- + "/correct-conf/ec2-account.xml");
- return;
- }
-
- if (conf.getString("job.cloud.awsaccessid", null) == null) {
- System.err.println("Please set the key job.cloud.awsaccessid in " + System.getProperty("user.dir")
+ if (conf.getString(EC2CloudManager.AWS_ACCESS_ID_KEY, null) == null) {
+ System.err.println("Please set the key " + EC2CloudManager.AWS_ACCESS_ID_KEY + " in "
+ + System.getProperty("user.dir")
+ "/correct-conf/ec2-account.xml");
return;
}
- if (conf.getString("job.cloud.awssecretkey", null) == null) {
- System.err.println("Please set the key job.cloud.awssecretkey in " + System.getProperty("user.dir")
+ if (conf.getString(EC2CloudManager.AWS_SECRET_KEY_KEY, null) == null) {
+ System.err.println("Please set the key " + EC2CloudManager.AWS_SECRET_KEY_KEY + " in "
+ + System.getProperty("user.dir")
+ "/correct-conf/ec2-account.xml");
return;
}
@@ -271,23 +261,23 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
Object jobToInstancesMap = new Object();
try {
- Field f1 = CloudManager.class.getDeclaredField("reservedInstances");
+ Field f1 = EC2CloudManager.class.getDeclaredField("reservedInstances");
f1.setAccessible(true);
reservedInstances = f1.get(cm);
- Field f2 = CloudManager.class.getDeclaredField("cloudInstances");
+ Field f2 = EC2CloudManager.class.getDeclaredField("cloudInstances");
f2.setAccessible(true);
cloudInstances = f2.get(cm);
- Field f3 = CloudManager.class.getDeclaredField("floatingInstances");
+ Field f3 = EC2CloudManager.class.getDeclaredField("floatingInstances");
f3.setAccessible(true);
floatingInstances = f3.get(cm);
- Field f4 = CloudManager.class.getDeclaredField("floatingInstanceIDs");
+ Field f4 = EC2CloudManager.class.getDeclaredField("floatingInstanceIDs");
f4.setAccessible(true);
floatingInstanceIDs = f4.get(cm);
- Field f5 = CloudManager.class.getDeclaredField("jobToInstancesMap");
+ Field f5 = EC2CloudManager.class.getDeclaredField("jobToInstancesMap");
f5.setAccessible(true);
jobToInstancesMap = f5.get(cm);
@@ -302,7 +292,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
}
assertEquals(0, ((Map) reservedInstances).size());
- assertEquals(0, ((List) cloudInstances).size());
+ assertEquals(0, ((List) cloudInstances).size());
assertEquals(0, ((Map) floatingInstances).size());
assertEquals(0, ((Map) floatingInstanceIDs).size());
assertEquals(0, ((Map) jobToInstancesMap).size());
@@ -319,7 +309,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
}
assertEquals(1, ((Map) reservedInstances).size());
- assertEquals(0, ((List) cloudInstances).size());
+ assertEquals(0, ((List) cloudInstances).size());
assertEquals(0, ((Map) floatingInstances).size());
assertEquals(0, ((Map) floatingInstanceIDs).size());
assertEquals(1, ((Map) jobToInstancesMap).size());
@@ -334,7 +324,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
* String.class, String.class });
* m1.setAccessible(true);
* Object instanceList = m1.invoke(cm, new Object[] { conf.getString("job.cloud.username", null),
- * conf.getString("job.cloud.awsaccessid", null), conf.getString("job.cloud.awssecretkey", null) });
+ * conf.getString("job.ec2.awsaccessid", null), conf.getString("job.ec2.awssecretkey", null) });
* assertEquals(1, ((List) instanceList).size());
* com.xerox.amazonws.ec2.ReservationDescription.Instance instance =
* ((List) instanceList)
@@ -352,17 +342,17 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
}
assertEquals(0, ((Map) reservedInstances).size());
- assertEquals(1, ((List) cloudInstances).size());
+ assertEquals(1, ((List) cloudInstances).size());
assertEquals(0, ((Map) floatingInstances).size());
assertEquals(0, ((Map) floatingInstanceIDs).size());
assertEquals(1, ((Map) jobToInstancesMap).size());
// release instance
- CloudInstance ci = ((List) cloudInstances).get(0);
+ EC2CloudInstance ci = ((List) cloudInstances).get(0);
cm.releaseAllocatedResource(jobID, conf, ci.asAllocatedResource());
assertEquals(0, ((Map) reservedInstances).size());
- assertEquals(0, ((List) cloudInstances).size());
+ assertEquals(0, ((List) cloudInstances).size());
assertEquals(1, ((Map) floatingInstances).size());
assertEquals(1, ((Map) floatingInstanceIDs).size());
assertEquals(1, ((Map) jobToInstancesMap).size());
@@ -371,7 +361,8 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() {
assertNotNull(instanceID);
try {
- Method m2 = CloudManager.class.getDeclaredMethod("destroyCloudInstance", new Class[] { Configuration.class,
+ Method m2 = EC2CloudManager.class.getDeclaredMethod("destroyCloudInstance", new Class[] {
+ Configuration.class,
String.class });
m2.setAccessible(true);
Object terminatedID = m2.invoke(cm, new Object[] { conf, instanceID });
diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/FloatingInstanceTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/FloatingInstanceTest.java
similarity index 89%
rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/FloatingInstanceTest.java
rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/FloatingInstanceTest.java
index 522fdc0769cf1..222d5ea59db8d 100644
--- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/FloatingInstanceTest.java
+++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/FloatingInstanceTest.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import static org.junit.Assert.*;
@@ -22,7 +22,7 @@
import org.junit.Test;
import eu.stratosphere.nephele.instance.InstanceConnectionInfo;
-import eu.stratosphere.nephele.instance.cloud.FloatingInstance;
+import eu.stratosphere.nephele.instance.ec2.FloatingInstance;
public class FloatingInstanceTest {
@@ -30,7 +30,8 @@ public class FloatingInstanceTest {
public void testHeartBeat() {
FloatingInstance fi = new FloatingInstance("i-1234ABCD", new InstanceConnectionInfo(new InetSocketAddress(
- "localhost", 6122).getAddress(), 6122, 6121), System.currentTimeMillis(),null, null, null);
+ "localhost", 6122).getAddress(), 6122, 6121), System.currentTimeMillis(),
+ EC2CloudManager.DEFAULT_LEASE_PERIOD, null, null, null, null);
long lastHeartBeat = fi.getLastReceivedHeartBeat();
try {
@@ -42,4 +43,4 @@ public void testHeartBeat() {
assertTrue(fi.getLastReceivedHeartBeat() - lastHeartBeat > 0);
}
-}
+}
\ No newline at end of file
diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMappingTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMappingTest.java
similarity index 76%
rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMappingTest.java
rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMappingTest.java
index e1b415937a39d..0c8ef04616a87 100644
--- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMappingTest.java
+++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMappingTest.java
@@ -13,7 +13,7 @@
*
**********************************************************************************************************************/
-package eu.stratosphere.nephele.instance.cloud;
+package eu.stratosphere.nephele.instance.ec2;
import static org.junit.Assert.*;
@@ -26,8 +26,8 @@
import eu.stratosphere.nephele.instance.HardwareDescriptionFactory;
import eu.stratosphere.nephele.instance.InstanceConnectionInfo;
import eu.stratosphere.nephele.instance.InstanceTypeFactory;
-import eu.stratosphere.nephele.instance.cloud.CloudInstance;
-import eu.stratosphere.nephele.instance.cloud.JobToInstancesMapping;
+import eu.stratosphere.nephele.instance.ec2.EC2CloudInstance;
+import eu.stratosphere.nephele.instance.ec2.JobToInstancesMapping;
import eu.stratosphere.nephele.topology.NetworkTopology;
public class JobToInstancesMappingTest {
@@ -36,15 +36,18 @@ public class JobToInstancesMappingTest {
public void testAssignedInstances() {
final NetworkTopology networkTopology = NetworkTopology.createEmptyTopology();
- final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L*1024L*1024L, 2048L*1024L*1024L);
-
+ final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L * 1024L * 1024L,
+ 2048L * 1024L * 1024L);
+
JobToInstancesMapping map = new JobToInstancesMapping("1234567", "abcdefg");
- CloudInstance ci = new CloudInstance("i-1234ABCD", InstanceTypeFactory.constructFromDescription("m1.small,1,1,2048,40,10"),
+ EC2CloudInstance ci = new EC2CloudInstance("i-1234ABCD",
+ InstanceTypeFactory.constructFromDescription("m1.small,1,1,2048,40,10"),
new InstanceConnectionInfo(new InetSocketAddress("localhost", 6122).getAddress(), 6122, 6121), 1234567890,
- networkTopology.getRootNode(), networkTopology, hardwareDescription, null, null);
+ EC2CloudManager.DEFAULT_LEASE_PERIOD, networkTopology.getRootNode(), networkTopology, hardwareDescription,
+ null, null);
assertEquals(0, map.getNumberOfAssignedInstances());
- assertEquals(new ArrayList(), map.getAssignedInstances());
+ assertEquals(new ArrayList(), map.getAssignedInstances());
map.assignInstanceToJob(ci);
@@ -57,6 +60,6 @@ public void testAssignedInstances() {
map.unassignInstanceFromJob(ci);
assertEquals(0, map.getNumberOfAssignedInstances());
- assertEquals(new ArrayList(), map.getAssignedInstances());
+ assertEquals(new ArrayList(), map.getAssignedInstances());
}
-}
+}
\ No newline at end of file
diff --git a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml
index 64844a603c3be..56643ea0f2bdd 100644
--- a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml
+++ b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml
@@ -9,23 +9,15 @@
6123
- jobmanager.instancemanager.classname
- de.tu_berlin.cit.nephele.instance.cloud.CloudManager
+ jobmanager.instancemanager.cloud.classname
+ eu.stratosphere.nephele.instance.ec2.EC2CloudManager
- cloud.ec2ws.secure
- false
+ instancemanager.ec2.endpoint
+ cloud01.cit.tu-berlin.de:8773
- cloud.ec2ws.server
- cloud01.cit.tu-berlin.de
-
-
- cloud.ec2ws.port
- 8773
-
-
- ec2.image.id
+ job.ec2.ami
emi-E9B11C08
@@ -52,4 +44,4 @@
fs.hdfs.hdfsdefault
/home/wenjun/hadoop-0.19.1/conf/hadoop-default.xml
-
\ No newline at end of file
+
diff --git a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml
index 8c7821003bc59..33be1a19ceff1 100644
--- a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml
+++ b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml
@@ -1,38 +1,32 @@
-
-
- cloudmgr.nrtypes
- 5
-
-
- cloudmgr.instancetype.1
+ instancemanager.ec2.type.1
m1.small,1,1,2048,40,10
- cloudmgr.instancetype.2
+ instancemanager.ec2.type.2
c1.medium,2,2,4096,80,20
- cloudmgr.instancetype.3
+ instancemanager.ec2.type.3
m1.large,4,4,6144,160,40
- cloudmgr.instancetype.4
+ instancemanager.ec2.type.4
m1.xlarge,4,4,12288,160,60
- cloudmgr.instancetype.5
+ instancemanager.ec2.type.5
c1.xlarge,8,8,28672,280,80
- cloudmgr.instancetype.defaultInstance
- m1.small
+ instancemanager.ec2.defaulttype
+ 1
-
\ No newline at end of file
+
diff --git a/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java b/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java
index 044e4980e59b0..c9fb8d62031bc 100644
--- a/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java
+++ b/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java
@@ -159,20 +159,11 @@ public String getIdentifier() {
*/
@Override
public String toString() {
- return this.identifier;
- }
- /**
- * Returns a String representation of this instance in the same form as it is parsed by the
- * {@link #getTypeFromString(java.lang.String)} method.
- *
- * @return A String representation of this instance type.
- */
- public String toStringRepresentation() {
-
final StringBuilder bld = new StringBuilder(32);
bld.append(this.identifier);
- bld.append(',');
+ bld.append(' ');
+ bld.append('(');
bld.append(this.numberOfComputeUnits);
bld.append(',');
bld.append(this.numberOfCores);
@@ -182,6 +173,7 @@ public String toStringRepresentation() {
bld.append(this.diskCapacity);
bld.append(',');
bld.append(this.pricePerHour);
+ bld.append(')');
return bld.toString();
}
diff --git a/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java b/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java
index b0cc78a380947..8b28d6e07019d 100644
--- a/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java
+++ b/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java
@@ -98,10 +98,11 @@ InternalInstanceProfilingData generateProfilingData(long timestamp) throws Profi
private void updateMemoryUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException {
+ BufferedReader in = null;
+
try {
- final FileReader memReader = new FileReader(PROC_MEMINFO);
- final BufferedReader in = new BufferedReader(memReader);
+ in = new BufferedReader(new FileReader(PROC_MEMINFO));
long freeMemory = 0;
long totalMemory = 0;
@@ -145,6 +146,13 @@ private void updateMemoryUtilization(InternalInstanceProfilingData profilingData
} catch (IOException ioe) {
throw new ProfilingException("Error while reading network utilization: "
+ StringUtils.stringifyException(ioe));
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ }
+ }
}
}
@@ -161,9 +169,11 @@ private long extractMemoryValue(String line) throws ProfilingException {
private void updateNetworkUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException {
+ BufferedReader in = null;
+
try {
- final BufferedReader in = new BufferedReader(new FileReader(PROC_NET_DEV));
+ in = new BufferedReader(new FileReader(PROC_NET_DEV));
long receivedSum = 0;
long transmittedSum = 0;
@@ -184,6 +194,7 @@ private void updateNetworkUtilization(InternalInstanceProfilingData profilingDat
}
in.close();
+ in = null;
profilingData.setReceivedBytes(receivedSum - this.lastReceivedBytes);
profilingData.setTransmittedBytes(transmittedSum - this.lastTramsmittedBytes);
@@ -198,20 +209,30 @@ private void updateNetworkUtilization(InternalInstanceProfilingData profilingDat
} catch (NumberFormatException nfe) {
throw new ProfilingException("Error while reading network utilization: "
+ StringUtils.stringifyException(nfe));
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ }
+ }
}
}
private void updateCPUUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException {
+ BufferedReader in = null;
+
try {
- final BufferedReader in = new BufferedReader(new FileReader(PROC_STAT));
+ in = new BufferedReader(new FileReader(PROC_STAT));
final String output = in.readLine();
if (output == null) {
throw new ProfilingException("Cannot read CPU utilization, return value is null");
}
in.close();
+ in = null;
final Matcher cpuMatcher = CPU_PATTERN.matcher(output);
if (!cpuMatcher.matches()) {
@@ -282,6 +303,13 @@ private void updateCPUUtilization(InternalInstanceProfilingData profilingData) t
throw new ProfilingException("Error while reading CPU utilization: " + StringUtils.stringifyException(ioe));
} catch (NumberFormatException nfe) {
throw new ProfilingException("Error while reading CPU utilization: " + StringUtils.stringifyException(nfe));
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ }
+ }
}
}
diff --git a/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java b/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java
index 11bf02a7a1809..24a5aef584943 100644
--- a/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java
+++ b/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java
@@ -152,15 +152,17 @@ public void schedulJob(final ExecutionGraph executionGraph) throws SchedulingExc
// Add job to the job queue (important to add job to queue before requesting instances)
synchronized (this.jobQueue) {
this.jobQueue.add(executionGraph);
- }
- // Request resources for the first stage of the job
- final ExecutionStage executionStage = executionGraph.getCurrentExecutionStage();
- try {
- requestInstances(executionStage);
- } catch (InstanceException e) {
- // TODO: Handle this error correctly
- LOG.error(StringUtils.stringifyException(e));
+ // Request resources for the first stage of the job
+ final ExecutionStage executionStage = executionGraph.getCurrentExecutionStage();
+ try {
+ requestInstances(executionStage);
+ } catch (InstanceException e) {
+ final String exceptionMessage = StringUtils.stringifyException(e);
+ LOG.error(exceptionMessage);
+ this.jobQueue.remove(executionGraph);
+ throw new SchedulingException(exceptionMessage);
+ }
}
}
diff --git a/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java b/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java
index b09c124963082..c0c5c34a76095 100644
--- a/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java
+++ b/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java
@@ -178,7 +178,7 @@ public void testResourceAllocated() throws Exception {
final DeploymentManager deploymentManager = new TestDeploymentManager();
final QueueScheduler toTest = spy(new QueueScheduler(deploymentManager, this.instanceManager));
- final JobID jobid = mock(JobID.class);
+ final JobID jobid = new JobID();
final AllocatedResource resource = mock(AllocatedResource.class);
final List resources = new ArrayList();
resources.add(resource);
diff --git a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java
index f1a3c2acd82e6..2f83ec3224622 100644
--- a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java
+++ b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java
@@ -41,12 +41,12 @@
import eu.stratosphere.nephele.io.channels.ChannelType;
import eu.stratosphere.nephele.io.channels.bytebuffered.NetworkOutputChannel;
import eu.stratosphere.nephele.io.compression.CompressionLevel;
+import eu.stratosphere.nephele.jobgraph.AbstractJobInputVertex;
import eu.stratosphere.nephele.jobgraph.AbstractJobVertex;
import eu.stratosphere.nephele.jobgraph.JobEdge;
import eu.stratosphere.nephele.jobgraph.JobFileOutputVertex;
import eu.stratosphere.nephele.jobgraph.JobGraph;
import eu.stratosphere.nephele.jobgraph.JobID;
-import eu.stratosphere.nephele.jobgraph.JobInputVertex;
import eu.stratosphere.nephele.template.AbstractInputTask;
import eu.stratosphere.nephele.template.AbstractInvokable;
import eu.stratosphere.nephele.template.IllegalConfigurationException;
@@ -546,8 +546,8 @@ private ExecutionVertex createVertex(AbstractJobVertex jobVertex, InstanceManage
try {
ev = new ExecutionVertex(jobVertex.getJobGraph().getJobID(), invokableClass, this,
groupVertex);
- } catch (Exception e) {
- throw new GraphConversionException(StringUtils.stringifyException(e));
+ } catch (Throwable t) {
+ throw new GraphConversionException(StringUtils.stringifyException(t));
}
// Run the configuration check the user has provided for the vertex
@@ -593,7 +593,8 @@ private ExecutionVertex createVertex(AbstractJobVertex jobVertex, InstanceManage
null));
// Register input and output vertices separately
- if (jobVertex instanceof JobInputVertex) {
+ if (jobVertex instanceof AbstractJobInputVertex) {
+
final InputSplit[] inputSplits;
// let the task code compute the input splits
@@ -611,6 +612,13 @@ private ExecutionVertex createVertex(AbstractJobVertex jobVertex, InstanceManage
"BUG: JobInputVertex contained a task class which was not an input task.");
}
+ if (inputSplits == null) {
+ LOG.info("Job input vertex " + jobVertex.getName() + " generated 0 input splits");
+ } else {
+ LOG.info("Job input vertex " + jobVertex.getName() + " generated " + inputSplits.length
+ + " input splits");
+ }
+
// assign input splits
groupVertex.setInputSplits(inputSplits);
}
diff --git a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java
index 4b11d65481fbb..a8a5b73629c3a 100644
--- a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java
+++ b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java
@@ -16,7 +16,6 @@
package eu.stratosphere.nephele.instance.local;
import java.io.File;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
diff --git a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java
index 3a9150dc0414d..1abfe86afd4c1 100644
--- a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java
+++ b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java
@@ -103,125 +103,4 @@ public void schedulJob(ExecutionGraph executionGraph) throws SchedulingException
final ExecutionStage stage = executionGraph.getStage(i);
stage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED);
- final Iterator> it = instanceRequestMap.getMinimumIterator();
- while (it.hasNext()) {
-
- final Map.Entry entry = it.next();
- final InstanceTypeDescription descr = availableInstances.get(entry.getKey());
- if (descr == null) {
- throw new SchedulingException("Unable to schedule job: No instance of type " + entry.getKey()
- + " available");
- }
-
- if (descr.getMaximumNumberOfAvailableInstances() != -1
- && descr.getMaximumNumberOfAvailableInstances() < entry.getValue().intValue()) {
- throw new SchedulingException("Unable to schedule job: " + entry.getValue().intValue()
- + " instances of type " + entry.getKey() + " required, but only "
- + descr.getMaximumNumberOfAvailableInstances() + " are available");
- }
- }
- }
-
- // Subscribe to job status notifications
- executionGraph.registerJobStatusListener(this);
-
- final ExecutionGraphIterator it2 = new ExecutionGraphIterator(executionGraph, true);
- while (it2.hasNext()) {
-
- final ExecutionVertex vertex = it2.next();
- vertex.getEnvironment().registerExecutionListener(new LocalExecutionListener(this, vertex));
- }
-
- // Register the scheduler as an execution stage listener
- executionGraph.registerExecutionStageListener(this);
-
- // Add job to the job queue (important to add job to queue before requesting instances)
- synchronized (this.jobQueue) {
- this.jobQueue.add(executionGraph);
- }
-
- // Request resources for the first stage of the job
- final ExecutionStage executionStage = executionGraph.getCurrentExecutionStage();
- try {
- requestInstances(executionStage);
- } catch (InstanceException e) {
- // TODO: Handle this error correctly
- LOG.error(StringUtils.stringifyException(e));
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public ExecutionGraph getExecutionGraphByID(JobID jobID) {
-
- synchronized (this.jobQueue) {
-
- final Iterator it = this.jobQueue.iterator();
- while (it.hasNext()) {
-
- final ExecutionGraph executionGraph = it.next();
- if (executionGraph.getJobID().equals(jobID)) {
- return executionGraph;
- }
- }
- }
-
- return null;
- }
-
-
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void allocatedResourcesDied(final JobID jobID, final List allocatedResource) {
- // TODO Auto-generated method stub
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void shutdown() {
-
- synchronized (this.jobQueue) {
- this.jobQueue.clear();
- }
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void jobStatusHasChanged(final ExecutionGraph executionGraph, final InternalJobStatus newJobStatus,
- final String optionalMessage) {
-
- if (newJobStatus == InternalJobStatus.FAILED || newJobStatus == InternalJobStatus.FINISHED
- || newJobStatus == InternalJobStatus.CANCELED) {
- removeJobFromSchedule(executionGraph);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void nextExecutionStageEntered(final JobID jobID, final ExecutionStage executionStage) {
-
- // Request new instances if necessary
- try {
- requestInstances(executionStage);
- } catch (InstanceException e) {
- // TODO: Handle this error correctly
- LOG.error(StringUtils.stringifyException(e));
- }
-
- // Deploy the assigned vertices
- deployAssignedVertices(executionStage.getExecutionGraph());
- }
-}
+ final Iterator getInputSplitType()
@Override
public void open(GenericInputSplit split) throws IOException
{
- this.partitionNumber = split.getPartitionNumber();
+ this.partitionNumber = split.getSplitNumber();
}
/* (non-Javadoc)
diff --git a/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java b/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java
index 3741bbe89c47f..648a9e672b583 100644
--- a/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java
+++ b/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java
@@ -27,12 +27,12 @@
import eu.stratosphere.nephele.io.channels.ChannelType;
import eu.stratosphere.nephele.io.compression.CompressionLevel;
import eu.stratosphere.nephele.jobgraph.AbstractJobVertex;
-import eu.stratosphere.nephele.jobgraph.JobGenericInputVertex;
-import eu.stratosphere.nephele.jobgraph.JobGenericOutputVertex;
+import eu.stratosphere.nephele.jobgraph.JobInputVertex;
+import eu.stratosphere.nephele.jobgraph.JobOutputVertex;
import eu.stratosphere.nephele.jobgraph.JobGraph;
import eu.stratosphere.nephele.jobgraph.JobGraphDefinitionException;
import eu.stratosphere.nephele.jobgraph.JobInputVertex;
-import eu.stratosphere.nephele.jobgraph.JobOutputVertex;
+import eu.stratosphere.nephele.jobgraph.AbstractJobOutputVertex;
import eu.stratosphere.nephele.jobgraph.JobTaskVertex;
import eu.stratosphere.pact.common.contract.GenericDataSink;
import eu.stratosphere.pact.common.contract.GenericDataSource;
@@ -579,7 +579,7 @@ private JobInputVertex generateDataSourceVertex(OptimizerNode sourceNode) throws
GenericDataSource, ?> contract = dsn.getPactContract();
// create task vertex
- JobGenericInputVertex sourceVertex = new JobGenericInputVertex(contract.getName(), this.jobGraph);
+ JobInputVertex sourceVertex = new JobInputVertex(contract.getName(), this.jobGraph);
// set task class
sourceVertex.setInputClass(DataSourceTask.class);
@@ -608,13 +608,13 @@ private JobInputVertex generateDataSourceVertex(OptimizerNode sourceNode) throws
* @return
* @throws CompilerException
*/
- private JobOutputVertex generateDataSinkVertex(OptimizerNode sinkNode) throws CompilerException
+ private AbstractJobOutputVertex generateDataSinkVertex(OptimizerNode sinkNode) throws CompilerException
{
DataSinkNode sNode = (DataSinkNode) sinkNode;
GenericDataSink, ?> sinkContract = sNode.getPactContract();
// create task vertex
- JobGenericOutputVertex sinkVertex = new JobGenericOutputVertex(sinkNode.getPactContract().getName(), this.jobGraph);
+ JobOutputVertex sinkVertex = new JobOutputVertex(sinkNode.getPactContract().getName(), this.jobGraph);
// set task class
sinkVertex.setOutputClass(DataSinkTask.class);
diff --git a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java
index 05396b2174fe6..6f99a22f1773e 100644
--- a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java
+++ b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java
@@ -48,17 +48,16 @@
* The TPC-H is a decision support benchmark on relational data.
* Its documentation and the data generator (DBGEN) can be found
* on http://www.tpc.org/tpch/ .This implementation is tested with
- * the DB2 data format.
- * THe PACT program implements a modified version of the query 3 of
+ * the DB2 data format.
+ * THe PACT program implements a modified version of the query 3 of
* the TPC-H benchmark including one join, some filtering and an
* aggregation.
- *
* SELECT l_orderkey, o_shippriority, sum(l_extendedprice) as revenue
- * FROM orders, lineitem
- * WHERE l_orderkey = o_orderkey
- * AND o_orderstatus = "X"
- * AND YEAR(o_orderdate) > Y
- * AND o_orderpriority LIKE "Z%"
+ * FROM orders, lineitem
+ * WHERE l_orderkey = o_orderkey
+ * AND o_orderstatus = "X"
+ * AND YEAR(o_orderdate) > Y
+ * AND o_orderpriority LIKE "Z%"
* GROUP BY l_orderkey, o_shippriority;
*/
public class TPCHQuery3 implements PlanAssembler, PlanAssemblerDescription {
@@ -81,8 +80,10 @@ public N_IntStringPair() {
/**
* Initializes the concatenation of integer and string.
*
- * @param first Integer value for concatenating
- * @param second String value for concatenating
+ * @param first
+ * Integer value for concatenating
+ * @param second
+ * String value for concatenating
*/
public N_IntStringPair(PactInteger first, PactString second) {
super(first, second);
@@ -93,14 +94,14 @@ public N_IntStringPair(PactInteger first, PactString second) {
* Map PACT implements the filter on the orders table. The SameKey
* OutputContract is annotated because the key does not change during
* filtering.
- *
*/
@SameKey
public static class FilterO extends MapStub {
private int yearFilter;
+
private String prioFilter;
-
+
@Override
public void configure(Configuration parameters) {
this.yearFilter = parameters.getInteger("YEAR_FILTER", 1990);
@@ -109,14 +110,12 @@ public void configure(Configuration parameters) {
/**
* Filters the orders table by year, orderstatus and orderpriority
- *
- * o_orderstatus = "X"
- * AND YEAR(o_orderdate) > Y
- * AND o_orderpriority LIKE "Z"
- *
- * Output Schema:
- * Key: ORDERKEY
- * Value: 0:ORDERKEY, 1:SHIPPRIORITY
+ * o_orderstatus = "X"
+ * AND YEAR(o_orderdate) > Y
+ * AND o_orderpriority LIKE "Z"
+ * Output Schema:
+ * Key: ORDERKEY
+ * Value: 0:ORDERKEY, 1:SHIPPRIORITY
*/
@Override
public void map(final PactInteger oKey, final Tuple value, final Collector out) {
@@ -143,22 +142,19 @@ public void map(final PactInteger oKey, final Tuple value, final Collector {
/**
- * Does the projection on the LineItem table
- *
+ * Does the projection on the LineItem table
* Output Schema:
- * Key: ORDERKEY
- * Value: 0:ORDERKEY, 1:EXTENDEDPRICE
+ * Key: ORDERKEY
+ * Value: 0:ORDERKEY, 1:EXTENDEDPRICE
*/
@Override
public void map(PactInteger oKey, Tuple value, Collector out) {
@@ -168,23 +164,20 @@ public void map(PactInteger oKey, Tuple value, Collector out
}
/**
- * Match PACT realizes the join between LineItem and Order table. The
+ * Match PACT realizes the join between LineItem and Order table. The
* SuperKey OutputContract is annotated because the new key is
* built of the keys of the inputs.
- *
*/
@SuperKey
public static class JoinLiO extends MatchStub {
/**
- * Implements the join between LineItem and Order table on the
+ * Implements the join between LineItem and Order table on the
* order key.
- *
* WHERE l_orderkey = o_orderkey
- *
* Output Schema:
- * Key: ORDERKEY, SHIPPRIORITY
- * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICE
+ * Key: ORDERKEY, SHIPPRIORITY
+ * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICE
*/
@Override
public void match(PactInteger oKey, Tuple oVal, Tuple liVal, Collector out) {
@@ -199,24 +192,20 @@ public void match(PactInteger oKey, Tuple oVal, Tuple liVal, Collector {
/**
- * Does the aggregation of the query.
- *
+ * Does the aggregation of the query.
* sum(l_extendedprice) as revenue
* GROUP BY l_orderkey, o_shippriority;
- *
* Output Schema:
- * Key: ORDERKEY
- * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICESUM
- *
+ * Key: ORDERKEY
+ * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICESUM
*/
@Override
public void reduce(N_IntStringPair oKeyShipPrio, Iterator values, Collector out) {
@@ -269,10 +258,16 @@ public void combine(N_IntStringPair oKeyShipPrio, Iterator values, Collec
public Plan getPlan(final String... args) {
// parse program parameters
- int noSubtasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
- String ordersPath = (args.length > 1 ? args[1] : "");
+ int noSubtasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
+ String ordersPath = (args.length > 1 ? args[1] : "");
String lineitemsPath = (args.length > 2 ? args[2] : "");
- String output = (args.length > 3 ? args[3] : "");
+ String output = (args.length > 3 ? args[3] : "");
+
+ // optional parameters to run this job on Amazon EC2
+ String awsAccessID = (args.length > 4 ? args[4] : null);
+ String awsSecretKey = (args.length > 5 ? args[5] : null);
+ String awsImageID = (args.length > 6 ? args[6] : null);
+ String sshKeyPair = (args.length > 7 ? args[7] : null);
// create DataSourceContract for Orders input
FileDataSourceContract orders = new FileDataSourceContract(
@@ -342,8 +337,22 @@ public Plan getPlan(final String... args) {
joinLiO.setSecondInput(projectLi);
projectLi.setInput(lineitems);
+ Plan plan = new Plan(result, "TPCH Q3");
+ if (awsAccessID != null) {
+ plan.getPlanConfiguration().setNepheleString("job.ec2.awsaccessid", awsAccessID);
+ }
+ if (awsSecretKey != null) {
+ plan.getPlanConfiguration().setNepheleString("job.ec2.awssecretkey", awsSecretKey);
+ }
+ if (awsImageID != null) {
+ plan.getPlanConfiguration().setNepheleString("job.ec2.ami", awsImageID);
+ }
+ if (sshKeyPair != null) {
+ plan.getPlanConfiguration().setNepheleString("job.ec2.sshkeypair", sshKeyPair);
+ }
+
// return the PACT plan
- return new Plan(result, "TPCH Q3");
+ return plan;
}
/**
diff --git a/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh b/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh
index 59b5a8d0a2078..4f5fe1a5ae25d 100755
--- a/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh
+++ b/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh
@@ -67,8 +67,6 @@ constructJobManagerClassPath() {
add=1
elif [[ "$jarfile" =~ 'commons-httpclient' ]]; then
add=1
- elif [[ "$jarfile" =~ 'typica' ]]; then
- add=1
elif [[ "$jarfile" =~ 'pact-common' ]]; then
add=1
elif [[ "$jarfile" =~ 'pact-runtime' ]]; then
diff --git a/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml b/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml
index ae6e119b6ec68..7c06fa65bf323 100644
--- a/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml
+++ b/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml
@@ -63,7 +63,28 @@
instancemanager.cluster.defaulttype
1
-
+
+
+
+
+ instancemanager.ec2.type.1
+ t1.micro,2,1,613,8,3
+
+
+
+ instancemanager.ec2.type.2
+ m1.large,4,2,7680,850,38
+
+
+
+ instancemanager.ec2.type.3
+ m1.xlarge,8,4,15360,1690,76
+
+
+
+ instancemanager.ec2.defaulttype
+ 1
+
jobmanager.instancemanager.cloud.classname
- eu.stratosphere.nephele.instance.cloud.CloudManager
+ eu.stratosphere.nephele.instance.ec2.EC2CloudManager