diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java index aa453a0ef644f..3a3aac17fc149 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/AbstractJobResult.java @@ -38,7 +38,16 @@ public abstract class AbstractJobResult implements IOReadableWritable { * @author warneke */ public enum ReturnCode { - SUCCESS, ERROR + + /** + * The success return code. + */ + SUCCESS, + + /** + * The error return code. + */ + ERROR }; /** @@ -156,23 +165,23 @@ public boolean equals(final Object obj) { return true; } - + /** * {@inheritDoc} */ @Override public int hashCode() { - + long hashCode = 0; - - if(this.returnCode != null) { + + if (this.returnCode != null) { hashCode += this.returnCode.hashCode(); } - if(this.description != null) { + if (this.description != null) { hashCode += this.description.hashCode(); } - + return (int) (hashCode % Integer.MAX_VALUE); } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java index ffbb78f174254..f73ccd686ac8d 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobCancelResult.java @@ -33,7 +33,7 @@ public class JobCancelResult extends AbstractJobResult { * @param description * the optional error description */ - public JobCancelResult(ReturnCode returnCode, String description) { + public JobCancelResult(final ReturnCode returnCode, final String description) { super(returnCode, description); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java index 2be224bfd2a3e..b4b2718f742ec 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobClient.java @@ -38,6 +38,7 @@ /** * The job client is able to submit, control, and abort jobs. + *

* This class is thread-safe. * * @author warneke @@ -45,7 +46,7 @@ public class JobClient { /** - * The logging object used for debugging + * The logging object used for debugging. */ private static final Log LOG = LogFactory.getLog(JobClient.class); @@ -94,7 +95,7 @@ public static class JobCleanUp extends Thread { * @param jobClient * the job client this clean up object belongs to */ - public JobCleanUp(JobClient jobClient) { + public JobCleanUp(final JobClient jobClient) { this.jobClient = jobClient; } @@ -134,7 +135,7 @@ public void run() { * @throws IOException * thrown on error while initializing the RPC connection to the job manager */ - public JobClient(JobGraph jobGraph) throws IOException { + public JobClient(final JobGraph jobGraph) throws IOException { this(jobGraph, new Configuration()); } @@ -150,7 +151,7 @@ public JobClient(JobGraph jobGraph) throws IOException { * @throws IOException * thrown on error while initializing the RPC connection to the job manager */ - public JobClient(JobGraph jobGraph, Configuration configuration) throws IOException { + public JobClient(final JobGraph jobGraph, final Configuration configuration) throws IOException { final String address = configuration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null); final int port = configuration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, @@ -163,8 +164,7 @@ public JobClient(JobGraph jobGraph, Configuration configuration) throws IOExcept this.configuration = configuration; this.jobCleanUp = new JobCleanUp(this); } - - + /** * Constructs a new job client object and instantiates a local * RPC proxy for the {@link JobSubmissionProtocol}. @@ -177,16 +177,18 @@ public JobClient(JobGraph jobGraph, Configuration configuration) throws IOExcept * IP/Port of the jobmanager (not taken from provided configuration object). * @throws IOException * thrown on error while initializing the RPC connection to the job manager - */ - public JobClient(JobGraph jobGraph, Configuration configuration, InetSocketAddress jobManagerAddress) throws IOException { - + */ + public JobClient(final JobGraph jobGraph, final Configuration configuration, + final InetSocketAddress jobManagerAddress) + throws IOException { - this.jobSubmitClient = (JobManagementProtocol) RPC.getProxy(JobManagementProtocol.class, jobManagerAddress, NetUtils - .getSocketFactory()); + this.jobSubmitClient = (JobManagementProtocol) RPC.getProxy(JobManagementProtocol.class, jobManagerAddress, + NetUtils + .getSocketFactory()); this.jobGraph = jobGraph; this.configuration = configuration; this.jobCleanUp = new JobCleanUp(this); - } + } /** * Close the JobClient. @@ -328,8 +330,11 @@ public void submitJobAndWait() throws IOException, JobExecutionException { } else if (jobStatus == JobStatus.CANCELED || jobStatus == JobStatus.FAILED) { Runtime.getRuntime().removeShutdownHook(this.jobCleanUp); LOG.info(jobEvent.getOptionalMessage()); - throw new JobExecutionException(jobEvent.getOptionalMessage(), - (jobStatus == JobStatus.CANCELED) ? true : false); + if (jobStatus == JobStatus.CANCELED) { + throw new JobExecutionException(jobEvent.getOptionalMessage(), true); + } else { + throw new JobExecutionException(jobEvent.getOptionalMessage(), false); + } } } } @@ -353,7 +358,7 @@ public void submitJobAndWait() throws IOException, JobExecutionException { * @throws IOException * thrown after the error message is written to the log */ - private void logErrorAndRethrow(String errorMessage) throws IOException { + private void logErrorAndRethrow(final String errorMessage) throws IOException { LOG.error(errorMessage); throw new IOException(errorMessage); @@ -366,7 +371,7 @@ private void logErrorAndRethrow(String errorMessage) throws IOException { * @param sleepTime * the sleep time in milliseconds after which old events shall be removed from the processed event queue */ - private void cleanUpOldEvents(long sleepTime) { + private void cleanUpOldEvents(final long sleepTime) { long mostRecentTimestamp = 0; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java index 6e6cbb86d4130..58f6f0d0c8db5 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobExecutionException.java @@ -28,6 +28,9 @@ public class JobExecutionException extends Exception { */ private static final long serialVersionUID = 2818087325120827525L; + /** + * Indicates whether the job has been aborted as a result of user request. + */ private final boolean canceledByUser; /** diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java index 631f4cc968a52..4104eed9f8072 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobProgressResult.java @@ -46,7 +46,9 @@ public class JobProgressResult extends AbstractJobResult { * @param events * the job events to be transported within this object */ - public JobProgressResult(ReturnCode returnCode, String description, SerializableArrayList events) { + public JobProgressResult(final ReturnCode returnCode, final String description, + final SerializableArrayList events) { + super(returnCode, description); this.events = events; @@ -65,7 +67,7 @@ public JobProgressResult() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { super.read(in); this.events.read(in); @@ -75,7 +77,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { super.write(out); this.events.write(out); @@ -90,36 +92,36 @@ public Iterator getEvents() { return this.events.iterator(); } - + /** * {@inheritDoc} */ @Override - public boolean equals(Object obj) { - - if(!super.equals(obj)) { + public boolean equals(final Object obj) { + + if (!super.equals(obj)) { return false; } - - if(!(obj instanceof JobProgressResult)) { + + if (!(obj instanceof JobProgressResult)) { return false; } - + final JobProgressResult jpr = (JobProgressResult) obj; - - if(!this.events.equals(jpr.events)) { + + if (!this.events.equals(jpr.events)) { return false; } - + return true; } - + /** * {@inheritDoc} */ @Override public int hashCode() { - + return super.hashCode(); } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java index 34d19846cd523..0322b46c4eb04 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/client/JobSubmissionResult.java @@ -37,7 +37,7 @@ public class JobSubmissionResult extends AbstractJobResult { * @param description * the error description */ - public JobSubmissionResult(ReturnCode returnCode, String description) { + public JobSubmissionResult(final ReturnCode returnCode, final String description) { super(returnCode, description); } @@ -54,7 +54,7 @@ public JobSubmissionResult() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { super.read(in); } @@ -62,7 +62,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { super.write(out); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java index 669d7ab480377..700aca1ca3972 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/ConfigConstants.java @@ -77,7 +77,7 @@ public final class ConfigConstants { public static final int DEFAULT_JOB_MANAGER_IPC_PORT = 6123; /** - * The default network port the task manager expects incoming IPC connections + * The default network port the task manager expects incoming IPC connections. */ public static final int DEFAULT_TASK_MANAGER_IPC_PORT = 6122; @@ -97,7 +97,7 @@ public final class ConfigConstants { public static final long DEFAULT_MEMORY_MANAGER_MIN_UNRESERVED_MEMORY = 256 * 1024 * 1024; /** - * The default directory for temporary files of the task manager + * The default directory for temporary files of the task manager. */ public static final String DEFAULT_TASK_MANAGER_TMP_PATH = System.getProperty("java.io.tmpdir"); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java index 1d4365d97ee1f..6283a67dae80a 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/Configuration.java @@ -59,7 +59,7 @@ public class Configuration implements IOReadableWritable { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public String getString(String key, String defaultValue) { + public String getString(final String key, final String defaultValue) { synchronized (this.confData) { @@ -88,10 +88,11 @@ public String getString(String key, String defaultValue) { * @see #setClass(String, Class) */ @SuppressWarnings("unchecked") - public Class getClass(String key, Class defaultValue, Class ancestor) { + public Class getClass(final String key, final Class defaultValue, final Class ancestor) { String className = getString(key, null); - if (className == null) + if (className == null) { return (Class) defaultValue; + } try { return (Class) Class.forName(className); } catch (ClassNotFoundException e) { @@ -111,7 +112,7 @@ public Class getClass(String key, Class defaultValue, Class< * if the class identified by the associated value cannot be resolved * @see #setClass(String, Class) */ - public Class getClass(String key, Class defaultValue) { + public Class getClass(final String key, final Class defaultValue) { return getClass(key, defaultValue, Object.class); } @@ -126,7 +127,7 @@ public Class getClass(String key, Class defaultValue) { * @see #getClass(String, Class) * @see #getClass(String, Class, Class) */ - public void setClass(String key, Class klazz) { + public void setClass(final String key, final Class klazz) { setString(key, klazz.getName()); } @@ -139,7 +140,7 @@ public void setClass(String key, Class klazz) { * @param value * the value of the key/value pair to be added */ - public void setString(String key, String value) { + public void setString(final String key, final String value) { if (key == null || value == null) { // TODO: should probably throw an NullPointerException @@ -161,7 +162,7 @@ public void setString(String key, String value) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public int getInteger(String key, int defaultValue) { + public int getInteger(final String key, final int defaultValue) { int retVal = defaultValue; @@ -188,7 +189,7 @@ public int getInteger(String key, int defaultValue) { * @param value * the value of the key/value pair to be added */ - public void setInteger(String key, int value) { + public void setInteger(final String key, final int value) { if (key == null) { LOG.warn("Cannot set integer: Given key is null!"); @@ -199,7 +200,7 @@ public void setInteger(String key, int value) { this.confData.put(key, Integer.toString(value)); } } - + /** * Returns the value associated with the given key as a long. * @@ -209,7 +210,7 @@ public void setInteger(String key, int value) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public long getLong(String key, long defaultValue) { + public long getLong(final String key, final long defaultValue) { try { synchronized (this.confData) { String val = this.confData.get(key); @@ -217,8 +218,7 @@ public long getLong(String key, long defaultValue) { return Long.parseLong(val); } } - } - catch (NumberFormatException e) { + } catch (NumberFormatException e) { LOG.debug(e); } @@ -234,7 +234,7 @@ public long getLong(String key, long defaultValue) { * @param value * the value of the key/value pair to be added */ - public void setLong(String key, long value) { + public void setLong(final String key, final long value) { if (key == null) { LOG.warn("Cannot set integer: Given key is null!"); return; @@ -254,7 +254,7 @@ public void setLong(String key, long value) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public boolean getBoolean(String key, boolean defaultValue) { + public boolean getBoolean(final String key, final boolean defaultValue) { boolean retVal = defaultValue; @@ -277,7 +277,7 @@ public boolean getBoolean(String key, boolean defaultValue) { * @param value * the value of the key/value pair to be added */ - public void setBoolean(String key, boolean value) { + public void setBoolean(final String key, final boolean value) { if (key == null) { LOG.warn("Cannot set boolean: Given key is null!"); @@ -288,7 +288,7 @@ public void setBoolean(String key, boolean value) { this.confData.put(key, Boolean.toString(value)); } } - + /** * Returns the value associated with the given key as a float. * @@ -298,9 +298,8 @@ public void setBoolean(String key, boolean value) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public float getFloat(String key, float defaultValue) { - synchronized (this.confData) - { + public float getFloat(final String key, final float defaultValue) { + synchronized (this.confData) { String val = this.confData.get(key); return val == null ? defaultValue : Float.parseFloat(val); } @@ -315,7 +314,7 @@ public float getFloat(String key, float defaultValue) { * @param value * the value of the key/value pair to be added */ - public void setFloat(String key, float value) { + public void setFloat(final String key, final float value) { if (key == null) { LOG.warn("Cannot set boolean: Given key is null!"); @@ -353,7 +352,7 @@ public Set keySet() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { synchronized (this.confData) { @@ -371,7 +370,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { synchronized (this.confData) { @@ -396,7 +395,7 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java index 3b2ff9be144b6..c5751f35ecf41 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/configuration/GlobalConfiguration.java @@ -45,7 +45,7 @@ * * @author warneke */ -public class GlobalConfiguration { +public final class GlobalConfiguration { /** * The log object used for debugging. @@ -72,7 +72,7 @@ public class GlobalConfiguration { * * @return the global configuration object */ - private synchronized static GlobalConfiguration get() { + private static synchronized GlobalConfiguration get() { if (configuration == null) { configuration = new GlobalConfiguration(); @@ -96,7 +96,7 @@ private GlobalConfiguration() { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public static String getString(String key, String defaultValue) { + public static String getString(final String key, final String defaultValue) { return get().getStringInternal(key, defaultValue); } @@ -110,7 +110,7 @@ public static String getString(String key, String defaultValue) { * defaultValue the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - private String getStringInternal(String key, String defaultValue) { + private String getStringInternal(final String key, final String defaultValue) { synchronized (this.confData) { @@ -131,7 +131,7 @@ private String getStringInternal(String key, String defaultValue) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public static int getInteger(String key, int defaultValue) { + public static int getInteger(final String key, final int defaultValue) { return get().getIntegerInternal(key, defaultValue); } @@ -145,7 +145,7 @@ public static int getInteger(String key, int defaultValue) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - private int getIntegerInternal(String key, int defaultValue) { + private int getIntegerInternal(final String key, final int defaultValue) { int retVal = defaultValue; @@ -157,7 +157,10 @@ private int getIntegerInternal(String key, int defaultValue) { } } } catch (NumberFormatException e) { - // Nothing to do here + + if (LOG.isDebugEnabled()) { + LOG.debug(StringUtils.stringifyException(e)); + } } return retVal; @@ -172,7 +175,7 @@ private int getIntegerInternal(String key, int defaultValue) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - public static boolean getBoolean(String key, boolean defaultValue) { + public static boolean getBoolean(final String key, final boolean defaultValue) { return get().getBooleanInternal(key, defaultValue); } @@ -186,7 +189,7 @@ public static boolean getBoolean(String key, boolean defaultValue) { * the default value which is returned in case there is no value associated with the given key * @return the (default) value associated with the given key */ - private boolean getBooleanInternal(String key, boolean defaultValue) { + private boolean getBooleanInternal(final String key, final boolean defaultValue) { boolean retVal = defaultValue; @@ -207,7 +210,7 @@ private boolean getBooleanInternal(String key, boolean defaultValue) { * @param configDir * the directory which contains the configuration files */ - public static void loadConfiguration(String configDir) { + public static void loadConfiguration(final String configDir) { if (configDir == null) { @@ -223,9 +226,9 @@ public static void loadConfiguration(String configDir) { } // get all XML files in the directory - File[] files = confDirFile.listFiles(new FilenameFilter() { + final File[] files = confDirFile.listFiles(new FilenameFilter() { @Override - public boolean accept(File dir, String name) { + public boolean accept(final File dir, final String name) { return dir == confDirFile && name != null && name.endsWith(".xml"); } @@ -253,7 +256,7 @@ public boolean accept(File dir, String name) { * @param uri * the URI pointing to the XML document */ - private void loadResource(String uri) { + private void loadResource(final String uri) { final DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); // Ignore comments in the XML file @@ -397,7 +400,7 @@ public static Configuration getConfiguration() { * array of keys specifying the subset of pairs to copy. * @return the {@link Configuration} object including the key/value pairs */ - public static Configuration getConfiguration(String[] keys) { + public static Configuration getConfiguration(final String[] keys) { return get().getConfigurationInternal(keys); } @@ -409,7 +412,7 @@ public static Configuration getConfiguration(String[] keys) { * array of keys specifying the subset of pairs to copy. * @return the {@link Configuration} object including the key/value pairs */ - private Configuration getConfigurationInternal(String[] keys) { + private Configuration getConfigurationInternal(final String[] keys) { Configuration conf = new Configuration(); @@ -450,7 +453,7 @@ private Configuration getConfigurationInternal(String[] keys) { * @param conf * the {@link Configuration} object to merge into the global configuration */ - public static void includeConfiguration(Configuration conf) { + public static void includeConfiguration(final Configuration conf) { get().includeConfigurationInternal(conf); } @@ -461,7 +464,7 @@ public static void includeConfiguration(Configuration conf) { * @param conf * the {@link Configuration} object to merge into the global configuration */ - private void includeConfigurationInternal(Configuration conf) { + private void includeConfigurationInternal(final Configuration conf) { if (conf == null) { LOG.error("Given configuration object is null, ignoring it..."); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java index 2602eabe1c341..53af7b813a44a 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/AbstractEvent.java @@ -32,9 +32,9 @@ public abstract class AbstractEvent implements IOReadableWritable { /** - * Auxiliary object which helps to convert a {@link Date} object to the given string representation + * Auxiliary object which helps to convert a {@link Date} object to the given string representation. */ - private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); + private static final SimpleDateFormat DATA_FORMATTER = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); /** * The timestamp of the event. @@ -47,7 +47,7 @@ public abstract class AbstractEvent implements IOReadableWritable { * @param timestamp * the timestamp of the event. */ - public AbstractEvent(long timestamp) { + public AbstractEvent(final long timestamp) { this.timestamp = timestamp; } @@ -63,7 +63,7 @@ public AbstractEvent() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { // Read the timestamp this.timestamp = in.readLong(); @@ -73,7 +73,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { // Write the timestamp out.writeLong(this.timestamp); @@ -96,9 +96,9 @@ public long getTimestamp() { * the timestamp in milliseconds since the beginning of "the epoch" * @return the string unified representation of the timestamp */ - public static String timestampToString(long timestamp) { + public static String timestampToString(final long timestamp) { - return dateFormatter.format(new Date(timestamp)); + return DATA_FORMATTER.format(new Date(timestamp)); } @@ -106,7 +106,7 @@ public static String timestampToString(long timestamp) { * {@inheritDoc} */ @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (obj instanceof AbstractEvent) { diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java index c414fd7497588..ac6eda30bfd14 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/JobEvent.java @@ -51,7 +51,7 @@ public class JobEvent extends AbstractEvent { * @param optionalMessage * an optional message that shall be attached to this event, possibly null */ - public JobEvent(long timestamp, JobStatus currentJobStatus, String optionalMessage) { + public JobEvent(final long timestamp, final JobStatus currentJobStatus, final String optionalMessage) { super(timestamp); this.currentJobStatus = currentJobStatus; @@ -73,7 +73,7 @@ public JobEvent() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { super.read(in); // Read job status @@ -87,7 +87,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { super.write(out); // Write job status @@ -128,7 +128,7 @@ public String toString() { * {@inheritDoc} */ @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (!super.equals(obj)) { return false; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java index fef5d39a0d28f..45213848d682e 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/job/VertexEvent.java @@ -81,8 +81,9 @@ public class VertexEvent extends AbstractEvent { * @param description * an optional description */ - public VertexEvent(long timestamp, JobVertexID jobVertexID, String jobVertexName, int totalNumberOfSubtasks, - int indexOfSubtask, ExecutionState currentExecutionState, String description) { + public VertexEvent(final long timestamp, final JobVertexID jobVertexID, final String jobVertexName, + final int totalNumberOfSubtasks, final int indexOfSubtask, final ExecutionState currentExecutionState, + final String description) { super(timestamp); this.jobVertexID = jobVertexID; this.jobVertexName = jobVertexName; @@ -112,7 +113,7 @@ public VertexEvent() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { super.read(in); @@ -128,7 +129,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { super.write(out); @@ -210,7 +211,7 @@ public String toString() { * {@inheritDoc} */ @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (!super.equals(obj)) { return false; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java index 169562fa98635..c1fd6d4dc89f5 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/event/task/EventNotificationManager.java @@ -43,7 +43,7 @@ public class EventNotificationManager { * @param eventType * the event type the given listener object wants to be notified about */ - public void subscribeToEvent(EventListener eventListener, Class eventType) { + public void subscribeToEvent(final EventListener eventListener, final Class eventType) { synchronized (this.subscriptions) { @@ -65,7 +65,7 @@ public void subscribeToEvent(EventListener eventListener, Class eventType) { + public void unsubscribeFromEvent(final EventListener eventListener, final Class eventType) { synchronized (this.subscriptions) { @@ -87,7 +87,7 @@ public void unsubscribeFromEvent(EventListener eventListener, Class invokableClass, - Configuration runtimeConfiguration) { + public Environment(final JobID jobID, final String taskName, + final Class invokableClass, final Configuration runtimeConfiguration) { this.jobID = jobID; this.taskName = taskName; this.invokableClass = invokableClass; @@ -216,7 +216,7 @@ public JobID getJobID() { * @param executionListener * the object to be notified for important events during the task execution */ - public void registerExecutionListener(ExecutionListener executionListener) { + public void registerExecutionListener(final ExecutionListener executionListener) { synchronized (this.executionListeners) { @@ -233,7 +233,7 @@ public void registerExecutionListener(ExecutionListener executionListener) { * @param executionListener * the lister object to be unregistered */ - public void unregisterExecutionListener(ExecutionListener executionListener) { + public void unregisterExecutionListener(final ExecutionListener executionListener) { synchronized (this.executionListeners) { this.executionListeners.remove(executionListener); @@ -267,7 +267,7 @@ public boolean hasUnboundOutputGates() { * the index of the unbound output gate * @return the unbound output gate with the given ID, or null if no such gate exists */ - public OutputGate getUnboundOutputGate(int gateID) { + public OutputGate getUnboundOutputGate(final int gateID) { if (this.unboundOutputGates.size() == 0) { LOG.debug("No unbound output gates"); @@ -283,7 +283,7 @@ public OutputGate getUnboundOutputGate(int gateID) { * the index of the unbound input gate * @return the unbound input gate with the given ID, or null if no such gate exists */ - public InputGate getUnboundInputGate(int gateID) { + public InputGate getUnboundInputGate(final int gateID) { if (this.unboundInputGates.size() == 0) { LOG.debug("No unbound input gates"); @@ -440,7 +440,7 @@ private void activateInputChannels() throws IOException, InterruptedException { * @param outputGate * the output gate to be registered with the environment */ - public void registerOutputGate(OutputGate outputGate) { + public void registerOutputGate(final OutputGate outputGate) { LOG.debug("Registering output gate"); this.outputGates.add(outputGate); } @@ -451,7 +451,7 @@ public void registerOutputGate(OutputGate outputGate) { * @param inputGate * the input gate to be registered with the environment */ - public void registerInputGate(InputGate inputGate) { + public void registerInputGate(final InputGate inputGate) { LOG.debug("Registering input gate"); this.inputGates.add(inputGate); } @@ -481,7 +481,7 @@ public int getNumberOfInputGates() { * the index of the input gate to return * @return the input gate at index pos or null if no such index exists */ - public InputGate getInputGate(int pos) { + public InputGate getInputGate(final int pos) { if (pos < this.inputGates.size()) { return this.inputGates.get(pos); } @@ -496,7 +496,7 @@ public InputGate getInputGate(int pos) { * the index of the output gate to return * @return the output gate at index pos or null if no such index exists */ - public OutputGate getOutputGate(int pos) { + public OutputGate getOutputGate(final int pos) { if (pos < this.outputGates.size()) { return this.outputGates.get(pos); } @@ -510,7 +510,11 @@ public OutputGate getOutputGate(int pos) { public void startExecution() { if (this.executingThread == null) { - this.executingThread = new Thread(this, this.taskName); + if (this.taskName != null) { + this.executingThread = new Thread(this, this.taskName); + } else { + this.executingThread = new Thread(this); + } this.executingThread.start(); } } @@ -560,7 +564,7 @@ public void cancelExecution() { */ @SuppressWarnings("unchecked") @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { // Read job vertex id this.jobID = new JobID(); @@ -715,7 +719,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { // Write out job vertex id if (this.jobID == null) { @@ -915,7 +919,7 @@ public IOManager getIOManager() { * @param memoryManager * the new {@link IOManager} */ - public void setIOManager(IOManager ioManager) { + public void setIOManager(final IOManager ioManager) { this.ioManager = ioManager; } @@ -934,7 +938,7 @@ public MemoryManager getMemoryManager() { * @param memoryManager * the new {@link MemoryManager} */ - public void setMemoryManager(MemoryManager memoryManager) { + public void setMemoryManager(final MemoryManager memoryManager) { this.memoryManager = memoryManager; } @@ -963,7 +967,7 @@ public int getCurrentNumberOfSubtasks() { * @param currentNumberOfSubtasks * the current number of subtasks the respective task is split into */ - public void setCurrentNumberOfSubtasks(int currentNumberOfSubtasks) { + public void setCurrentNumberOfSubtasks(final int currentNumberOfSubtasks) { this.currentNumberOfSubtasks = currentNumberOfSubtasks; } @@ -984,12 +988,12 @@ public int getIndexInSubtaskGroup() { * @param indexInSubtaskGroup * the index of this subtask in the subtask group */ - public void setIndexInSubtaskGroup(int indexInSubtaskGroup) { + public void setIndexInSubtaskGroup(final int indexInSubtaskGroup) { this.indexInSubtaskGroup = indexInSubtaskGroup; } - public void changeExecutionState(ExecutionState newExecutionState, String optionalMessage) { + public void changeExecutionState(final ExecutionState newExecutionState, final String optionalMessage) { // Ignore state changes in final states if (this.executionState == ExecutionState.CANCELED || this.executionState == ExecutionState.FINISHED @@ -1132,7 +1136,7 @@ public InputSplitProvider getInputSplitProvider() { * @param userThread * the user thread which has been started */ - public void userThreadStarted(Thread userThread) { + public void userThreadStarted(final Thread userThread) { synchronized (this.executionListeners) { final Iterator it = this.executionListeners.iterator(); @@ -1149,7 +1153,7 @@ public void userThreadStarted(Thread userThread) { * @param userThread * the user thread which has finished */ - public void userThreadFinished(Thread userThread) { + public void userThreadFinished(final Thread userThread) { synchronized (this.executionListeners) { final Iterator it = this.executionListeners.iterator(); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java index d8ad91e887736..5e803a53d03a9 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/ExecutionSignature.java @@ -63,7 +63,7 @@ public final class ExecutionSignature { * @param signature * the byte buffer containing the signature. */ - private ExecutionSignature(byte[] signature) { + private ExecutionSignature(final byte[] signature) { this.signature = signature; } @@ -76,7 +76,8 @@ private ExecutionSignature(byte[] signature) { * the ID of the job * @return the cryptographic signature of this vertex */ - public static ExecutionSignature createSignature(Class invokableClass, JobID jobID) { + public static ExecutionSignature createSignature(final Class invokableClass, + final JobID jobID) { // First, try to load message digest algorithm, if necessary if (messageDigest == null) { @@ -117,7 +118,7 @@ public static ExecutionSignature createSignature(Classnull if the specified job ID is unknown */ - private String[] getRequiredJarFilesInternal(JobID id) { + private String[] getRequiredJarFilesInternal(final JobID id) { LibraryManagerEntry entry = null; @@ -479,7 +442,7 @@ private String[] getRequiredJarFilesInternal(JobID id) { * @throws IOException * thrown if an error occurs while writing the data */ - public static void writeLibraryToStream(String libraryFileName, DataOutput out) throws IOException { + public static void writeLibraryToStream(final String libraryFileName, final DataOutput out) throws IOException { final LibraryCacheManager lib = get(); lib.writeLibraryToStreamInternal(libraryFileName, out); @@ -496,7 +459,7 @@ public static void writeLibraryToStream(String libraryFileName, DataOutput out) * @throws IOException * thrown if an error occurs while writing the data */ - private void writeLibraryToStreamInternal(String libraryFileName, DataOutput out) throws IOException { + private void writeLibraryToStreamInternal(final String libraryFileName, final DataOutput out) throws IOException { if (libraryFileName == null) { throw new IOException("libraryName is null!"); @@ -535,7 +498,7 @@ private void writeLibraryToStreamInternal(String libraryFileName, DataOutput out * @throws IOException * throws if an error occurs while reading from the stream */ - public static void readLibraryFromStream(DataInput in) throws IOException { + public static void readLibraryFromStream(final DataInput in) throws IOException { final LibraryCacheManager lib = get(); lib.readLibraryFromStreamInternal(in); @@ -550,7 +513,7 @@ public static void readLibraryFromStream(DataInput in) throws IOException { * @throws IOException * throws if an error occurs while reading from the stream */ - private void readLibraryFromStreamInternal(DataInput in) throws IOException { + private void readLibraryFromStreamInternal(final DataInput in) throws IOException { final String libraryFileName = StringRecord.readString(in); @@ -564,7 +527,7 @@ private void readLibraryFromStreamInternal(DataInput in) throws IOException { throw new IOException("Submitted jar file " + libraryFileName + " is too large"); } - final byte buf[] = new byte[(int) length]; + final byte[] buf = new byte[(int) length]; in.readFully(buf); final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName); @@ -596,7 +559,8 @@ private void readLibraryFromStreamInternal(DataInput in) throws IOException { * thrown if the library cache manager could not be instantiated or an error occurred while reading the * library data from the input stream */ - public static void addLibrary(JobID jobID, Path name, long size, DataInput in) throws IOException { + public static void addLibrary(final JobID jobID, final Path name, final long size, final DataInput in) + throws IOException { final LibraryCacheManager lib = get(); lib.addLibraryInternal(jobID, name, size, in); @@ -617,14 +581,15 @@ public static void addLibrary(JobID jobID, Path name, long size, DataInput in) t * @throws IOException * thrown if an error occurred while reading the library data from the input stream */ - private void addLibraryInternal(JobID jobID, Path name, long size, DataInput in) throws IOException { + private void addLibraryInternal(final JobID jobID, final Path name, final long size, final DataInput in) + throws IOException { if (size > (long) Integer.MAX_VALUE) { throw new IOException("Submitted jar file " + name + " is too large"); } // Map the entire jar file to memory - final byte buf[] = new byte[(int) size]; + final byte[] buf = new byte[(int) size]; in.readFully(buf); // Reset and calculate message digest from jar file @@ -685,7 +650,7 @@ private static class LibraryManagerEntry { * an array with the names of required libraries by the corresponding job (URL objects required by the * class loader) */ - public LibraryManagerEntry(JobID id, String[] requiredJarFiles, URL[] urls) { + public LibraryManagerEntry(final JobID id, final String[] requiredJarFiles, URL[] urls) { String[] temp = requiredJarFiles; if (temp == null) { @@ -749,7 +714,7 @@ private static class LibraryTranslationKey { * @param clientPath * the client path */ - public LibraryTranslationKey(JobID jobID, Path clientPath) { + public LibraryTranslationKey(final JobID jobID, final Path clientPath) { this.jobID = jobID; this.clientPath = clientPath; @@ -770,7 +735,7 @@ public int hashCode() { * {@inheritDoc} */ @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (obj == null) { return false; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java index 15c67b56cbe26..c90c19f6aa4c4 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileRequest.java @@ -51,7 +51,7 @@ public String[] getRequiredLibraries() { * @param requiredLibraries * the names of libraries whose cache status is to be retrieved */ - public void setRequiredLibraries(String[] requiredLibraries) { + public void setRequiredLibraries(final String[] requiredLibraries) { this.requiredLibraries = requiredLibraries; } @@ -59,7 +59,7 @@ public void setRequiredLibraries(String[] requiredLibraries) { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { // Read required jar files this.requiredLibraries = new String[in.readInt()]; @@ -73,7 +73,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { if (this.requiredLibraries == null) { throw new IOException("requiredLibraries is null"); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java index 386bbdb6b3144..73da0c6f635de 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheProfileResponse.java @@ -48,7 +48,7 @@ public class LibraryCacheProfileResponse implements IOReadableWritable { * @param request * the library cache profile request the response belongs to */ - public LibraryCacheProfileResponse(LibraryCacheProfileRequest request) { + public LibraryCacheProfileResponse(final LibraryCacheProfileRequest request) { this.requiredLibraries = request.getRequiredLibraries(); this.cached = new boolean[this.requiredLibraries.length]; @@ -69,7 +69,7 @@ public LibraryCacheProfileResponse() { * @param cached * true if the library at the given position is in the local cache, false otherwise */ - public void setCached(int pos, boolean cached) { + public void setCached(final int pos, final boolean cached) { if (pos < this.cached.length) { this.cached[pos] = cached; @@ -84,7 +84,7 @@ public void setCached(int pos, boolean cached) { * @return true if the library at the given position is in the local cache, false * otherwise */ - public boolean isCached(int pos) { + public boolean isCached(final int pos) { if (pos < this.cached.length) { return this.cached[pos]; @@ -97,7 +97,7 @@ public boolean isCached(int pos) { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { // Read the names of the required jar files this.requiredLibraries = new String[in.readInt()]; @@ -117,7 +117,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { if (this.requiredLibraries == null) { throw new IOException("requiredLibraries is null"); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java index 8714871d0c509..9b0bde097ca9b 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/execution/librarycache/LibraryCacheUpdate.java @@ -39,7 +39,7 @@ public class LibraryCacheUpdate implements IOReadableWritable { * @param libraryFileName * the name of the library that should be transported within this object. */ - public LibraryCacheUpdate(String libraryFileName) { + public LibraryCacheUpdate(final String libraryFileName) { this.libraryFileName = libraryFileName; } @@ -53,7 +53,7 @@ public LibraryCacheUpdate() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { LibraryCacheManager.readLibraryFromStream(in); } @@ -62,7 +62,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { if (this.libraryFileName == null) { throw new IOException("libraryFileName is null"); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java index f97964f8dc991..e80fcbef00360 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileInputSplit.java @@ -55,7 +55,7 @@ public class FileInputSplit implements InputSplit { * List of hosts (hostnames) containing the block, possibly null. */ private String[] hosts; - + /** * The logical number of the split. */ @@ -64,6 +64,8 @@ public class FileInputSplit implements InputSplit { /** * Constructs a split with host information. * + * @param num + * the number of this input split * @param file * the file name * @param start @@ -73,7 +75,7 @@ public class FileInputSplit implements InputSplit { * @param hosts * the list of hosts containing the block, possibly null */ - public FileInputSplit(int num, Path file, long start, long length, String[] hosts) { + public FileInputSplit(final int num, final Path file, final long start, final long length, final String[] hosts) { this.partitionNumber = num; this.file = file; this.start = start; @@ -127,11 +129,12 @@ public String[] getHostNames() { } } - /* (non-Javadoc) + /* + * (non-Javadoc) * @see eu.stratosphere.nephele.template.InputSplit#getPartitionNumber() */ @Override - public int getPartitionNumber() { + public int getSplitNumber() { return this.partitionNumber; } @@ -147,11 +150,10 @@ public String toString() { * {@inheritDoc} */ @Override - public void write(final DataOutput out) throws IOException - { + public void write(final DataOutput out) throws IOException { // write partition number out.writeInt(this.partitionNumber); - + // write file if (this.file != null) { out.writeBoolean(true); @@ -163,7 +165,7 @@ public void write(final DataOutput out) throws IOException // write start and length out.writeLong(this.start); out.writeLong(this.length); - + // write hosts if (this.hosts == null) { out.writeBoolean(false); @@ -180,11 +182,10 @@ public void write(final DataOutput out) throws IOException * {@inheritDoc} */ @Override - public void read(final DataInput in) throws IOException - { + public void read(final DataInput in) throws IOException { // read partition number this.partitionNumber = in.readInt(); - + // read file path boolean isNotNull = in.readBoolean(); if (isNotNull) { diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java index 6d3532f936bee..73ed1062d019a 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/FileSystem.java @@ -49,7 +49,7 @@ public abstract class FileSystem { /** * Object used to protect calls to specific methods. */ - private static final Object synchronizationObject = new Object(); + private static final Object SYNCHRONIZATION_OBJECT = new Object(); /** * An auxiliary class to identify a file system by its scheme @@ -78,7 +78,7 @@ public static class FSKey { * @param authority * the authority of the file system */ - public FSKey(String scheme, String authority) { + public FSKey(final String scheme, final String authority) { this.scheme = scheme; this.authority = authority; } @@ -87,7 +87,7 @@ public FSKey(String scheme, String authority) { * {@inheritDoc} */ @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (obj instanceof FSKey) { final FSKey key = (FSKey) obj; @@ -187,7 +187,7 @@ public static FileSystem get(final URI uri) throws IOException { FileSystem fs = null; - synchronized (synchronizationObject) { + synchronized (SYNCHRONIZATION_OBJECT) { if (uri.getScheme() == null) { throw new IOException("FileSystem: Scheme is null"); @@ -313,7 +313,7 @@ public long getDefaultBlockSize() { * @param f * source file */ - public boolean exists(Path f) throws IOException { + public boolean exists(final Path f) throws IOException { try { return (getFileStatus(f) != null); @@ -386,7 +386,7 @@ public abstract FSDataOutputStream create(Path f, boolean overwrite, int bufferS * @return the number of block's thie file/directory consists of * @throws IOException */ - public int getNumberOfBlocks(FileStatus file) throws IOException { + public int getNumberOfBlocks(final FileStatus file) throws IOException { int numberOfBlocks = 0; @@ -400,7 +400,7 @@ public int getNumberOfBlocks(FileStatus file) throws IOException { } // file is a directory - FileStatus[] files = this.listStatus(file.getPath()); + final FileStatus[] files = this.listStatus(file.getPath()); for (int i = 0; i < files.length; i++) { if (!files[i].isDir()) { @@ -411,7 +411,7 @@ public int getNumberOfBlocks(FileStatus file) throws IOException { return numberOfBlocks; } - private int getNumberOfBlocks(long length, long blocksize) { + private int getNumberOfBlocks(final long length, final long blocksize) { if (blocksize != 0) { int numberOfBlocks; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java index 897d8a31690ea..0723f6b6f7d7e 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalBlockLocation.java @@ -31,7 +31,7 @@ public class LocalBlockLocation implements BlockLocation { private final String[] hosts; - public LocalBlockLocation(String host, long length) { + public LocalBlockLocation(final String host, final long length) { this.hosts = new String[] { host }; this.length = length; } @@ -62,12 +62,11 @@ public long getOffset() { return 0; } - /* - * (non-Javadoc) - * @see java.lang.Comparable#compareTo(java.lang.Object) + /** + * {@inheritDoc} */ @Override - public int compareTo(BlockLocation o) { + public int compareTo(final BlockLocation o) { return 0; } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java index 191e5fbac57eb..caf152e5a5139 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataInputStream.java @@ -47,8 +47,7 @@ public class LocalDataInputStream extends FSDataInputStream { * @throws IOException * thrown if the data input stream cannot be created */ - public LocalDataInputStream(File file) - throws IOException { + public LocalDataInputStream(final File file) throws IOException { this.fis = new FileInputStream(file); this.position = 0; @@ -58,7 +57,7 @@ public LocalDataInputStream(File file) * {@inheritDoc} */ @Override - public void seek(long desired) throws IOException { + public void seek(final long desired) throws IOException { this.fis.getChannel().position(desired); this.position = desired; @@ -82,7 +81,7 @@ public int read() throws IOException { * {@inheritDoc} */ @Override - public int read(byte[] buffer, int offset, int length) throws IOException { + public int read(final byte[] buffer, final int offset, final int length) throws IOException { final int value = this.fis.read(buffer, offset, length); if (value > 0) { @@ -113,7 +112,7 @@ public int available() throws IOException { * {@inheritDoc} */ @Override - public long skip(long n) throws IOException { + public long skip(final long n) throws IOException { return this.fis.skip(n); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java index 655ab0d63f6cb..ed9eec460f51e 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalDataOutputStream.java @@ -47,8 +47,7 @@ public class LocalDataOutputStream extends FSDataOutputStream { * @throws IOException * thrown if the data output stream cannot be created */ - public LocalDataOutputStream(File file) - throws IOException { + public LocalDataOutputStream(final File file) throws IOException { this.fos = new FileOutputStream(file); this.position = 0; @@ -58,7 +57,7 @@ public LocalDataOutputStream(File file) * {@inheritDoc} */ @Override - public void write(int b) throws IOException { + public void write(final int b) throws IOException { fos.write(b); position++; } @@ -67,7 +66,7 @@ public void write(int b) throws IOException { * {@inheritDoc} */ @Override - public void write(byte b[], int off, int len) throws IOException { + public void write(final byte[] b, final int off, final int len) throws IOException { fos.write(b, off, len); position += len; // update position } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java index d91884929ad94..79ee643c3ff7b 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileStatus.java @@ -47,7 +47,7 @@ public class LocalFileStatus implements FileStatus { * @param fs * the file system the corresponding file has been read from */ - public LocalFileStatus(File f, FileSystem fs) { + public LocalFileStatus(final File f, final FileSystem fs) { this.file = f; this.path = new Path(fs.getUri().getScheme() + ":" + f.getAbsolutePath()); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java index 361c6d1110d9f..4984e1a981da6 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/fs/file/LocalFileSystem.java @@ -84,9 +84,10 @@ public LocalFileSystem() { * {@inheritDoc} */ @Override - public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { + public BlockLocation[] getFileBlockLocations(final FileStatus file, final long start, final long len) + throws IOException { - BlockLocation[] blockLocations = new BlockLocation[1]; + final BlockLocation[] blockLocations = new BlockLocation[1]; blockLocations[0] = new LocalBlockLocation(this.hostName, file.getLen()); return blockLocations; @@ -129,7 +130,7 @@ public Path getWorkingDirectory() { * {@inheritDoc} */ @Override - public void initialize(URI name) throws IOException { + public void initialize(final URI name) throws IOException { // TODO Auto-generated method stub } @@ -138,7 +139,7 @@ public void initialize(URI name) throws IOException { * {@inheritDoc} */ @Override - public FSDataInputStream open(Path f, int bufferSize) throws IOException { + public FSDataInputStream open(final Path f, final int bufferSize) throws IOException { // TODO Auto-generated method stub return null; } @@ -147,7 +148,7 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException { * {@inheritDoc} */ @Override - public FSDataInputStream open(Path f) throws IOException { + public FSDataInputStream open(final Path f) throws IOException { final File file = pathToFile(f); @@ -169,7 +170,7 @@ private File pathToFile(Path path) { * {@inheritDoc} */ @Override - public FileStatus[] listStatus(Path f) throws IOException { + public FileStatus[] listStatus(final Path f) throws IOException { final File localf = pathToFile(f); FileStatus[] results; @@ -197,7 +198,7 @@ public FileStatus[] listStatus(Path f) throws IOException { * {@inheritDoc} */ @Override - public boolean delete(Path f, boolean recursive) throws IOException { + public boolean delete(final Path f, final boolean recursive) throws IOException { final File file = pathToFile(f); if (file.isFile()) { @@ -218,13 +219,13 @@ public boolean delete(Path f, boolean recursive) throws IOException { * @throws IOException * thrown if an error occurred while deleting the files/directories */ - private boolean delete(File f) throws IOException { + private boolean delete(final File f) throws IOException { if (f.isDirectory()) { final File[] files = f.listFiles(); for (int i = 0; i < files.length; i++) { - boolean del = delete(files[i]); + final boolean del = delete(files[i]); if (del == false) { return false; } @@ -246,7 +247,7 @@ private boolean delete(File f) throws IOException { * @throws IOException * thrown if an error occurred while creating the directory/directories */ - public boolean mkdirs(Path f) throws IOException { + public boolean mkdirs(final Path f) throws IOException { final Path parent = f.getParent(); final File p2f = pathToFile(f); @@ -257,8 +258,8 @@ public boolean mkdirs(Path f) throws IOException { * {@inheritDoc} */ @Override - public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize) - throws IOException { + public FSDataOutputStream create(final Path f, final boolean overwrite, final int bufferSize, + final short replication, final long blockSize) throws IOException { if (exists(f) && !overwrite) { throw new IOException("File already exists:" + f); @@ -277,9 +278,8 @@ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, shor * {@inheritDoc} */ @Override - public FSDataOutputStream create(Path f, boolean overwrite) throws IOException { + public FSDataOutputStream create(final Path f, final boolean overwrite) throws IOException { return create(f, overwrite, 0, (short) 0, 0); } - } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java index a83705dfdb54c..9095e2a2fa8d7 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/AbstractID.java @@ -42,7 +42,7 @@ public abstract class AbstractID implements IOReadableWritable { /** * Constructs a new ID with a specific bytes value. */ - public AbstractID(byte[] bytes) { + public AbstractID(final byte[] bytes) { if(bytes.length == SIZE) { System.arraycopy(bytes, 0, this.bytes, 0, SIZE); @@ -65,16 +65,16 @@ public AbstractID() { * @param src * the bytes the ID consists of */ - private void setBytes(byte[] src) { + private void setBytes(final byte[] src) { if (src == null) { return; } - if(src.length != SIZE) { + if (src.length != SIZE) { return; } - + System.arraycopy(src, 0, this.bytes, 0, SIZE); } @@ -93,7 +93,7 @@ private byte[] getBytes() { * @param src * the source ID */ - public void setID(AbstractID src) { + public void setID(final AbstractID src) { setBytes(src.getBytes()); } @@ -101,7 +101,7 @@ public void setID(AbstractID src) { * {@inheritDoc} */ @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (!(obj instanceof AbstractID)) { return false; @@ -163,7 +163,7 @@ public int hashCode() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { in.readFully(this.bytes); } @@ -172,7 +172,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { // Write the particular bytes out.write(this.bytes); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java index b39e8241e356d..1efb01e678121 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/BipartiteDistributionPattern.java @@ -27,7 +27,8 @@ public class BipartiteDistributionPattern implements DistributionPattern { * {@inheritDoc} */ @Override - public boolean createWire(int nodeLowerStage, int nodeUpperStage, int sizeSetLowerStage, int sizeSetUpperStage) { + public boolean createWire(final int nodeLowerStage, final int nodeUpperStage, final int sizeSetLowerStage, + final int sizeSetUpperStage) { return true; } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java index ecc63535d2084..7618befaad3e7 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/ChannelSelector.java @@ -15,7 +15,6 @@ package eu.stratosphere.nephele.io; -import eu.stratosphere.nephele.io.channels.AbstractOutputChannel; import eu.stratosphere.nephele.types.Record; /** @@ -39,5 +38,5 @@ public interface ChannelSelector extends IOReadableWritable { * @return a (possibly empty) array of integer numbers which indicate the indices of the output channels through * which the record shall be forwarded */ - int[] selectChannels(T record, int numberOfOutpuChannels); + int[] selectChannels(T record, int numberOfOutputChannels); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java index 79f81d3dbdbe7..504c87c89130e 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultChannelSelector.java @@ -31,6 +31,9 @@ */ public class DefaultChannelSelector implements ChannelSelector { + /** + * Stores the index of the channel to send the next record to. + */ private final int[] nextChannelToSendTo = new int[1]; /** @@ -44,7 +47,7 @@ public DefaultChannelSelector() { * {@inheritDoc} */ @Override - public int[] selectChannels(T record, int numberOfOutpuChannels) { + public int[] selectChannels(final T record, final int numberOfOutpuChannels) { this.nextChannelToSendTo[0] = (this.nextChannelToSendTo[0] + 1) % numberOfOutpuChannels; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java index 4680f4b6de581..43fe817ae7f98 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/DefaultRecordDeserializer.java @@ -117,4 +117,4 @@ record = recordClass.newInstance(); } return record; } -} \ No newline at end of file +} diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java index 78abed20dd698..d80c21cb253e9 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/PointwiseDistributionPattern.java @@ -30,7 +30,8 @@ public class PointwiseDistributionPattern implements DistributionPattern { * {@inheritDoc} */ @Override - public boolean createWire(int nodeLowerStage, int nodeUpperStage, int sizeSetLowerStage, int sizeSetUpperStage) { + public boolean createWire(final int nodeLowerStage, final int nodeUpperStage, final int sizeSetLowerStage, + final int sizeSetUpperStage) { if (sizeSetLowerStage < sizeSetUpperStage) { if (nodeLowerStage == (nodeUpperStage % sizeSetLowerStage)) { diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java index b1125b6f2c3cf..83507c6a640c3 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/RecordBuffer.java @@ -55,7 +55,7 @@ public class RecordBuffer { * the maximum size of the buffer */ @SuppressWarnings("unchecked") - public RecordBuffer(Class type, int maxSize) { + public RecordBuffer(final Class type, final int maxSize) { this.buffer = (T[]) Array.newInstance(type, maxSize); } @@ -66,7 +66,7 @@ public RecordBuffer(Class type, int maxSize) { * @param record * the record to be added to the buffer. */ - public void put(T record) { + public void put(final T record) { if (record == null) { return; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java index efdfeb9fb1eec..ad21366396fb3 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/DeserializationBuffer.java @@ -52,7 +52,7 @@ public class DeserializationBuffer { /** * Buffer to reconstruct the length field. */ - ByteBuffer lengthBuf = ByteBuffer.allocate(SIZEOFINT); + private ByteBuffer lengthBuf = ByteBuffer.allocate(SIZEOFINT); /** * Size of the record to be deserialized in bytes. @@ -64,7 +64,7 @@ public class DeserializationBuffer { /** * Temporary buffer. */ - ByteBuffer tempBuffer = null; + private ByteBuffer tempBuffer = null; /** * Constructs a new deserialization buffer with the specified type. @@ -72,10 +72,10 @@ public class DeserializationBuffer { * @param type * the type of the record the deserialization buffer can be used for * @param propagateEndOfStream - * true> if end of stream notifications during the + * true if end of stream notifications during the * deserialization process shall be propagated to the caller, false otherwise */ - public DeserializationBuffer(RecordDeserializer deserializer, boolean propagateEndOfStream) { + public DeserializationBuffer(final RecordDeserializer deserializer, final boolean propagateEndOfStream) { this.deserializer = deserializer; this.propagateEndOfStream = propagateEndOfStream; } @@ -89,7 +89,7 @@ public DeserializationBuffer(RecordDeserializer deserializer, boolean propaga * @throws IOException * thrown if an error occurs while reading the data or deserializing the object */ - public T readData(T target, ReadableByteChannel readableByteChannel) throws IOException { + public T readData(final T target, final ReadableByteChannel readableByteChannel) throws IOException { if (this.recordLength < 0) { if (readableByteChannel.read(this.lengthBuf) == -1 && this.propagateEndOfStream) { @@ -129,8 +129,8 @@ public T readData(T target, ReadableByteChannel readableByteChannel) throws IOEx return null; } - deserializationBuffer.reset(tempBuffer.array(), this.recordLength); - final T record = deserializer.deserialize(target, deserializationBuffer); + this.deserializationBuffer.reset(tempBuffer.array(), this.recordLength); + final T record = deserializer.deserialize(target, this.deserializationBuffer); this.recordLength = -1; this.lengthBuf.clear(); @@ -145,7 +145,7 @@ public T readData(T target, ReadableByteChannel readableByteChannel) throws IOEx * the array of bytes used as input. * @return the resulting integer */ - private int byteArrayToInt(byte[] arr) { + private int byteArrayToInt(final byte[] arr) { int number = 0; for (int i = 0; i < SIZEOFINT; ++i) { diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java index 8c785ef6f4eac..9a3b4b3df1d57 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/io/channels/SerializationBuffer.java @@ -47,10 +47,10 @@ public class SerializationBuffer { * @param arr * The byte buffer to store the data of the integer */ - private void integerToByteBuffer(int val, ByteBuffer byteBuffer) { + private void integerToByteBuffer(final int val, final ByteBuffer byteBuffer) { for (int i = 0; i < SIZEOFINT; ++i) { - int shift = i << (SIZEOFINT - 1); // i * 8 + final int shift = i << (SIZEOFINT - 1); // i * 8 byteBuffer.put(SIZEOFINT - 1 - i, (byte) ((val & (0xff << shift)) >>> shift)); } @@ -68,8 +68,9 @@ private void integerToByteBuffer(int val, ByteBuffer byteBuffer) { */ public boolean dataLeftFromPreviousSerialization() { - if (leftInSerializationBuffer() > 0) + if (leftInSerializationBuffer() > 0) { return true; + } return false; } @@ -84,7 +85,7 @@ public boolean dataLeftFromPreviousSerialization() { * @throws IOException * thrown if an error occurs while writing to serialized data to the channel */ - public int read(WritableByteChannel writableByteChannel) throws IOException { + public int read(final WritableByteChannel writableByteChannel) throws IOException { int bytesReadFromLengthBuf = 0; @@ -126,17 +127,18 @@ private int leftInSerializationBuffer() { * Thrown if data from a previous serialization process is still in the internal buffer and has not yet been * transfered to a byte buffer */ - public void serialize(T record) throws IOException { + public void serialize(final T record) throws IOException { // Check if there is data left in the buffer - if (dataLeftFromPreviousSerialization()) + if (dataLeftFromPreviousSerialization()) { throw new IOException("Cannot write new data, " + leftInSerializationBuffer() + " bytes still left from previous call"); + } record.write(this.serializationBuffer); // serializationBuffer grows dynamically // Now record is completely in serializationBuffer; - integerToByteBuffer(serializationBuffer.getLength(), this.lengthBuf); + integerToByteBuffer(this.serializationBuffer.getLength(), this.lengthBuf); } public void clear() { diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobInputVertex.java new file mode 100644 index 0000000000000..4fe00f3d782ef --- /dev/null +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobInputVertex.java @@ -0,0 +1,40 @@ +/*********************************************************************************************************************** + * + * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu) + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + **********************************************************************************************************************/ + +package eu.stratosphere.nephele.jobgraph; + +/** + * An abstract base class for input vertices in Nephele. + * + * @author warneke + */ +public abstract class AbstractJobInputVertex extends AbstractJobVertex { + + /** + * Constructs a new job input vertex with the given name. + * + * @param name + * the name of the new job input vertex + * @param id + * the ID of this vertex + * @param jobGraph + * the job graph this vertex belongs to + */ + protected AbstractJobInputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { + super(name, id, jobGraph); + + jobGraph.addVertex(this); + } +} diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobOutputVertex.java new file mode 100644 index 0000000000000..4718c026c7dd8 --- /dev/null +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobOutputVertex.java @@ -0,0 +1,40 @@ +/*********************************************************************************************************************** + * + * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu) + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + **********************************************************************************************************************/ + +package eu.stratosphere.nephele.jobgraph; + +/** + * An abstract base class for output vertices in Nephele. + * + * @author warneke + */ +public abstract class AbstractJobOutputVertex extends AbstractJobVertex { + + /** + * Constructs a new job output vertex with the given name. + * + * @param name + * the name of the new job output vertex + * @param id + * the ID of this vertex + * @param jobGraph + * the job graph this vertex belongs to + */ + protected AbstractJobOutputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { + super(name, id, jobGraph); + + jobGraph.addVertex(this); + } +} diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java index 50788c9fecd01..67d87e332c627 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/AbstractJobVertex.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import eu.stratosphere.nephele.configuration.Configuration; +import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager; import eu.stratosphere.nephele.io.IOReadableWritable; import eu.stratosphere.nephele.io.channels.ChannelType; import eu.stratosphere.nephele.io.compression.CompressionLevel; @@ -87,6 +88,11 @@ public abstract class AbstractJobVertex implements IOReadableWritable { */ private final Configuration configuration = new Configuration(); + /** + * The class of the invokable. + */ + protected Class invokableClass = null; + /** * Constructs a new job vertex and assigns it with the given name. * @@ -97,7 +103,7 @@ public abstract class AbstractJobVertex implements IOReadableWritable { * @param jobGraph * the job graph this vertex belongs to */ - protected AbstractJobVertex(String name, JobVertexID id, JobGraph jobGraph) { + protected AbstractJobVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { this.name = name; this.id = (id == null) ? new JobVertexID() : id; @@ -112,7 +118,7 @@ protected AbstractJobVertex(String name, JobVertexID id, JobGraph jobGraph) { * @throws JobGraphDefinitionException * thrown if the given vertex cannot be connected to vertex in the requested manner */ - public void connectTo(AbstractJobVertex vertex) throws JobGraphDefinitionException { + public void connectTo(final AbstractJobVertex vertex) throws JobGraphDefinitionException { this.connectTo(vertex, null, null, -1, -1); } @@ -130,7 +136,7 @@ public void connectTo(AbstractJobVertex vertex) throws JobGraphDefinitionExcepti * @throws JobGraphDefinitionException * thrown if the given vertex cannot be connected to vertex in the requested manner */ - public void connectTo(AbstractJobVertex vertex, int indexOfOutputGate, int indexOfInputGate) + public void connectTo(final AbstractJobVertex vertex, final int indexOfOutputGate, final int indexOfInputGate) throws JobGraphDefinitionException { this.connectTo(vertex, null, null, indexOfOutputGate, indexOfInputGate); } @@ -147,8 +153,8 @@ public void connectTo(AbstractJobVertex vertex, int indexOfOutputGate, int index * @throws JobGraphDefinitionException * thrown if the given vertex cannot be connected to vertex in the requested manner */ - public void connectTo(AbstractJobVertex vertex, ChannelType channelType, CompressionLevel compressionLevel) - throws JobGraphDefinitionException { + public void connectTo(final AbstractJobVertex vertex, final ChannelType channelType, + final CompressionLevel compressionLevel) throws JobGraphDefinitionException { this.connectTo(vertex, channelType, compressionLevel, -1, -1); } @@ -170,8 +176,9 @@ public void connectTo(AbstractJobVertex vertex, ChannelType channelType, Compres * @throws JobGraphDefinitionException * thrown if the given vertex cannot be connected to vertex in the requested manner */ - public void connectTo(AbstractJobVertex vertex, ChannelType channelType, CompressionLevel compressionLevel, - int indexOfOutputGate, int indexOfInputGate) throws JobGraphDefinitionException { + public void connectTo(final AbstractJobVertex vertex, final ChannelType channelType, + final CompressionLevel compressionLevel, int indexOfOutputGate, int indexOfInputGate) + throws JobGraphDefinitionException { if (vertex == null) { throw new JobGraphDefinitionException("Target vertex is null!"); @@ -253,8 +260,8 @@ protected int getFirstFreeInputGateIndex() { * @param indexOfInputGate * index of the consuming task's input gate to be used */ - private void connectBacklink(AbstractJobVertex vertex, ChannelType channelType, CompressionLevel compressionLevel, - int indexOfOutputGate, int indexOfInputGate) { + private void connectBacklink(final AbstractJobVertex vertex, final ChannelType channelType, + final CompressionLevel compressionLevel, final int indexOfOutputGate, final int indexOfInputGate) { // Make sure the array is big enough for (int i = this.backwardEdges.size(); i <= indexOfInputGate; i++) { @@ -298,7 +305,7 @@ public int getNumberOfBackwardConnections() { * the index of the edge * @return the forward edge or null if no edge exists at the specified index. */ - public JobEdge getForwardConnection(int index) { + public JobEdge getForwardConnection(final int index) { if (index < this.forwardEdges.size()) { return this.forwardEdges.get(index); @@ -314,7 +321,7 @@ public JobEdge getForwardConnection(int index) { * the index of the edge * @return the backward edge or null if no edge exists at the specified index */ - public JobEdge getBackwardConnection(int index) { + public JobEdge getBackwardConnection(final int index) { if (index < this.backwardEdges.size()) { return this.backwardEdges.get(index); @@ -361,8 +368,9 @@ public JobVertexID getID() { /** * {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { if (jobGraph == null) { throw new IOException("jobGraph is null, cannot deserialize"); @@ -418,13 +426,36 @@ public void read(DataInput in) throws IOException { this.forwardEdges.add(null); } } + + // Read the invokable class + final boolean isNotNull = in.readBoolean(); + if (!isNotNull) { + return; + } + + // Read the name of the class and try to instantiate the class object + + final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID()); + if (cl == null) { + throw new IOException("Cannot find class loader for vertex " + getID()); + } + + // Read the name of the expected class + final String className = StringRecord.readString(in); + + try { + this.invokableClass = (Class) Class.forName(className, true, cl); + } catch (ClassNotFoundException cnfe) { + throw new IOException("Class " + className + " not found in one of the supplied jar files: " + + StringUtils.stringifyException(cnfe)); + } } /** * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { // Instance type StringRecord.writeString(out, this.instanceType); @@ -463,6 +494,17 @@ public void write(DataOutput out) throws IOException { out.writeInt(edge.getIndexOfInputGate()); } } + + // Write the invokable class + if (this.invokableClass == null) { + out.writeBoolean(false); + return; + } + + out.writeBoolean(true); + + // Write out the name of the class + StringRecord.writeString(out, this.invokableClass.getName()); } /** @@ -480,7 +522,7 @@ public JobGraph getJobGraph() { * @param numberOfSubtasks * the number of subtasks this vertex represents should be split into at runtime */ - public void setNumberOfSubtasks(int numberOfSubtasks) { + public void setNumberOfSubtasks(final int numberOfSubtasks) { this.numberOfSubtasks = numberOfSubtasks; } @@ -500,7 +542,7 @@ public int getNumberOfSubtasks() { * @param instanceType * the instance type the task this vertex represents should run on */ - public void setInstanceType(String instanceType) { + public void setInstanceType(final String instanceType) { this.instanceType = instanceType; } @@ -519,7 +561,7 @@ public String getInstanceType() { * @param numberOfSubtasksPerInstance * the number of subtasks that should be assigned to the same instance */ - public void setNumberOfSubtasksPerInstance(int numberOfSubtasksPerInstance) { + public void setNumberOfSubtasksPerInstance(final int numberOfSubtasksPerInstance) { this.numberOfSubtasksPerInstance = numberOfSubtasksPerInstance; } @@ -538,7 +580,7 @@ public int getNumberOfSubtasksPerInstance() { * @param vertex * the vertex this vertex should share its instances with at runtime */ - public void setVertexToShareInstancesWith(AbstractJobVertex vertex) { + public void setVertexToShareInstancesWith(final AbstractJobVertex vertex) { this.vertexToShareInstancesWith = vertex; } @@ -569,7 +611,23 @@ public Configuration getConfiguration() { * @throws IllegalConfigurationException * thrown if the respective tasks is not configured properly */ - public abstract void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException; + public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException { + + if (invokable == null) { + throw new IllegalArgumentException("Argument invokable is null"); + } + + // see if the task itself has a valid configuration + // because this is user code running on the master, we embed it in a catch-all block + try { + invokable.checkConfiguration(); + } catch (IllegalConfigurationException icex) { + throw icex; // simply forward + } catch (Throwable t) { + throw new IllegalConfigurationException("Checking the invokable's configuration caused an error: " + + StringUtils.stringifyException(t)); + } + } /** * Returns the minimum number of subtasks the respective task @@ -579,7 +637,14 @@ public Configuration getConfiguration() { * an instance of the task this vertex represents * @return the minimum number of subtasks the respective task must be split into at runtime */ - public abstract int getMinimumNumberOfSubtasks(AbstractInvokable invokable); + public int getMinimumNumberOfSubtasks(final AbstractInvokable invokable) { + + if (invokable == null) { + throw new IllegalArgumentException("Argument invokable is null"); + } + + return invokable.getMinimumNumberOfSubtasks(); + } /** * Returns the maximum number of subtasks the respective task @@ -590,12 +655,22 @@ public Configuration getConfiguration() { * @return the maximum number of subtasks the respective task can be split into at runtime, -1 for * infinity */ - public abstract int getMaximumNumberOfSubtasks(AbstractInvokable invokable); + public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) { + + if (invokable == null) { + throw new IllegalArgumentException("Argument invokable is null"); + } + + return invokable.getMaximumNumberOfSubtasks(); + } /** * Returns the invokable class which represents the task of this vertex * * @return the invokable class, null if it is not set */ - public abstract Class getInvokableClass(); + public Class getInvokableClass() { + + return this.invokableClass; + } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java index 4ef6ebdd08e54..83adde429febc 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobEdge.java @@ -58,8 +58,8 @@ public class JobEdge { * @param indexOfInputGate * index of the consuming task's input gate that this edge connects to */ - public JobEdge(AbstractJobVertex connectedVertex, ChannelType channelType, CompressionLevel compressionLevel, - int indexOfInputGate) { + public JobEdge(final AbstractJobVertex connectedVertex, final ChannelType channelType, + final CompressionLevel compressionLevel, final int indexOfInputGate) { this.connectedVertex = connectedVertex; this.channelType = channelType; this.compressionLevel = compressionLevel; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java index 321523cea2aee..545ee03153700 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileInputVertex.java @@ -23,20 +23,19 @@ import eu.stratosphere.nephele.fs.FileSystem; import eu.stratosphere.nephele.fs.Path; import eu.stratosphere.nephele.template.AbstractFileInputTask; -import eu.stratosphere.nephele.template.AbstractInputTask; import eu.stratosphere.nephele.template.AbstractInvokable; import eu.stratosphere.nephele.template.IllegalConfigurationException; import eu.stratosphere.nephele.util.StringUtils; /** - * A JobFileInputVertex is a specific subtype of a {@link JobInputVertex} and is designed + * A JobFileInputVertex is a specific subtype of a {@link AbstractJobInputVertex} and is designed * for Nephele tasks which read data from a local or distributed file system. As every job input vertex * A JobFileInputVertex must not have any further input. * * @author warneke */ -public class JobFileInputVertex extends JobGenericInputVertex -{ +public final class JobFileInputVertex extends AbstractJobInputVertex { + /** * The path pointing to the input file/directory. */ @@ -52,7 +51,7 @@ public class JobFileInputVertex extends JobGenericInputVertex * @param jobGraph * the job graph this vertex belongs to */ - public JobFileInputVertex(String name, JobVertexID id, JobGraph jobGraph) { + public JobFileInputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { super(name, id, jobGraph); } @@ -64,7 +63,7 @@ public JobFileInputVertex(String name, JobVertexID id, JobGraph jobGraph) { * @param jobGraph * the job graph this vertex belongs to */ - public JobFileInputVertex(String name, JobGraph jobGraph) { + public JobFileInputVertex(final String name, final JobGraph jobGraph) { super(name, null, jobGraph); } @@ -74,7 +73,7 @@ public JobFileInputVertex(String name, JobGraph jobGraph) { * @param jobGraph * the job graph this vertex belongs to */ - public JobFileInputVertex(JobGraph jobGraph) { + public JobFileInputVertex(final JobGraph jobGraph) { super(null, null, jobGraph); } @@ -84,7 +83,7 @@ public JobFileInputVertex(JobGraph jobGraph) { * @param path * the path of the file the job file input vertex's task should read from */ - public void setFilePath(Path path) { + public void setFilePath(final Path path) { this.path = path; } @@ -104,8 +103,8 @@ public Path getFilePath() { * @param inputClass * the class of the vertex's input task. */ - public void setFileInputClass(Class inputClass) { - this.inputClass = (Class>) inputClass; + public void setFileInputClass(final Class inputClass) { + this.invokableClass = inputClass; } /** @@ -115,19 +114,18 @@ public void setFileInputClass(Class inputClass) */ @SuppressWarnings("unchecked") public Class getFileInputClass() { - return (Class) this.inputClass; + return (Class) this.invokableClass; } /** * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException - { + public void read(final DataInput in) throws IOException { super.read(in); // Read path of the input file - boolean isNotNull = in.readBoolean(); + final boolean isNotNull = in.readBoolean(); if (isNotNull) { this.path = new Path(); this.path.read(in); @@ -138,8 +136,7 @@ public void read(DataInput in) throws IOException * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException - { + public void write(final DataOutput out) throws IOException { super.write(out); // Write out the path of the input file @@ -156,7 +153,7 @@ public void write(DataOutput out) throws IOException * {@inheritDoc} */ @Override - public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException { + public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException { // Check if the user has specified a path if (this.path == null) { @@ -170,14 +167,14 @@ public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfig if (f == null) { throw new IOException(path.toString() + " led to a null object"); } - } - catch (IOException e) { + } catch (IOException e) { throw new IllegalConfigurationException("Cannot access file or directory: " + StringUtils.stringifyException(e)); } - + // register the path in the configuration - invokable.getRuntimeConfiguration().setString(AbstractFileInputTask.INPUT_PATH_CONFIG_KEY, this.path.toString()); + invokable.getRuntimeConfiguration() + .setString(AbstractFileInputTask.INPUT_PATH_CONFIG_KEY, this.path.toString()); // Finally, see if the task itself has a valid configuration super.checkConfiguration(invokable); @@ -187,7 +184,7 @@ public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfig * {@inheritDoc} */ @Override - public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) { + public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) { int numberOfBlocks = -1; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java index 2c275317564d9..150190add0484 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobFileOutputVertex.java @@ -28,14 +28,14 @@ import eu.stratosphere.nephele.template.IllegalConfigurationException; /** - * A JobFileOutputVertex is a specific subtype of a {@link JobOutputVertex} and is designed + * A JobFileOutputVertex is a specific subtype of a {@link AbstractJobOutputVertex} and is designed * for Nephele tasks which write data to a local or distributed file system. As every job output vertex * A JobFileOutputVertex must not have any further output. * * @author warneke */ -public class JobFileOutputVertex extends JobGenericOutputVertex -{ +public class JobFileOutputVertex extends AbstractJobOutputVertex { + /** * The path pointing to the output file/directory. */ @@ -51,7 +51,7 @@ public class JobFileOutputVertex extends JobGenericOutputVertex * @param jobGraph * the job graph this vertex belongs to */ - public JobFileOutputVertex(String name, JobVertexID id, JobGraph jobGraph) { + public JobFileOutputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { super(name, id, jobGraph); } @@ -63,7 +63,7 @@ public JobFileOutputVertex(String name, JobVertexID id, JobGraph jobGraph) { * @param jobGraph * the job graph this vertex belongs to */ - public JobFileOutputVertex(String name, JobGraph jobGraph) { + public JobFileOutputVertex(final String name, final JobGraph jobGraph) { super(name, null, jobGraph); } @@ -73,7 +73,7 @@ public JobFileOutputVertex(String name, JobGraph jobGraph) { * @param jobGraph * the job graph this vertex belongs to */ - public JobFileOutputVertex(JobGraph jobGraph) { + public JobFileOutputVertex(final JobGraph jobGraph) { super(null, null, jobGraph); } @@ -83,7 +83,7 @@ public JobFileOutputVertex(JobGraph jobGraph) { * @param path * the path of the file the job file input vertex's task should write to */ - public void setFilePath(Path path) { + public void setFilePath(final Path path) { this.path = path; } @@ -104,8 +104,8 @@ public Path getFilePath() { * @param outputClass * the class of the vertex's output task. */ - public void setFileOutputClass(Class outputClass) { - this.outputClass = outputClass; + public void setFileOutputClass(final Class outputClass) { + this.invokableClass = outputClass; } /** @@ -115,17 +115,16 @@ public void setFileOutputClass(Class outputCla */ @SuppressWarnings("unchecked") public Class getFileOutputClass() { - return (Class) this.outputClass; + return (Class) this.invokableClass; } /** * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException - { + public void read(final DataInput in) throws IOException { super.read(in); - + // Read path of the input file boolean isNotNull = in.readBoolean(); if (isNotNull) { @@ -138,8 +137,7 @@ public void read(DataInput in) throws IOException * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException - { + public void write(final DataOutput out) throws IOException { super.write(out); // Write out the path of the input file @@ -155,7 +153,7 @@ public void write(DataOutput out) throws IOException * {@inheritDoc} */ @Override - public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException { + public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException { // Check if the user has specified a path if (this.path == null) { @@ -169,8 +167,8 @@ public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfig * {@inheritDoc} */ @Override - public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) - { + public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) { + if (this.path == null) { return 0; } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericInputVertex.java deleted file mode 100644 index e39d90d8a2f39..0000000000000 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericInputVertex.java +++ /dev/null @@ -1,182 +0,0 @@ -/*********************************************************************************************************************** - * - * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu) - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - **********************************************************************************************************************/ - -package eu.stratosphere.nephele.jobgraph; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager; -import eu.stratosphere.nephele.template.AbstractInputTask; -import eu.stratosphere.nephele.template.AbstractInvokable; -import eu.stratosphere.nephele.template.IllegalConfigurationException; -import eu.stratosphere.nephele.types.StringRecord; -import eu.stratosphere.nephele.util.StringUtils; - -public class JobGenericInputVertex extends JobInputVertex -{ - /** - * Class of input task. - */ - protected Class> inputClass = null; - - /** - * Creates a new job input vertex with the specified name. - * - * @param name The name of the new job file input vertex. - * @param id The ID of this vertex. - * @param jobGraph The job graph this vertex belongs to. - */ - public JobGenericInputVertex(String name, JobVertexID id, JobGraph jobGraph) { - super(name, id, jobGraph); - } - - /** - * Creates a new job file input vertex with the specified name. - * - * @param name The name of the new job file input vertex. - * @param jobGraph The job graph this vertex belongs to. - */ - public JobGenericInputVertex(String name, JobGraph jobGraph) { - super(name, null, jobGraph); - } - - /** - * Creates a new job file input vertex. - * - * @param jobGraph The job graph this vertex belongs to. - */ - public JobGenericInputVertex(JobGraph jobGraph) { - super(null, null, jobGraph); - } - - /** - * Sets the class of the vertex's input task. - * - * @param inputClass The class of the vertex's input task. - */ - public void setInputClass(Class> inputClass) { - this.inputClass = inputClass; - } - - /** - * Returns the class of the vertex's input task. - * - * @return the class of the vertex's input task or null if no task has yet been set - */ - public Class> getInputClass() { - return this.inputClass; - } - - /** - * {@inheritDoc} - */ - @SuppressWarnings("unchecked") - @Override - public void read(DataInput in) throws IOException - { - super.read(in); - - // Read class - boolean isNotNull = in.readBoolean(); - if (isNotNull) { - // Read the name of the class and try to instantiate the class object - final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID()); - if (cl == null) { - throw new IOException("Cannot find class loader for vertex " + getID()); - } - - // Read the name of the expected class - final String className = StringRecord.readString(in); - - try { - this.inputClass = (Class>) Class.forName(className, true, cl).asSubclass(AbstractInputTask.class); - } - catch (ClassNotFoundException cnfe) { - throw new IOException("Class " + className + " not found in one of the supplied jar files: " - + StringUtils.stringifyException(cnfe)); - } - catch (ClassCastException ccex) { - throw new IOException("Class " + className + " is not a subclass of " - + AbstractInputTask.class.getName() + ": " + StringUtils.stringifyException(ccex)); - } - } - } - - /** - * {@inheritDoc} - */ - @Override - public void write(DataOutput out) throws IOException - { - super.write(out); - - // Write out the name of the class - if (this.inputClass == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - StringRecord.writeString(out, this.inputClass.getName()); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException - { - // see if the task itself has a valid configuration - // because this is user code running on the master, we embed it in a catch-all block - try { - invokable.checkConfiguration(); - } - catch (IllegalConfigurationException icex) { - throw icex; // simply forward - } - catch (Throwable t) { - throw new IllegalConfigurationException("Checking the invokable's configuration caused an error: " - + StringUtils.stringifyException(t)); - } - } - - /** - * {@inheritDoc} - */ - @Override - public Class getInvokableClass() { - - return this.inputClass; - } - - /** - * {@inheritDoc} - */ - @Override - public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) - { - return invokable.getMaximumNumberOfSubtasks(); - } - - /** - * {@inheritDoc} - */ - @Override - public int getMinimumNumberOfSubtasks(AbstractInvokable invokable) { - - return invokable.getMinimumNumberOfSubtasks(); - } -} diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericOutputVertex.java deleted file mode 100644 index 6edbaba78bb80..0000000000000 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGenericOutputVertex.java +++ /dev/null @@ -1,197 +0,0 @@ -/*********************************************************************************************************************** - * - * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu) - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - **********************************************************************************************************************/ - -package eu.stratosphere.nephele.jobgraph; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager; -import eu.stratosphere.nephele.template.AbstractInvokable; -import eu.stratosphere.nephele.template.AbstractOutputTask; -import eu.stratosphere.nephele.template.IllegalConfigurationException; -import eu.stratosphere.nephele.types.StringRecord; -import eu.stratosphere.nephele.util.StringUtils; - -/** - * A JobGenericOutputVertex is a specific subtype of a {@link JobOutputVertex} and is designed - * for Nephele tasks which sink data in a not further specified way. As every job output vertex, - * a JobGenericOutputVertex must not have any further output. - * - * @author warneke - */ -public class JobGenericOutputVertex extends JobOutputVertex { - - /** - * The class of the output task. - */ - protected Class outputClass = null; - - - /** - * Creates a new job file output vertex with the specified name. - * - * @param name - * the name of the new job file output vertex - * @param id - * the ID of this vertex - * @param jobGraph - * the job graph this vertex belongs to - */ - public JobGenericOutputVertex(String name, JobVertexID id, JobGraph jobGraph) { - super(name, id, jobGraph); - } - - /** - * Creates a new job file output vertex with the specified name. - * - * @param name - * the name of the new job file output vertex - * @param jobGraph - * the job graph this vertex belongs to - */ - public JobGenericOutputVertex(String name, JobGraph jobGraph) { - super(name, null, jobGraph); - } - - /** - * Creates a new job file input vertex. - * - * @param jobGraph - * the job graph this vertex belongs to - */ - public JobGenericOutputVertex(JobGraph jobGraph) { - super(null, null, jobGraph); - } - - /** - * Sets the class of the vertex's output task. - * - * @param outputClass The class of the vertex's output task. - */ - public void setOutputClass(Class outputClass) { - this.outputClass = outputClass; - } - - /** - * Returns the class of the vertex's output task. - * - * @return The class of the vertex's output task or null if no task has yet been set. - */ - public Class getOutputClass() { - return this.outputClass; - } - - /** - * {@inheritDoc} - */ - @Override - public void read(DataInput in) throws IOException { - super.read(in); - - // Read class - boolean isNotNull = in.readBoolean(); - if (isNotNull) { - - // Read the name of the class and try to instantiate the class object - final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID()); - if (cl == null) { - throw new IOException("Cannot find class loader for vertex " + getID()); - } - - // Read the name of the expected class - final String className = StringRecord.readString(in); - - try { - this.outputClass = Class.forName(className, true, cl).asSubclass(AbstractOutputTask.class); - } - catch (ClassNotFoundException cnfe) { - throw new IOException("Class " + className + " not found in one of the supplied jar files: " - + StringUtils.stringifyException(cnfe)); - } - catch (ClassCastException ccex) { - throw new IOException("Class " + className + " is not a subclass of " - + AbstractOutputTask.class.getName() + ": " + StringUtils.stringifyException(ccex)); - } - } - } - - /** - * {@inheritDoc} - */ - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - - // Write out the name of the class - if (this.outputClass == null) { - out.writeBoolean(false); - } - else { - out.writeBoolean(true); - StringRecord.writeString(out, this.outputClass.getName()); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException - { - // see if the task itself has a valid configuration - // because this is user code running on the master, we embed it in a catch-all block - try { - invokable.checkConfiguration(); - } - catch (IllegalConfigurationException icex) { - throw icex; // simply forward - } - catch (Throwable t) { - throw new IllegalConfigurationException("Checking the invokable's configuration caused an error: " - + StringUtils.stringifyException(t)); - } - } - - /** - * {@inheritDoc} - */ - @Override - public Class getInvokableClass() { - - return this.outputClass; - } - - /** - * {@inheritDoc} - */ - @Override - public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) - { - // Delegate call to invokable - return invokable.getMaximumNumberOfSubtasks(); - } - - /** - * {@inheritDoc} - */ - @Override - public int getMinimumNumberOfSubtasks(AbstractInvokable invokable) - { - // Delegate call to invokable - return invokable.getMinimumNumberOfSubtasks(); - } -} diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java index 889e6b8df7ed3..702197450c4c0 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobGraph.java @@ -52,12 +52,12 @@ public class JobGraph implements IOReadableWritable { /** * List of input vertices included in this job graph. */ - private Map inputVertices = new HashMap(); + private Map inputVertices = new HashMap(); /** * List of output vertices included in this job graph. */ - private Map outputVertices = new HashMap(); + private Map outputVertices = new HashMap(); /** * List of task vertices included in this job graph. @@ -107,7 +107,7 @@ public JobGraph() { * @param jobName * the name for this job graph */ - public JobGraph(String jobName) { + public JobGraph(final String jobName) { this(); this.jobName = jobName; } @@ -148,7 +148,7 @@ public Configuration getTaskmanagerConfiguration() { * @param inputVertex * the new input vertex to be added */ - public void addVertex(JobInputVertex inputVertex) { + public void addVertex(final AbstractJobInputVertex inputVertex) { if (!inputVertices.containsKey(inputVertex.getID())) { inputVertices.put(inputVertex.getID(), inputVertex); @@ -161,7 +161,7 @@ public void addVertex(JobInputVertex inputVertex) { * @param taskVertex * the new task vertex to be added */ - public void addVertex(JobTaskVertex taskVertex) { + public void addVertex(final JobTaskVertex taskVertex) { if (!taskVertices.containsKey(taskVertex.getID())) { taskVertices.put(taskVertex.getID(), taskVertex); @@ -174,7 +174,7 @@ public void addVertex(JobTaskVertex taskVertex) { * @param outputVertex * the new output vertex to be added */ - public void addVertex(JobOutputVertex outputVertex) { + public void addVertex(final AbstractJobOutputVertex outputVertex) { if (!outputVertices.containsKey(outputVertex.getID())) { outputVertices.put(outputVertex.getID(), outputVertex); @@ -213,9 +213,9 @@ public int getNumberOfTaskVertices() { * * @return an iterator to iterate all input vertices registered with the job graph */ - public Iterator getInputVertices() { + public Iterator getInputVertices() { - final Collection coll = this.inputVertices.values(); + final Collection coll = this.inputVertices.values(); return coll.iterator(); } @@ -225,9 +225,9 @@ public Iterator getInputVertices() { * * @return an iterator to iterate all output vertices registered with the job graph */ - public Iterator getOutputVertices() { + public Iterator getOutputVertices() { - final Collection coll = this.outputVertices.values(); + final Collection coll = this.outputVertices.values(); return coll.iterator(); } @@ -278,12 +278,12 @@ public AbstractJobVertex[] getAllJobVertices() { final AbstractJobVertex[] vertices = new AbstractJobVertex[inputVertices.size() + outputVertices.size() + taskVertices.size()]; - final Iterator iv = getInputVertices(); + final Iterator iv = getInputVertices(); while (iv.hasNext()) { vertices[i++] = iv.next(); } - final Iterator ov = getOutputVertices(); + final Iterator ov = getOutputVertices(); while (ov.hasNext()) { vertices[i++] = ov.next(); } @@ -304,10 +304,10 @@ public AbstractJobVertex[] getAllJobVertices() { * @param collector * a temporary list to store the vertices that have already been visisted */ - private void collectVertices(AbstractJobVertex jv, List collector) { + private void collectVertices(final AbstractJobVertex jv, final List collector) { if (jv == null) { - final Iterator iter = getInputVertices(); + final Iterator iter = getInputVertices(); while (iter.hasNext()) { collectVertices(iter.next(), collector); } @@ -341,18 +341,18 @@ public JobID getJobID() { * the ID of the vertex to search for * @return the vertex with the matching ID or null if no vertex with such ID could be found */ - public AbstractJobVertex findVertexByID(JobVertexID id) { + public AbstractJobVertex findVertexByID(final JobVertexID id) { - if (inputVertices.containsKey(id)) { - return inputVertices.get(id); + if (this.inputVertices.containsKey(id)) { + return this.inputVertices.get(id); } - if (outputVertices.containsKey(id)) { - return outputVertices.get(id); + if (this.outputVertices.containsKey(id)) { + return this.outputVertices.get(id); } - if (taskVertices.containsKey(id)) { - return taskVertices.get(id); + if (this.taskVertices.containsKey(id)) { + return this.taskVertices.get(id); } return null; @@ -366,17 +366,17 @@ public AbstractJobVertex findVertexByID(JobVertexID id) { * @return true if a vertex with the given ID is registered with the job graph, false * otherwise. */ - private boolean includedInJobGraph(JobVertexID id) { + private boolean includedInJobGraph(final JobVertexID id) { - if (inputVertices.containsKey(id)) { + if (this.inputVertices.containsKey(id)) { return true; } - if (outputVertices.containsKey(id)) { + if (this.outputVertices.containsKey(id)) { return true; } - if (taskVertices.containsKey(id)) { + if (this.taskVertices.containsKey(id)) { return true; } @@ -449,8 +449,9 @@ public boolean isAcyclic() { * Auxiliary method implementing Tarjan's algorithm for strongly-connected components to determine whether the job * graph is acyclic. */ - private boolean tarjan(AbstractJobVertex jv, Integer index, HashMap indexMap, - HashMap lowLinkMap, Stack stack) { + private boolean tarjan(final AbstractJobVertex jv, Integer index, + final HashMap indexMap, final HashMap lowLinkMap, + final Stack stack) { indexMap.put(jv, Integer.valueOf(index)); lowLinkMap.put(jv, Integer.valueOf(index)); @@ -501,7 +502,7 @@ private boolean tarjan(AbstractJobVertex jv, Integer index, HashMap iter = getInputVertices(); + final Iterator iter = getInputVertices(); while (iter.hasNext()) { final AbstractJobVertex jv = iter.next(); @@ -523,7 +524,7 @@ public AbstractJobVertex areVertexDegreesCorrect() { } // Check output vertices - final Iterator iter3 = getOutputVertices(); + final Iterator iter3 = getOutputVertices(); while (iter3.hasNext()) { final AbstractJobVertex jv = iter3.next(); @@ -540,7 +541,7 @@ public AbstractJobVertex areVertexDegreesCorrect() { * {@inheritDoc} */ @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { // Read job id this.jobID.read(in); @@ -625,7 +626,7 @@ public void read(DataInput in) throws IOException { * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { // Write job ID this.jobID.write(out); @@ -671,7 +672,7 @@ public void write(DataOutput out) throws IOException { * @throws IOException * thrown if an error occurs while writing to the stream */ - private void writeRequiredJarFiles(DataOutput out, AbstractJobVertex[] jobVertices) throws IOException { + private void writeRequiredJarFiles(final DataOutput out, final AbstractJobVertex[] jobVertices) throws IOException { // Now check if all the collected jar files really exist final FileSystem fs = FileSystem.getLocalFileSystem(); @@ -716,7 +717,7 @@ private void writeRequiredJarFiles(DataOutput out, AbstractJobVertex[] jobVertic * @throws IOException * thrown if an error occurs while reading the stream */ - private void readRequiredJarFiles(DataInput in) throws IOException { + private void readRequiredJarFiles(final DataInput in) throws IOException { // Do jar files follow; final int numJars = in.readInt(); @@ -748,7 +749,7 @@ private void readRequiredJarFiles(DataInput in) throws IOException { * @param jar * path of the JAR file required to run the job on a task manager */ - public void addJar(Path jar) { + public void addJar(final Path jar) { if (jar == null) { return; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java index df1377557adb6..1a17fe66d43fb 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobID.java @@ -23,7 +23,7 @@ * * @author warneke */ -public class JobID extends AbstractID { +public final class JobID extends AbstractID { /** * Constructs a new random ID from a uniform distribution. */ @@ -33,8 +33,11 @@ public JobID() { /** * Constructs a new ID with a specific bytes value. + * + * @param bytes + * the ID in byte representation */ - public JobID(byte[] bytes) { + public JobID(final byte[] bytes) { super(bytes); } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java index f7066877b83f9..a610eb6ef1e4b 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobInputVertex.java @@ -15,46 +15,63 @@ package eu.stratosphere.nephele.jobgraph; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; +import eu.stratosphere.nephele.template.AbstractInputTask; -/** - * An abstract base class for input vertices in Nephele. - * - * @author warneke - */ -public abstract class JobInputVertex extends AbstractJobVertex { +public class JobInputVertex extends AbstractJobInputVertex { /** - * Constructs a new job input vertex with the given name. + * Creates a new job input vertex with the specified name. * * @param name - * the name of the new job input vertex + * The name of the new job file input vertex. * @param id - * the ID of this vertex + * The ID of this vertex. * @param jobGraph - * the job graph this vertex belongs to + * The job graph this vertex belongs to. */ - protected JobInputVertex(String name, JobVertexID id, JobGraph jobGraph) { + public JobInputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { super(name, id, jobGraph); + } + + /** + * Creates a new job file input vertex with the specified name. + * + * @param name + * The name of the new job file input vertex. + * @param jobGraph + * The job graph this vertex belongs to. + */ + public JobInputVertex(final String name, final JobGraph jobGraph) { + super(name, null, jobGraph); + } - jobGraph.addVertex(this); + /** + * Creates a new job file input vertex. + * + * @param jobGraph + * The job graph this vertex belongs to. + */ + public JobInputVertex(final JobGraph jobGraph) { + super(null, null, jobGraph); } /** - * {@inheritDoc} + * Sets the class of the vertex's input task. + * + * @param inputClass + * The class of the vertex's input task. */ - @Override - public void read(DataInput in) throws IOException { - super.read(in); + public void setInputClass(final Class> inputClass) { + this.invokableClass = inputClass; } /** - * {@inheritDoc} + * Returns the class of the vertex's input task. + * + * @return the class of the vertex's input task or null if no task has yet been set */ - @Override - public void write(DataOutput out) throws IOException { - super.write(out); + @SuppressWarnings("unchecked") + public Class> getInputClass() { + return (Class>) this.invokableClass; } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java index b97375351013c..1519ff466bbfb 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobOutputVertex.java @@ -15,26 +15,70 @@ package eu.stratosphere.nephele.jobgraph; +import eu.stratosphere.nephele.template.AbstractOutputTask; + /** - * An abstract base class for output vertices in Nephele. + * A JobOutputVertex is a specific subtype of a {@link AbstractJobOutputVertex} and is designed + * for Nephele tasks which sink data in a not further specified way. As every job output vertex, + * a JobOutputVertex must not have any further output. * * @author warneke */ -public abstract class JobOutputVertex extends AbstractJobVertex { +public class JobOutputVertex extends AbstractJobOutputVertex { /** - * Constructs a new job output vertex with the given name. + * Creates a new job file output vertex with the specified name. * * @param name - * the name of the new job output vertex + * the name of the new job file output vertex * @param id * the ID of this vertex * @param jobGraph * the job graph this vertex belongs to */ - protected JobOutputVertex(String name, JobVertexID id, JobGraph jobGraph) { + public JobOutputVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { super(name, id, jobGraph); + } - jobGraph.addVertex(this); + /** + * Creates a new job file output vertex with the specified name. + * + * @param name + * the name of the new job file output vertex + * @param jobGraph + * the job graph this vertex belongs to + */ + public JobOutputVertex(final String name, final JobGraph jobGraph) { + super(name, null, jobGraph); + } + + /** + * Creates a new job file input vertex. + * + * @param jobGraph + * the job graph this vertex belongs to + */ + public JobOutputVertex(final JobGraph jobGraph) { + super(null, null, jobGraph); + } + + /** + * Sets the class of the vertex's output task. + * + * @param outputClass + * The class of the vertex's output task. + */ + public void setOutputClass(final Class outputClass) { + this.invokableClass = outputClass; + } + + /** + * Returns the class of the vertex's output task. + * + * @return The class of the vertex's output task or null if no task has yet been set. + */ + @SuppressWarnings("unchecked") + public Class getOutputClass() { + return (Class) this.invokableClass; } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java index 6a46f757181ae..2e5fc70e7f40d 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobStatus.java @@ -31,7 +31,7 @@ public enum JobStatus { CREATED, /** - * All tasks of the job have been accepted by the scheduler, resources have been requested + * All tasks of the job have been accepted by the scheduler, resources have been requested. */ SCHEDULED, @@ -55,4 +55,4 @@ public enum JobStatus { * All of the job's tasks have successfully finished. */ FINISHED -}; \ No newline at end of file +}; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java index 437f2cf979760..ea777e2465854 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/jobgraph/JobTaskVertex.java @@ -15,16 +15,8 @@ package eu.stratosphere.nephele.jobgraph; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager; import eu.stratosphere.nephele.template.AbstractInvokable; import eu.stratosphere.nephele.template.AbstractTask; -import eu.stratosphere.nephele.template.IllegalConfigurationException; -import eu.stratosphere.nephele.types.StringRecord; -import eu.stratosphere.nephele.util.StringUtils; /** * A JobTaskVertex is the vertex type for regular tasks (with both input and output) in Nephele. @@ -34,11 +26,6 @@ */ public class JobTaskVertex extends AbstractJobVertex { - /** - * The task attached to this vertex. - */ - private Class taskClass = null; - /** * Creates a new job task vertex with the specified name. * @@ -49,7 +36,7 @@ public class JobTaskVertex extends AbstractJobVertex { * @param jobGraph * the job graph this vertex belongs to */ - public JobTaskVertex(String name, JobVertexID id, JobGraph jobGraph) { + public JobTaskVertex(final String name, final JobVertexID id, final JobGraph jobGraph) { super(name, id, jobGraph); jobGraph.addVertex(this); @@ -63,7 +50,7 @@ public JobTaskVertex(String name, JobVertexID id, JobGraph jobGraph) { * @param jobGraph * the job graph this vertex belongs to */ - public JobTaskVertex(String name, JobGraph jobGraph) { + public JobTaskVertex(final String name, final JobGraph jobGraph) { super(name, null, jobGraph); jobGraph.addVertex(this); @@ -75,7 +62,7 @@ public JobTaskVertex(String name, JobGraph jobGraph) { * @param jobGraph * the job graph this vertex belongs to */ - public JobTaskVertex(JobGraph jobGraph) { + public JobTaskVertex(final JobGraph jobGraph) { super(null, null, jobGraph); jobGraph.addVertex(this); @@ -87,8 +74,8 @@ public JobTaskVertex(JobGraph jobGraph) { * @param taskClass * the class of the vertex's task */ - public void setTaskClass(Class taskClass) { - this.taskClass = taskClass; + public void setTaskClass(final Class taskClass) { + this.invokableClass = taskClass; } /** @@ -96,75 +83,16 @@ public void setTaskClass(Class taskClass) { * * @return the class of the vertex's task or null if the class has not yet been set */ - public Class getTaskClass() { - return this.taskClass; - } - - /** - * {@inheritDoc} - */ @SuppressWarnings("unchecked") - @Override - public void read(DataInput in) throws IOException { - super.read(in); - - final boolean isNotNull = in.readBoolean(); - if (!isNotNull) { - return; - } - - // Read the name of the class and try to instantiate the class object - - final ClassLoader cl = LibraryCacheManager.getClassLoader(this.getJobGraph().getJobID()); - if (cl == null) { - throw new IOException("Cannot find class loader for vertex " + getID()); - } - - // Read the name of the expected class - final String className = StringRecord.readString(in); - - try { - this.taskClass = (Class) Class.forName(className, true, cl); - } catch (ClassNotFoundException cnfe) { - throw new IOException("Class " + className + " not found in one of the supplied jar files: " - + StringUtils.stringifyException(cnfe)); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - - if (this.taskClass == null) { - out.writeBoolean(false); - return; - } - - out.writeBoolean(true); - - // Write out the name of the class - StringRecord.writeString(out, this.taskClass.getName()); - - } - - /** - * {@inheritDoc} - */ - @Override - public void checkConfiguration(AbstractInvokable invokable) throws IllegalConfigurationException { - - // Delegate check to invokable - invokable.checkConfiguration(); + public Class getTaskClass() { + return (Class) this.invokableClass; } /** * {@inheritDoc} */ @Override - public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) { + public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) { // Delegate call to invokable return invokable.getMaximumNumberOfSubtasks(); @@ -174,18 +102,9 @@ public int getMaximumNumberOfSubtasks(AbstractInvokable invokable) { * {@inheritDoc} */ @Override - public int getMinimumNumberOfSubtasks(AbstractInvokable invokable) { + public int getMinimumNumberOfSubtasks(final AbstractInvokable invokable) { // Delegate call to invokable return invokable.getMinimumNumberOfSubtasks(); } - - /** - * {@inheritDoc} - */ - @Override - public Class getInvokableClass() { - - return this.taskClass; - } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java index 6df7323caf154..4c74139b7c67f 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/ServiceException.java @@ -16,21 +16,22 @@ package eu.stratosphere.nephele.services; public class ServiceException extends Exception { + private static final long serialVersionUID = 8751768275700167972L; protected ServiceException() { super(); } - - public ServiceException(String message) { + + public ServiceException(final String message) { super(message); } - public ServiceException(Throwable cause) { + public ServiceException(final Throwable cause) { super(cause); } - - public ServiceException(String message, Throwable cause) { + + public ServiceException(final String message, final Throwable cause) { super(message, cause); } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java index c843d41489578..ae5d107ebc453 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventDispatcher.java @@ -22,16 +22,16 @@ public class EventDispatcher { public final Set> handlers; public EventDispatcher() { - handlers = new HashSet>(); + this.handlers = new HashSet>(); } - public void register(EventHandler handler) { + public void register(final EventHandler handler) { } - public void unregister(EventHandler handler) { - handlers.remove(handler); + public void unregister(final EventHandler handler) { + this.handlers.remove(handler); } - protected void dispatch(T event) { + protected void dispatch(final T event) { } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java index 506e600287de2..a97f32f6f0bf1 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/event/EventHandler.java @@ -16,5 +16,5 @@ package eu.stratosphere.nephele.services.event; public interface EventHandler { - public void handleEvent(U event); + void handleEvent(U event); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java index b9b0f36f944fa..39a637f87152a 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/services/iomanager/Channel.java @@ -26,8 +26,7 @@ * * @author Alexander Alexandrov */ -public final class Channel -{ +public final class Channel { private static final int RANDOM_BYTES_LENGTH = 16; /** @@ -35,15 +34,14 @@ public final class Channel * * @author Alexander Alexandrov */ - public static class ID - { + public static class ID { private final String path; - protected ID(String path) { + protected ID(final String path) { this.path = path; } - protected ID(String basePath, Random random) { + protected ID(final String basePath, final Random random) { this.path = basePath + File.separator + randomString(random) + ".channel"; } @@ -61,21 +59,20 @@ public String toString() { } } - public static final class Enumerator - { - private static final String format = "%s.%06d.channel"; + public static final class Enumerator { + private static final String FORMAT = "%s.%06d.channel"; private final String pathPrefix; private int counter; - protected Enumerator(String basePath, Random random) { + protected Enumerator(final String basePath, final Random random) { this.pathPrefix = basePath + File.separator + randomString(random); this.counter = 0; } public ID next() { - return new ID(String.format(format, pathPrefix, ++counter)); + return new ID(String.format(FORMAT, pathPrefix, ++counter)); } } @@ -86,9 +83,8 @@ public ID next() { * The random number generator to be used. * @return A hex representation of the generated byte sequence */ - private static final String randomString(Random random) - { - byte[] bytes = new byte[RANDOM_BYTES_LENGTH]; + private static final String randomString(final Random random) { + final byte[] bytes = new byte[RANDOM_BYTES_LENGTH]; random.nextBytes(bytes); return StringUtils.byteToHexString(bytes); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java index 5a42815d23878..af3df12a2a392 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractFileInputTask.java @@ -35,6 +35,7 @@ * @author warneke */ public abstract class AbstractFileInputTask extends AbstractInputTask { + public static final String INPUT_PATH_CONFIG_KEY = "input.path"; /** @@ -59,8 +60,8 @@ public Iterator getFileInputSplits() { * {@inheritDoc} */ @Override - public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException { - + public FileInputSplit[] computeInputSplits(final int minNumSplits) throws IOException { + final String pathURI = getRuntimeConfiguration().getString(INPUT_PATH_CONFIG_KEY, null); if (pathURI == null) { throw new IOException("The path to the file was not found in the runtime configuration."); @@ -76,7 +77,7 @@ public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException final List inputSplits = new ArrayList(); // get all the files that are involved in the splits - List files = new ArrayList(); + final List files = new ArrayList(); long totalLength = 0; final FileSystem fs = path.getFileSystem(); @@ -128,7 +129,7 @@ public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException // get the block containing the majority of the data blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex); // create a new split - FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, splitSize, + final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, splitSize, blocks[blockIndex] .getHosts()); inputSplits.add(fis); @@ -175,7 +176,9 @@ public FileInputSplit[] computeInputSplits(int minNumSplits) throws IOException * The earliest index to look at. * @return The index of the block containing the given position. */ - private final int getBlockIndexForPosition(BlockLocation[] blocks, long offset, long halfSplitSize, int startIndex) { + private final int getBlockIndexForPosition(final BlockLocation[] blocks, final long offset, + final long halfSplitSize, final int startIndex) { + // go over all indexes after the startIndex for (int i = startIndex; i < blocks.length; i++) { long blockStart = blocks[i].getOffset(); @@ -193,13 +196,13 @@ private final int getBlockIndexForPosition(BlockLocation[] blocks, long offset, } throw new IllegalArgumentException("The given offset is not contained in the any block."); } - + /** * {@inheritDoc} */ @Override public Class getInputSplitType() { - + return FileInputSplit.class; } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputTask.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractGenericInputTask.java similarity index 93% rename from nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputTask.java rename to nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractGenericInputTask.java index 290806f41a516..2c588b6e93091 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputTask.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractGenericInputTask.java @@ -18,7 +18,7 @@ /** * An input task that processes generic input splits (partitions). */ -public abstract class GenericInputTask extends AbstractInputTask { +public abstract class AbstractGenericInputTask extends AbstractInputTask { /** * {@inheritDoc} diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java index 9800ee0249f57..9dd8b34254c6b 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/AbstractInputTask.java @@ -49,34 +49,38 @@ public abstract class AbstractInputTask extends AbstractIn * @return the type of input splits that is generated by this input task */ public abstract Class getInputSplitType(); - + /** * Returns an iterator to a (possible empty) list of input splits which is expected to be consumed by this * instance of the {@link AbstractInputTask}. * * @return an iterator to a (possible empty) list of input splits. */ - public Iterator getInputSplits() - { + public Iterator getInputSplits() { + final InputSplitProvider provider = getEnvironment().getInputSplitProvider(); + return new Iterator() { private T nextSplit; - + @Override public boolean hasNext() { + if (this.nextSplit == null) { - InputSplit split = provider.getNextInputSplit(); - if (split != null) - { + + final InputSplit split = provider.getNextInputSplit(); + if (split != null) { @SuppressWarnings("unchecked") - T tSplit = (T) split; + final T tSplit = (T) split; this.nextSplit = tSplit; return true; + } else { + return false; } - else return false; + } else { + return true; } - else return true; } @Override @@ -84,16 +88,16 @@ public T next() { if (this.nextSplit == null && !hasNext()) { throw new NoSuchElementException(); } - - T tmp = this.nextSplit; + + final T tmp = this.nextSplit; this.nextSplit = null; return tmp; } @Override public void remove() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException(); } }; } -} \ No newline at end of file +} diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java index 9ea1e3d4187cd..3797a26bc473d 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/GenericInputSplit.java @@ -19,67 +19,68 @@ import java.io.DataOutput; import java.io.IOException; - /** * A generic input split that has only a partition number. */ -public class GenericInputSplit implements InputSplit -{ - protected int partitionNumber; - +public class GenericInputSplit implements InputSplit { + + /** + * The number of this split. + */ + protected int number; + // -------------------------------------------------------------------------------------------- - + /** * Default constructor for instantiation during de-serialization. */ - public GenericInputSplit() - {} - + public GenericInputSplit() { + } + /** - * Creates a generic input split with the given partition number. + * Creates a generic input split with the given split number. * - * @param partitionNumber The partition number of the split. + * @param number + * the number of the split */ - public GenericInputSplit(int partitionNumber) - { - this.partitionNumber = partitionNumber; + public GenericInputSplit(final int number) { + this.number = number; } - - - // -------------------------------------------------------------------------------------------- - - /* (non-Javadoc) + + // -------------------------------------------------------------------------------------------- + + /* + * (non-Javadoc) * @see eu.stratosphere.nephele.io.IOReadableWritable#write(java.io.DataOutput) */ @Override - public void write(DataOutput out) throws IOException - { - out.writeInt(this.partitionNumber); + public void write(final DataOutput out) throws IOException { + out.writeInt(this.number); } - /* (non-Javadoc) + /* + * (non-Javadoc) * @see eu.stratosphere.nephele.io.IOReadableWritable#read(java.io.DataInput) */ @Override - public void read(DataInput in) throws IOException - { - this.partitionNumber = in.readInt(); + public void read(final DataInput in) throws IOException { + this.number = in.readInt(); } - /* (non-Javadoc) + /* + * (non-Javadoc) * @see eu.stratosphere.nephele.template.InputSplit#getPartitionNumber() */ @Override - public int getPartitionNumber() - { - return this.partitionNumber; + public int getSplitNumber() { + return this.number; } - /* (non-Javadoc) + /* + * (non-Javadoc) * @see java.lang.Object#toString() */ - public String toString() - { - return "[" + this.partitionNumber + "]"; + public String toString() { + return "[" + this.number + "]"; } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java index e3a01f48d41fd..2e86f4b052c8c 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/IllegalConfigurationException.java @@ -36,18 +36,20 @@ public class IllegalConfigurationException extends Exception { * @param errorMsg * the error message to be included in the exception */ - public IllegalConfigurationException(String errorMsg) { + public IllegalConfigurationException(final String errorMsg) { super(errorMsg); } - + /** * Constructs an new illegal configuration exception with the given error message * and a given cause. * - * @param errorMsg The error message to be included in the exception. - * @param cause The exception that caused this exception. + * @param errorMsg + * The error message to be included in the exception. + * @param cause + * The exception that caused this exception. */ - public IllegalConfigurationException(String errorMsg, Throwable cause) { + public IllegalConfigurationException(final String errorMsg, final Throwable cause) { super(errorMsg, cause); } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java index ed3463a3351a6..70fb7a19d6b56 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/template/InputSplit.java @@ -22,12 +22,11 @@ * * @author warneke */ -public interface InputSplit extends IOReadableWritable -{ +public interface InputSplit extends IOReadableWritable { /** - * Gets the partition number of this input split. - * - * @return The number of this input split. + * Returns the number of this input split. + * + * @return the number of this input split */ - public int getPartitionNumber(); + int getSplitNumber(); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java index 5e727bfb550dd..2003ff3d5bf42 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/FileRecord.java @@ -33,12 +33,12 @@ public FileRecord() { fileName = "empty"; } - public FileRecord(String fileName) { + public FileRecord(final String fileName) { this.bytes = EMPTY_BYTES; this.fileName = fileName; } - public void setFileName(String fileName) { + public void setFileName(final String fileName) { this.fileName = fileName; } @@ -60,13 +60,14 @@ public byte[] getDataBuffer() { * @param len * the number of bytes to append */ - public void append(byte[] data, int start, int len) { + public void append(final byte[] data, final int start, final int len) { final int oldLength = this.bytes.length; setCapacity(this.bytes.length + len, true); System.arraycopy(data, start, this.bytes, oldLength, len); } - private void setCapacity(int len, boolean keepData) { + private void setCapacity(final int len, final boolean keepData) { + if (this.bytes == null || this.bytes.length < len) { final byte[] newBytes = new byte[len]; if (this.bytes != null && keepData) { diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java index 78237a4120649..bf5cbfa61f31f 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/IntegerRecord.java @@ -37,7 +37,7 @@ public class IntegerRecord implements Record { * @param value * the integer value this record should wrap up */ - public IntegerRecord(int value) { + public IntegerRecord(final int value) { this.value = value; } @@ -63,7 +63,7 @@ public int getValue() { * @param value * the new value for this integer record */ - public void setValue(int value) { + public void setValue(final int value) { this.value = value; } @@ -91,7 +91,7 @@ public void write(final DataOutput out) throws IOException { * {@inheritDoc} */ @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (!(obj instanceof IntegerRecord)) { return false; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java index 0703d4c861e95..05ce666c19bc4 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/types/StringRecord.java @@ -48,14 +48,14 @@ */ public class StringRecord implements Record { - private static ThreadLocal ENCODER_FACTORY = new ThreadLocal() { + private static final ThreadLocal ENCODER_FACTORY = new ThreadLocal() { protected CharsetEncoder initialValue() { return Charset.forName("UTF-8").newEncoder().onMalformedInput(CodingErrorAction.REPORT) .onUnmappableCharacter(CodingErrorAction.REPORT); } }; - private static ThreadLocal DECODER_FACTORY = new ThreadLocal() { + private static final ThreadLocal DECODER_FACTORY = new ThreadLocal() { protected CharsetDecoder initialValue() { return Charset.forName("UTF-8").newDecoder().onMalformedInput(CodingErrorAction.REPORT) .onUnmappableCharacter(CodingErrorAction.REPORT); @@ -75,19 +75,19 @@ public StringRecord() { /** * Construct from a string. */ - public StringRecord(String string) { + public StringRecord(final String string) { set(string); } /** Construct from another text. */ - public StringRecord(StringRecord utf8) { + public StringRecord(final StringRecord utf8) { set(utf8); } /** * Construct from a byte array. */ - public StringRecord(byte[] utf8) { + public StringRecord(final byte[] utf8) { set(utf8); } @@ -119,7 +119,7 @@ public int charAt(final int position) { return bytesToCodePoint(bb.slice()); } - public int find(String what) { + public int find(final String what) { return find(what, 0); } @@ -175,7 +175,7 @@ public int find(final String what, final int start) { /** * Set to contain the contents of a string. */ - public void set(String string) { + public void set(final String string) { try { final ByteBuffer bb = encode(string, true); this.bytes = bb.array(); @@ -188,12 +188,12 @@ public void set(String string) { /** * Set to a utf8 byte array */ - public void set(byte[] utf8) { + public void set(final byte[] utf8) { set(utf8, 0, utf8.length); } /** copy a text. */ - public void set(StringRecord other) { + public void set(final StringRecord other) { set(other.getBytes(), 0, other.getLength()); } @@ -212,7 +212,7 @@ public int getLength() { * @param len * the number of bytes of the new string */ - public void set(byte[] utf8, int start, int len) { + public void set(final byte[] utf8, final int start, final int len) { setCapacity(len, false); System.arraycopy(utf8, start, bytes, 0, len); this.length = len; @@ -228,7 +228,7 @@ public void set(byte[] utf8, int start, int len) { * @param len * the number of bytes to append */ - public void append(byte[] utf8, int start, int len) { + public void append(final byte[] utf8, final int start, final int len) { setCapacity(length + len, true); System.arraycopy(utf8, start, bytes, length, len); this.length += len; @@ -250,7 +250,7 @@ public void clear() { * @param len the number of bytes we need * @param keepData should the old data be kept */ - private void setCapacity(int len, boolean keepData) { + private void setCapacity(final int len, final boolean keepData) { if (this.bytes == null || this.bytes.length < len) { final byte[] newBytes = new byte[len]; if (this.bytes != null && keepData) { @@ -276,7 +276,7 @@ public String toString() { /** * deserialize */ - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { final int newLength = in.readInt(); setCapacity(newLength, false); in.readFully(this.bytes, 0, newLength); @@ -284,12 +284,12 @@ public void read(DataInput in) throws IOException { } /** Skips over one Text in the input. */ - public static void skip(DataInput in) throws IOException { + public static void skip(final DataInput in) throws IOException { final int length = in.readInt(); skipFully(in, length); } - public static void skipFully(DataInput in, int len) throws IOException { + public static void skipFully(final DataInput in, final int len) throws IOException { int total = 0; int cur = 0; @@ -307,7 +307,7 @@ public static void skipFully(DataInput in, int len) throws IOException { * * @see Writable#read(DataOutput) */ - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { out.writeInt(this.length); out.write(this.bytes, 0, this.length); } @@ -357,11 +357,11 @@ public int hashCode() { * Converts the provided byte array to a String using the UTF-8 encoding. If * the input is malformed, replace by a default value. */ - public static String decode(byte[] utf8) throws CharacterCodingException { + public static String decode(final byte[] utf8) throws CharacterCodingException { return decode(ByteBuffer.wrap(utf8), true); } - public static String decode(byte[] utf8, int start, int length) throws CharacterCodingException { + public static String decode(final byte[] utf8, final int start, final int length) throws CharacterCodingException { return decode(ByteBuffer.wrap(utf8, start, length), true); } @@ -371,11 +371,12 @@ public static String decode(byte[] utf8, int start, int length) throws Character * substitution character, which is U+FFFD. Otherwise the method throws a * MalformedInputException. */ - public static String decode(byte[] utf8, int start, int length, boolean replace) throws CharacterCodingException { + public static String decode(final byte[] utf8, final int start, final int length, final boolean replace) + throws CharacterCodingException { return decode(ByteBuffer.wrap(utf8, start, length), replace); } - private static String decode(ByteBuffer utf8, boolean replace) throws CharacterCodingException { + private static String decode(final ByteBuffer utf8, final boolean replace) throws CharacterCodingException { final CharsetDecoder decoder = DECODER_FACTORY.get(); if (replace) { decoder.onMalformedInput(java.nio.charset.CodingErrorAction.REPLACE); @@ -398,7 +399,7 @@ private static String decode(ByteBuffer utf8, boolean replace) throws CharacterC * ByteBuffer.limit() */ - public static ByteBuffer encode(String string) throws CharacterCodingException { + public static ByteBuffer encode(final String string) throws CharacterCodingException { return encode(string, true); } @@ -411,7 +412,8 @@ public static ByteBuffer encode(String string) throws CharacterCodingException { * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is * ByteBuffer.limit() */ - public static ByteBuffer encode(String string, boolean replace) throws CharacterCodingException { + public static ByteBuffer encode(final String string, final boolean replace) throws CharacterCodingException { + final CharsetEncoder encoder = ENCODER_FACTORY.get(); if (replace) { encoder.onMalformedInput(CodingErrorAction.REPLACE); @@ -428,7 +430,7 @@ public static ByteBuffer encode(String string, boolean replace) throws Character /** * Read a UTF8 encoded string from in */ - public static String readString(DataInput in) throws IOException { + public static String readString(final DataInput in) throws IOException { if (in.readBoolean()) { final int length = in.readInt(); @@ -447,7 +449,7 @@ public static String readString(DataInput in) throws IOException { /** * Write a UTF8 encoded string to out */ - public static int writeString(DataOutput out, String s) throws IOException { + public static int writeString(final DataOutput out, final String s) throws IOException { int length = 0; @@ -479,7 +481,7 @@ public static int writeString(DataOutput out, String s) throws IOException { * @throws MalformedInputException * if the byte array contains invalid utf-8 */ - public static void validateUTF8(byte[] utf8) throws MalformedInputException { + public static void validateUTF8(final byte[] utf8) throws MalformedInputException { validateUTF8(utf8, 0, utf8.length); } @@ -495,7 +497,7 @@ public static void validateUTF8(byte[] utf8) throws MalformedInputException { * @throws MalformedInputException * if the byte array contains invalid bytes */ - public static void validateUTF8(byte[] utf8, int start, int len) throws MalformedInputException { + public static void validateUTF8(final byte[] utf8, final int start, final int len) throws MalformedInputException { int count = start; int leadByte = 0; int length = 0; @@ -593,7 +595,7 @@ public static void validateUTF8(byte[] utf8, int start, int len) throws Malforme * buffer's position will be incremented. Any mark set on this buffer will * be changed by this method! */ - public static int bytesToCodePoint(ByteBuffer bytes) { + public static int bytesToCodePoint(final ByteBuffer bytes) { bytes.mark(); final byte b = bytes.get(); bytes.reset(); @@ -637,7 +639,7 @@ public static int bytesToCodePoint(ByteBuffer bytes) { * text to encode * @return number of UTF-8 bytes required to encode */ - public static int utf8Length(String string) { + public static int utf8Length(final String string) { final CharacterIterator iter = new StringCharacterIterator(string); char ch = iter.first(); int size = 0; diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java index 690e282506d43..d94c1f69cdc75 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/ClassUtils.java @@ -27,7 +27,7 @@ * * @author warneke */ -public class ClassUtils { +public final class ClassUtils { /** * The log object used for debugging. @@ -51,7 +51,8 @@ private ClassUtils() { */ // TODO: See if we can improve type safety here @SuppressWarnings("unchecked") - public static Class getProtocolByName(String className) throws ClassNotFoundException { + public static Class getProtocolByName(final String className) + throws ClassNotFoundException { if (!className.contains("Protocol")) { throw new ClassNotFoundException("Only use this method for protocols!"); @@ -71,7 +72,7 @@ public static Class getProtocolByName(String classN */ // TODO: See if we can improve type safety here @SuppressWarnings("unchecked") - public static Class getRecordByName(String className) throws ClassNotFoundException { + public static Class getRecordByName(final String className) throws ClassNotFoundException { return (Class) Class.forName(className, true, getClassLoader()); @@ -88,7 +89,7 @@ public static Class getRecordByName(String className) throws C */ // TODO: See if we can improve type safety here @SuppressWarnings("unchecked") - public static Class getFileSystemByName(String className) throws ClassNotFoundException { + public static Class getFileSystemByName(final String className) throws ClassNotFoundException { return (Class) Class.forName(className, true, getClassLoader()); } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java index a83204760532a..eea2a8c4b006e 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/EnumUtils.java @@ -26,7 +26,7 @@ * * @author warneke */ -public class EnumUtils { +public final class EnumUtils { /** * Private constructor to overwrite public one. @@ -47,7 +47,7 @@ private EnumUtils() { * @throws IOException * thrown if any error occurred while reading data from the stream */ - public static > T readEnum(DataInput in, Class enumType) throws IOException { + public static > T readEnum(final DataInput in, final Class enumType) throws IOException { if (!in.readBoolean()) { return null; @@ -66,7 +66,7 @@ public static > T readEnum(DataInput in, Class enumType) th * @throws IOException * thrown if any error occurred while writing data to the stream */ - public static void writeEnum(DataOutput out, Enum enumVal) throws IOException { + public static void writeEnum(final DataOutput out, final Enum enumVal) throws IOException { if (enumVal == null) { out.writeBoolean(false); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java index bc88ceef4f299..da209b8ef7bc3 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/FileUtils.java @@ -20,18 +20,24 @@ * * @author warneke */ -public class FileUtils { +public final class FileUtils { /** * The alphabet to construct the random part of the filename from. */ - private static char[] ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd', 'e', - 'f' }; + private static final char[] ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd', + 'e', 'f' }; /** - * The length of the random part of the filename + * The length of the random part of the filename. */ - private static int LENGTH = 12; + private static final int LENGTH = 12; + + /** + * Empty private constructor to avoid instantiation. + */ + private FileUtils() { + } /** * Constructs a random filename with the given prefix and @@ -41,12 +47,12 @@ public class FileUtils { * the prefix to the filename to be constructed * @return the generated random filename with the given prefix */ - public static String getRandomFilename(String prefix) { + public static String getRandomFilename(final String prefix) { final StringBuilder stringBuilder = new StringBuilder(prefix); for (int i = 0; i < LENGTH; i++) { - stringBuilder.append(ALPHABET[(int) Math.floor(Math.random() * (double)ALPHABET.length)]); + stringBuilder.append(ALPHABET[(int) Math.floor(Math.random() * (double) ALPHABET.length)]); } return stringBuilder.toString(); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java index 69800bacd4175..31c2ceff892e5 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/IOUtils.java @@ -28,7 +28,7 @@ * * @author warneke */ -public class IOUtils { +public final class IOUtils { /** * The block size for byte operations in byte. @@ -56,10 +56,11 @@ private IOUtils() { * @throws IOException * thrown if an error occurred while writing to the output stream */ - public static void copyBytes(InputStream in, OutputStream out, int buffSize, boolean close) throws IOException { + public static void copyBytes(final InputStream in, final OutputStream out, final int buffSize, final boolean close) + throws IOException { final PrintStream ps = out instanceof PrintStream ? (PrintStream) out : null; - final byte buf[] = new byte[buffSize]; + final byte[] buf = new byte[buffSize]; try { int bytesRead = in.read(buf); while (bytesRead >= 0) { @@ -88,7 +89,7 @@ public static void copyBytes(InputStream in, OutputStream out, int buffSize, boo * @throws IOException * thrown if an I/O error occurs while copying */ - public static void copyBytes(InputStream in, OutputStream out) throws IOException { + public static void copyBytes(final InputStream in, final OutputStream out) throws IOException { copyBytes(in, out, BLOCKSIZE, true); } @@ -105,7 +106,7 @@ public static void copyBytes(InputStream in, OutputStream out) throws IOExceptio * @throws IOException * thrown if an I/O error occurs while copying */ - public static void copyBytes(InputStream in, OutputStream out, boolean close) throws IOException { + public static void copyBytes(final InputStream in, final OutputStream out, final boolean close) throws IOException { copyBytes(in, out, BLOCKSIZE, close); } @@ -123,7 +124,8 @@ public static void copyBytes(InputStream in, OutputStream out, boolean close) th * @throws IOException * if it could not read requested number of bytes for any reason (including EOF) */ - public static void readFully(InputStream in, byte buf[], int off, int len) throws IOException { + public static void readFully(final InputStream in, final byte[] buf, int off, final int len) + throws IOException { int toRead = len; while (toRead > 0) { final int ret = in.read(buf, off, toRead); @@ -145,7 +147,7 @@ public static void readFully(InputStream in, byte buf[], int off, int len) throw * @throws IOException * if it could not skip requested number of bytes for any reason (including EOF) */ - public static void skipFully(InputStream in, long len) throws IOException { + public static void skipFully(final InputStream in, long len) throws IOException { while (len > 0) { final long ret = in.skip(len); if (ret < 0) { @@ -164,7 +166,7 @@ public static void skipFully(InputStream in, long len) throws IOException { * @param closeables * the objects to close */ - public static void cleanup(Log log, java.io.Closeable... closeables) { + public static void cleanup(final Log log, final java.io.Closeable... closeables) { for (java.io.Closeable c : closeables) { if (c != null) { try { @@ -185,7 +187,7 @@ public static void cleanup(Log log, java.io.Closeable... closeables) { * @param stream * the stream to close */ - public static void closeStream(java.io.Closeable stream) { + public static void closeStream(final java.io.Closeable stream) { cleanup(null, stream); } @@ -195,7 +197,7 @@ public static void closeStream(java.io.Closeable stream) { * @param sock * the socket to close */ - public static void closeSocket(Socket sock) { + public static void closeSocket(final Socket sock) { // avoids try { close() } dance if (sock != null) { try { diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java index e88abe522ec63..599e649580ffe 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/JarFileCreator.java @@ -38,7 +38,7 @@ public class JarFileCreator { /** * The file extension of java classes. */ - private final static String CLASS_EXTENSION = ".class"; + private static final String CLASS_EXTENSION = ".class"; /** * A set of classes which shall be included in the final jar file. diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java index db4ea676c361c..9252e1a21a803 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/NativeCodeLoader.java @@ -28,7 +28,7 @@ * * @author warneke */ -public class NativeCodeLoader { +public final class NativeCodeLoader { /** * Set of the native libraries which are already loaded. @@ -36,7 +36,7 @@ public class NativeCodeLoader { private static Set loadedLibrarySet = new HashSet(); /** - * Directory prefix for native libraries inside JAR files + * Directory prefix for native libraries inside JAR files. */ private static final String JAR_PREFIX = "META-INF/lib/"; @@ -58,15 +58,10 @@ private NativeCodeLoader() { * the directory in which the library is supposed to be located * @param filename * filename of the library to be loaded - * @throws SecurityException - * thrown if a security exception occurs. - * @throws UnsatisfiedLinkError - * thrown if the library contains unsatisfied links * @throws IOException * thrown if an internal native library cannot be extracted */ - public static void loadLibraryFromFile(String directory, String filename) throws SecurityException, - UnsatisfiedLinkError, IOException { + public static void loadLibraryFromFile(final String directory, final String filename) throws IOException { final String libraryPath = directory + File.separator + filename; @@ -101,7 +96,7 @@ public static void loadLibraryFromFile(String directory, String filename) throws * @throws IOException * thrown if an I/O error occurs while copying the data */ - private static void copy(InputStream in, OutputStream out) throws IOException { + private static void copy(final InputStream in, final OutputStream out) throws IOException { final byte[] buf = new byte[BUFSIZE]; int len = 0; @@ -121,7 +116,7 @@ private static void copy(InputStream in, OutputStream out) throws IOException { * the filename of the library to check * @return true if the library is already loaded, false otherwise */ - public static boolean isLibraryLoaded(String libraryName) { + public static boolean isLibraryLoaded(final String libraryName) { synchronized (loadedLibrarySet) { return loadedLibrarySet.contains(libraryName); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java index fae8b8f3a47a4..79cf56ea0c269 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableArrayList.java @@ -46,7 +46,7 @@ public class SerializableArrayList extends ArrayLi * {@inheritDoc} */ @Override - public void write(DataOutput out) throws IOException { + public void write(final DataOutput out) throws IOException { out.writeInt(size()); final Iterator it = iterator(); @@ -64,9 +64,8 @@ public void write(DataOutput out) throws IOException { * {@inheritDoc} */ @SuppressWarnings("unchecked") - // TODO: See if type safety can be improved here @Override - public void read(DataInput in) throws IOException { + public void read(final DataInput in) throws IOException { // Make sure the list is empty clear(); diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java index 7438b984c6576..aad1491b93e5c 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/SerializableHashMap.java @@ -53,11 +53,11 @@ public class SerializableHashMap> it = entrySet().iterator(); - - while(it.hasNext()) { - + + while (it.hasNext()) { + final Map.Entry entry = it.next(); final K key = entry.getKey(); final V value = entry.getValue(); @@ -72,14 +72,13 @@ public void write(final DataOutput out) throws IOException { * {@inheritDoc} */ @SuppressWarnings("unchecked") - // TODO: See if type safety can be improved here @Override public void read(final DataInput in) throws IOException { final int numberOfMapEntries = in.readInt(); - - for(int i = 0; i < numberOfMapEntries; i++) { - + + for (int i = 0; i < numberOfMapEntries; i++) { + final String keyType = StringRecord.readString(in); Class keyClass = null; try { @@ -94,9 +93,9 @@ public void read(final DataInput in) throws IOException { } catch (Exception e) { throw new IOException(StringUtils.stringifyException(e)); } - + key.read(in); - + final String valueType = StringRecord.readString(in); Class valueClass = null; try { @@ -112,10 +111,9 @@ public void read(final DataInput in) throws IOException { throw new IOException(StringUtils.stringifyException(e)); } value.read(in); - + put(key, value); } - - } + } } diff --git a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java index 57d781548f4bc..0de923f09904b 100644 --- a/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java +++ b/nephele/nephele-common/src/main/java/eu/stratosphere/nephele/util/StringUtils.java @@ -29,7 +29,7 @@ * * @author warneke */ -public class StringUtils { +public final class StringUtils { /** * Empty private constructor to overwrite public one. @@ -44,7 +44,7 @@ private StringUtils() { * the exception to stringify * @return A string with exception name and call stack. */ - public static String stringifyException(Throwable e) { + public static String stringifyException(final Throwable e) { final StringWriter stm = new StringWriter(); final PrintWriter wrt = new PrintWriter(stm); e.printStackTrace(wrt); @@ -64,7 +64,7 @@ public static String stringifyException(Throwable e) { * end index, exclusively * @return hex string representation of the byte array */ - public static String byteToHexString(byte[] bytes, int start, int end) { + public static String byteToHexString(final byte[] bytes, final int start, final int end) { if (bytes == null) { throw new IllegalArgumentException("bytes == null"); } @@ -83,7 +83,7 @@ public static String byteToHexString(byte[] bytes, int start, int end) { * the bytes to convert in a hex string * @return hex string representation of the byte array */ - public static String byteToHexString(byte bytes[]) { + public static String byteToHexString(final byte[] bytes) { return byteToHexString(bytes, 0, bytes.length); } @@ -96,7 +96,7 @@ public static String byteToHexString(byte bytes[]) { * @return a byte array that is a hex string representation of the given * string. The size of the byte array is therefore hex.length/2 */ - public static byte[] hexStringToByte(String hex) { + public static byte[] hexStringToByte(final String hex) { final byte[] bts = new byte[hex.length() / 2]; for (int i = 0; i < bts.length; i++) { bts[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2ClientFactory.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2ClientFactory.java similarity index 85% rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2ClientFactory.java rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2ClientFactory.java index 0087e3fe06468..0ade972d70871 100644 --- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2ClientFactory.java +++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2ClientFactory.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import java.util.Hashtable; @@ -42,6 +42,14 @@ final class EC2ClientFactory { */ static synchronized AmazonEC2Client getEC2Client(final String awsAccessId, final String awsSecretKey) { + if(awsAccessId == null) { + throw new IllegalArgumentException("AWS access ID is null"); + } + + if(awsSecretKey == null) { + throw new IllegalArgumentException("AWS secret key is null"); + } + // Check if a client-object was already generated if (ec2clients.containsKey(awsAccessId)) { return ec2clients.get(awsAccessId); @@ -52,7 +60,7 @@ static synchronized AmazonEC2Client getEC2Client(final String awsAccessId, final final BasicAWSCredentials credentials = new BasicAWSCredentials(awsAccessId, awsSecretKey); final AmazonEC2Client client = new AmazonEC2Client(credentials); - final String endpoint = GlobalConfiguration.getString("ec2.webservice.endpoint", "ec2.eu-west-1.amazonaws.com"); + final String endpoint = GlobalConfiguration.getString("instancemanager.ec2.endpoint", "ec2.eu-west-1.amazonaws.com"); client.setEndpoint(endpoint); diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstance.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstance.java similarity index 82% rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstance.java rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstance.java index 5d84d568867d1..63fe77cc9bc25 100644 --- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstance.java +++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstance.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import eu.stratosphere.nephele.instance.AbstractInstance; import eu.stratosphere.nephele.instance.AllocatedResource; @@ -31,7 +31,7 @@ * * @author warneke */ -public class CloudInstance extends AbstractInstance { +public class EC2CloudInstance extends AbstractInstance { /** * The cached allocated resource object. @@ -40,16 +40,21 @@ public class CloudInstance extends AbstractInstance { /** The instance ID. */ private final String instanceID; - + /** The AWS Access Key to access this machine */ private String awsAccessKey; - + /** The AWS Secret Key to access this machine */ private String awsSecretKey; - + /** The time the instance was allocated. */ private final long launchTime; + /** + * The period of time for which the instance is leased from EC2. + */ + private final long leasePeriod; + /** The last received heart beat. */ private long lastReceivedHeartBeat = System.currentTimeMillis(); @@ -64,20 +69,23 @@ public class CloudInstance extends AbstractInstance { * the instance type * @param instanceConnectionInfo * the information required to connect to the instance's task manager - * @param allocationTime + * @param launchTime * the time the instance was allocated + * @param leasePeriod + * the period of time for which the instance is leased from EC2 * @param parentNode * the parent node in the network topology * @param hardwareDescription * the hardware description reported by the instance itself * @param awsAccessKey - * The AWS Access Key to access this machine + * The AWS Access Key to access this machine * @param awsSecretKey - * The AWS Secret Key to access this machine + * The AWS Secret Key to access this machine */ - public CloudInstance(String instanceID, InstanceType type, - InstanceConnectionInfo instanceConnectionInfo, long launchTime, NetworkNode parentNode, - NetworkTopology networkTopology, HardwareDescription hardwareDescription, String awsAccessKey, String awsSecretKey) { + public EC2CloudInstance(String instanceID, InstanceType type, + InstanceConnectionInfo instanceConnectionInfo, long launchTime, long leasePeriod, NetworkNode parentNode, + NetworkTopology networkTopology, HardwareDescription hardwareDescription, String awsAccessKey, + String awsSecretKey) { super(type, instanceConnectionInfo, parentNode, networkTopology, hardwareDescription); this.allocatedResource = new AllocatedResource(this, type, new AllocationID()); @@ -85,9 +93,11 @@ public CloudInstance(String instanceID, InstanceType type, this.instanceID = instanceID; this.launchTime = launchTime; - + + this.leasePeriod = leasePeriod; + this.awsAccessKey = awsAccessKey; - + this.awsSecretKey = awsSecretKey; } @@ -131,9 +141,11 @@ public AllocatedResource asAllocatedResource() { /** * Returns this Instance as a FloatingInstance object + * * @return */ - public FloatingInstance asFloatingInstance(){ - return new FloatingInstance(this.instanceID, this.getInstanceConnectionInfo(), this.launchTime, this.getType(), this.awsAccessKey, this.awsSecretKey); + public FloatingInstance asFloatingInstance() { + return new FloatingInstance(this.instanceID, this.getInstanceConnectionInfo(), this.launchTime, + this.leasePeriod, this.getType(), this.getHardwareDescription(), this.awsAccessKey, this.awsSecretKey); } } diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceNotifier.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceNotifier.java similarity index 92% rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceNotifier.java rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceNotifier.java index f2290b1ae5c20..052c181cf2bf3 100644 --- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceNotifier.java +++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceNotifier.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import java.util.List; @@ -32,7 +32,7 @@ * * @author warneke */ -final class CloudInstanceNotifier extends Thread { +final class EC2CloudInstanceNotifier extends Thread { /** * The {@link InstanceListener} object to send the notification to. @@ -53,7 +53,7 @@ final class CloudInstanceNotifier extends Thread { * @param allocatedResources * the resources which have been allocated for the job with the given ID */ - public CloudInstanceNotifier(final InstanceListener instanceListener, final JobID jobID, + public EC2CloudInstanceNotifier(final InstanceListener instanceListener, final JobID jobID, final List allocatedResources) { this.instanceListener = instanceListener; this.jobID = jobID; diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudManager.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManager.java similarity index 69% rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudManager.java rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManager.java index 76d22f1bf62af..531cc4124d84d 100644 --- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/CloudManager.java +++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManager.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import java.net.InetAddress; import java.net.UnknownHostException; @@ -32,11 +32,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.ec2.AmazonEC2Client; import com.amazonaws.services.ec2.model.BlockDeviceMapping; import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.DescribeInstancesResult; import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Placement; import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.RunInstancesResult; @@ -51,12 +53,12 @@ import eu.stratosphere.nephele.instance.InstanceException; import eu.stratosphere.nephele.instance.InstanceListener; import eu.stratosphere.nephele.instance.InstanceManager; -import eu.stratosphere.nephele.instance.InstanceRequestMap; import eu.stratosphere.nephele.instance.InstanceType; import eu.stratosphere.nephele.instance.InstanceTypeDescription; import eu.stratosphere.nephele.instance.InstanceTypeDescriptionFactory; import eu.stratosphere.nephele.instance.InstanceTypeFactory; import eu.stratosphere.nephele.jobgraph.JobID; +import eu.stratosphere.nephele.topology.NetworkNode; import eu.stratosphere.nephele.topology.NetworkTopology; import eu.stratosphere.nephele.util.SerializableHashMap; import eu.stratosphere.nephele.util.StringUtils; @@ -70,10 +72,37 @@ * user pays fee for allocated instances hourly. The cloud manager checks the floating instances every 60 seconds in * order to terminate the floating instances which expire. */ -public final class CloudManager extends TimerTask implements InstanceManager { +public final class EC2CloudManager extends TimerTask implements InstanceManager { - /** The log for the cloud manager. */ - private static final Log LOG = LogFactory.getLog(CloudManager.class); + /** + * The log for the EC2 cloud manager. + **/ + private static final Log LOG = LogFactory.getLog(EC2CloudManager.class); + + /** + * The key to access the lease period from the configuration. + */ + private static String LEASE_PERIOD_KEY = "instancemanager.ec2.leaseperiod"; + + /** + * The default lease period in milliseconds for instances on Amazon EC2. + */ + static final long DEFAULT_LEASE_PERIOD = 60 * 60 * 1000; // 1 hour in ms. + + /** + * The configuration key to access the AWS access ID of a job. + */ + static final String AWS_ACCESS_ID_KEY = "job.ec2.awsaccessid"; + + /** + * The configuration key to access the AWS secret key of a job. + */ + static final String AWS_SECRET_KEY_KEY = "job.ec2.awssecretkey"; + + /** + * The configuration key to access the AMI to run the task managers on. + */ + static final String AWS_AMI_KEY = "job.ec2.ami"; /** * The cloud manager checks the floating instances every base interval to terminate the floating instances which @@ -87,12 +116,26 @@ public final class CloudManager extends TimerTask implements InstanceManager { */ private static final int DEFAULTCLEANUPINTERVAL = 2 * 60 * 1000; // 2 min - /** TMs that send HeartBeats but do not belong to any job are kept in this set. */ - private final HashSet orphanedTMs = new HashSet(); + /** + * Instances that send heart beats but do not belong to any job are kept in this map. + **/ + private final Map orphanedInstances = new HashMap(); - /** The array of all available instance types in the cloud. */ + /** + * The array of all available instance types in the cloud. + **/ private final InstanceType[] availableInstanceTypes; + /** + * The lease period for instances on Amazon EC2 in milliseconds, potentially configured by the user. + */ + private final long leasePeriod; + + /** + * The default instance type. + */ + private InstanceType defaultInstanceType = null; + /** Mapping jobs to instances. */ private final Map jobToInstancesAssignmentMap = new HashMap(); @@ -115,19 +158,34 @@ public final class CloudManager extends TimerTask implements InstanceManager { private long cleanUpInterval = DEFAULTCLEANUPINTERVAL; /** - * The network topology inside the cloud (currently only a fake). + * The network topology for each job. + */ + private final Map networkTopologies = new HashMap(); + + /** + * The preferred availability for instances on EC2. */ - private final NetworkTopology networkTopology; + private final String availabilityZone; /** * Creates the cloud manager. */ - public CloudManager() { + public EC2CloudManager() { // Load the instance type this cloud can offer this.availableInstanceTypes = populateInstanceTypeArray(); - this.cleanUpInterval = (long) GlobalConfiguration.getInteger("cloud.ec2.cleanupinterval", + // Calculate lease period + long lp = GlobalConfiguration.getInteger(LEASE_PERIOD_KEY, -1); + if (lp > 0) { + LOG.info("Found user-defined lease period of " + lp + " minutes for instances"); + lp = lp * 1000L * 60L; // Convert to milliseconds + } else { + lp = DEFAULT_LEASE_PERIOD; + } + this.leasePeriod = lp; + + this.cleanUpInterval = (long) GlobalConfiguration.getInteger("instancemanager.ec2.cleanupinterval", DEFAULTCLEANUPINTERVAL); if ((this.cleanUpInterval % BASEINTERVAL) != 0) { @@ -135,7 +193,12 @@ public CloudManager() { this.cleanUpInterval = DEFAULTCLEANUPINTERVAL; } - this.networkTopology = NetworkTopology.createEmptyTopology(); + this.availabilityZone = GlobalConfiguration.getString("instancemanager.ec2.availabilityzone", null); + if (this.availabilityZone == null) { + LOG.info("No preferred availability zone configured"); + } else { + LOG.info("Found " + this.availabilityZone + " as preferred availability zone in configuration"); + } this.timer.schedule(this, (long) BASEINTERVAL, (long) BASEINTERVAL); } @@ -153,23 +216,36 @@ private InstanceType[] populateInstanceTypeArray() { final List instanceTypes = new ArrayList(); - // read the number of instance types - final int num = GlobalConfiguration.getInteger("cloudmgr.nrtypes", -1); - if (num <= 0) { - throw new RuntimeException("Illegal configuration, cloudmgr.nrtypes is not configured"); - } + int count = 1; + while (true) { - for (int i = 0; i < num; ++i) { - - final String key = "cloudmgr.instancetype." + (i + 1); + final String key = "instancemanager.ec2.type." + count; final String type = GlobalConfiguration.getString(key, null); if (type == null) { - throw new RuntimeException("Illegal configuration for " + key); + break; } - instanceTypes.add(InstanceTypeFactory.constructFromDescription(type)); + final InstanceType instanceType = InstanceTypeFactory.constructFromDescription(type); + LOG.info("Found instance type " + count + ": " + instanceType); + + instanceTypes.add(instanceType); + ++count; + } + + if (instanceTypes.isEmpty()) { + LOG.error("No instance types found in configuration"); + return new InstanceType[0]; + } + + int defaultIndex = GlobalConfiguration.getInteger("instancemanager.ec2.defaulttype", -1); + if (defaultIndex < 1 || defaultIndex >= (instanceTypes.size() + 1)) { + LOG.warn("Invalid index to default instance " + defaultIndex); + defaultIndex = 1; } + this.defaultInstanceType = instanceTypes.get(defaultIndex - 1); + LOG.info("Default instance type is " + this.defaultInstanceType); + // sort by price Collections.sort(instanceTypes, new Comparator() { @Override @@ -242,13 +318,15 @@ public synchronized void releaseAllocatedResource(final JobID jobID, final Confi if (jobToInstanceMapping.getAssignedInstances().contains(instance)) { // Unassigned Instance - jobToInstanceMapping.unassignInstanceFromJob((CloudInstance) instance); + jobToInstanceMapping.unassignInstanceFromJob((EC2CloudInstance) instance); // Make it a floating Instance this.floatingInstances.put(instance.getInstanceConnectionInfo(), - ((CloudInstance) instance).asFloatingInstance()); + ((EC2CloudInstance) instance).asFloatingInstance()); + + // TODO: Clean up job to instance mapping and network topology - LOG.info("Convert " + ((CloudInstance) instance).getInstanceID() + LOG.info("Converting " + ((EC2CloudInstance) instance).getInstanceID() + " from allocated instance to floating instance"); } else { @@ -265,8 +343,10 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo final HardwareDescription hardwareDescription) { // Check if this TM is orphaned - if (this.orphanedTMs.contains(instanceConnectionInfo)) { - LOG.debug("Received HeartBeat from orphaned TM " + instanceConnectionInfo); + if (this.orphanedInstances.containsKey(instanceConnectionInfo)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Received heart beat from orphaned instance " + instanceConnectionInfo); + } return; } @@ -278,7 +358,7 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo } // Check if heart beat belongs to an assigned instance - CloudInstance instance = isAssignedInstance(instanceConnectionInfo); + EC2CloudInstance instance = isAssignedInstance(instanceConnectionInfo); if (instance != null) { // If it's an assigned instance, just update the heart beat time stamp and leave instance.updateLastReceivedHeartBeat(); @@ -287,7 +367,7 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo // Check if heart beat belongs to a reserved instance try { - instance = isReservedInstance(instanceConnectionInfo); + instance = isReservedInstance(instanceConnectionInfo, hardwareDescription); } catch (InstanceException e) { LOG.error(e); } @@ -310,16 +390,15 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo mapping.assignInstanceToJob(instance); // Trigger notification that instance is available (outside synchronized section) - final List allocatedResources = new ArrayList(1); - allocatedResources.add(instance.asAllocatedResource()); - this.instanceListener.resourcesAllocated(jobID, allocatedResources); + this.instanceListener.resourceAllocated(jobID, instance.asAllocatedResource()); return; } // This TM seems to be unknown to the JobManager.. blacklist - LOG.info("Received HeartBeat from unknown TM. Put into orphaned TM set. Address is: " + instanceConnectionInfo); - this.orphanedTMs.add(instanceConnectionInfo); + LOG.info("Received heart beat from unknown instance " + instanceConnectionInfo + + ", converting it into orphaned instance"); + this.orphanedInstances.put(instanceConnectionInfo, hardwareDescription); } @@ -330,7 +409,7 @@ public synchronized void reportHeartBeat(final InstanceConnectionInfo instanceCo * the connection information object identifying the instance * @return a cloud instance */ - private CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceConnectionInfo) { + private EC2CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceConnectionInfo) { synchronized (this.jobToInstancesAssignmentMap) { @@ -340,7 +419,7 @@ private CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceCo final JobID jobID = it.next(); final JobToInstancesMapping m = this.jobToInstancesAssignmentMap.get(jobID); - final CloudInstance instance = m.getInstanceByConnectionInfo(instanceConnectionInfo); + final EC2CloudInstance instance = m.getInstanceByConnectionInfo(instanceConnectionInfo); if (instance != null) { return instance; } @@ -355,11 +434,14 @@ private CloudInstance isAssignedInstance(final InstanceConnectionInfo instanceCo * * @param instanceConnectionInfo * the {@link InstanceConnectionInfo} object identifying the instance + * @param hardwareDescription + * the actual hardware description of the instance * @return a cloud instance * @throws InstanceException * something wrong happens to the global configuration */ - private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceConnectionInfo) + private EC2CloudInstance isReservedInstance(final InstanceConnectionInfo instanceConnectionInfo, + final HardwareDescription hardwareDescription) throws InstanceException { if (instanceConnectionInfo == null) { @@ -380,18 +462,18 @@ private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceCo jobsWithReservedInstances.add(id); } - // Now we call the webservice for each job.. + // Now we call the web service for each job.. - for (JobID id : jobsWithReservedInstances) { + for (final JobID jobID : jobsWithReservedInstances) { JobToInstancesMapping mapping = null; synchronized (this.jobToInstancesAssignmentMap) { - mapping = this.jobToInstancesAssignmentMap.get(id); + mapping = this.jobToInstancesAssignmentMap.get(jobID); } if (mapping == null) { - LOG.error("Unknown mapping for job ID " + id); + LOG.error("Unknown mapping for job ID " + jobID); continue; } @@ -414,8 +496,18 @@ private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceCo } if (instanceConnectionInfo.getAddress().equals(candidateAddress)) { + + NetworkTopology networkTopology; + synchronized (this.networkTopologies) { + networkTopology = this.networkTopologies.get(jobID); + } + + if (networkTopology == null) { + throw new InstanceException("Cannot find network topology for job " + jobID); + } + return convertIntoCloudInstance(t, instanceConnectionInfo, mapping.getAwsAccessId(), - mapping.getAwsSecretKey()); + mapping.getAwsSecretKey(), networkTopology.getRootNode(), hardwareDescription); } } } @@ -436,8 +528,9 @@ private CloudInstance isReservedInstance(final InstanceConnectionInfo instanceCo * the information required to connect to the instance's task manager later on * @return a cloud instance */ - private CloudInstance convertIntoCloudInstance(final com.amazonaws.services.ec2.model.Instance instance, - final InstanceConnectionInfo instanceConnectionInfo, final String awsAccessKey, final String awsSecretKey) { + private EC2CloudInstance convertIntoCloudInstance(final Instance instance, + final InstanceConnectionInfo instanceConnectionInfo, final String awsAccessKey, final String awsSecretKey, + final NetworkNode parentNode, final HardwareDescription hardwareDescription) { InstanceType type = null; @@ -455,9 +548,9 @@ private CloudInstance convertIntoCloudInstance(final com.amazonaws.services.ec2. return null; } - final CloudInstance cloudInstance = new CloudInstance(instance.getInstanceId(), type, instanceConnectionInfo, - instance.getLaunchTime().getTime(), this.networkTopology.getRootNode(), this.networkTopology, null, - awsAccessKey, awsSecretKey); + final EC2CloudInstance cloudInstance = new EC2CloudInstance(instance.getInstanceId(), type, + instanceConnectionInfo, instance.getLaunchTime().getTime(), this.leasePeriod, parentNode, + parentNode.getNetworkTopology(), hardwareDescription, awsAccessKey, awsSecretKey); // TODO: Define hardware descriptions for cloud instance types @@ -468,8 +561,9 @@ private CloudInstance convertIntoCloudInstance(final com.amazonaws.services.ec2. * {@inheritDoc} */ @Override - public void requestInstance(final JobID jobID, Configuration conf, final InstanceRequestMap instanceRequestMap, - final List splitAffinityList) throws InstanceException { + public void requestInstance(final JobID jobID, Configuration conf, + final Map instanceRequestMap, final List splitAffinityList) + throws InstanceException { if (conf == null) { throw new IllegalArgumentException("No job configuration provided, unable to acquire credentials"); @@ -477,17 +571,21 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc // First check, if all required configuration entries are available - final String awsAccessId = conf.getString("job.cloud.awsaccessid", null); - LOG.info("found AWS access ID from Job Conf: " + awsAccessId); + final String awsAccessId = conf.getString(AWS_ACCESS_ID_KEY, null); if (awsAccessId == null) { throw new InstanceException("Unable to allocate cloud instance: Cannot find AWS access ID"); } - final String awsSecretKey = conf.getString("job.cloud.awssecretkey", null); + + final String awsSecretKey = conf.getString(AWS_SECRET_KEY_KEY, null); if (awsSecretKey == null) { throw new InstanceException("Unable to allocate cloud instance: Cannot find AWS secret key"); } - // First we check, if there are any orphaned TMs that are accessible with the provided configuration + if (conf.getString(AWS_AMI_KEY, null) == null) { + throw new InstanceException("Unable to allocate cloud instance: Cannot find AMI image ID"); + } + + // First we check, if there are any orphaned instances that are accessible with the provided configuration checkAndConvertOrphanedInstances(conf); // Check if there already exist a mapping for this job @@ -503,12 +601,22 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc } } + // Check if there already exists a network topology for this job + NetworkTopology networkTopology = null; + synchronized (this.networkTopologies) { + networkTopology = this.networkTopologies.get(jobID); + if (networkTopology == null) { + networkTopology = NetworkTopology.createEmptyTopology(); + this.networkTopologies.put(jobID, networkTopology); + } + } + // Our bill with all instances that we will provide... final LinkedList floatingInstances = new LinkedList(); final LinkedList requestedInstances = new LinkedList(); // We iterate over the maximum of requested Instances... - final Iterator> it = instanceRequestMap.getMaximumIterator(); + final Iterator> it = instanceRequestMap.entrySet().iterator(); while (it.hasNext()) { @@ -517,11 +625,12 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc // This is our actual type... final InstanceType actualtype = e.getKey(); final int maxcount = e.getValue(); - final int mincount = instanceRequestMap.getMinimumNumberOfInstances(actualtype); + final int mincount = maxcount; + LOG.info("Requesting " + maxcount + " instances of type " + actualtype + " for job " + jobID); // And this is the list of instances we will have... - LinkedList actualFloatingInstances = new LinkedList(); - LinkedList actualRequestedInstances = new LinkedList(); + LinkedList actualFloatingInstances = null; + LinkedList actualRequestedInstances = null; // Check if floating instances available... actualFloatingInstances = anyFloatingInstancesAvailable(awsAccessId, awsSecretKey, actualtype, maxcount); @@ -533,6 +642,8 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc actualRequestedInstances = allocateCloudInstance(conf, actualtype, minimumrequestcount, maximumrequestcount); + } else { + actualRequestedInstances = new LinkedList(); } // Add provided Instances to overall bill... @@ -541,8 +652,8 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc // Are we outer limits? if (actualRequestedInstances.size() + actualFloatingInstances.size() < mincount) { - LOG.error("Requested: " + mincount + " to " + maxcount + " instanes of type " - + actualtype.getIdentifier() + ". Could only provide " + LOG.error("Requested: " + mincount + " to " + maxcount + " instances of type " + + actualtype.getIdentifier() + ", but could only provide " + (actualRequestedInstances.size() + actualFloatingInstances.size()) + "."); // something went wrong.. give the floating instances back! @@ -551,25 +662,21 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc this.floatingInstances.put(i.getInstanceConnectionInfo(), i); } } - throw new InstanceException("Could not allocate enough cloud instances"); + throw new InstanceException("Could not allocate enough cloud instances. See logs for details."); } // End outer limits } // End iterating over instance types.. - // If we reached this point, everything went well - final List allocatedResources = new ArrayList(); - // Convert and allocate Floating Instances... for (final FloatingInstance fi : floatingInstances) { - final CloudInstance ci = fi.asCloudInstance(); + final EC2CloudInstance ci = fi.asCloudInstance(networkTopology.getRootNode()); jobToInstanceMapping.assignInstanceToJob(ci); - allocatedResources.add(ci.asAllocatedResource()); + final EC2CloudInstanceNotifier notifier = new EC2CloudInstanceNotifier(this.instanceListener, jobID, + ci.asAllocatedResource()); + notifier.start(); } // Finally, inform the scheduler about the instances which have been floating before - final CloudInstanceNotifier notifier = new CloudInstanceNotifier(this.instanceListener, jobID, - allocatedResources); - notifier.start(); // Add reserved Instances to Job Mapping... for (final String i : requestedInstances) { @@ -594,26 +701,18 @@ public void requestInstance(final JobID jobID, Configuration conf, final Instanc private LinkedList allocateCloudInstance(final Configuration conf, final InstanceType type, final int mincount, final int maxcount) { - final String awsAccessId = conf.getString("job.cloud.awsaccessid", null); - final String awsSecretKey = conf.getString("job.cloud.awssecretkey", null); - - String imageID = conf.getString("job.ec2.image.id", null); - LOG.info("EC2 Image ID from job conf: " + imageID); - if (imageID == null) { + final String awsAccessId = conf.getString(AWS_ACCESS_ID_KEY, null); + final String awsSecretKey = conf.getString(AWS_SECRET_KEY_KEY, null); - imageID = GlobalConfiguration.getString("ec2.image.id", null); - if (imageID == null) { - LOG.error("Unable to allocate instance: Image ID is unknown"); - return null; - } - } + final String imageID = conf.getString(AWS_AMI_KEY, null); + LOG.info("Read Amazon Machine Image from job configuration: " + imageID); final String jobManagerIPAddress = GlobalConfiguration.getString("jobmanager.rpc.address", null); if (jobManagerIPAddress == null) { LOG.error("JobManager IP address is not set (jobmanager.rpc.address)"); return null; } - final String sshKeyPair = conf.getString("job.cloud.sshkeypair", null); + final String sshKeyPair = conf.getString("job.ec2.sshkeypair", null); final AmazonEC2Client ec2client = EC2ClientFactory.getEC2Client(awsAccessId, awsSecretKey); final LinkedList instanceIDs = new LinkedList(); @@ -623,6 +722,21 @@ private LinkedList allocateCloudInstance(final Configuration conf, final final RunInstancesRequest request = new RunInstancesRequest(imageID, mincount, maxcount); request.setInstanceType(type.getIdentifier()); + // Set availability zone if configured + String av = null; + if (this.availabilityZone != null) { + av = this.availabilityZone; + } + final String jobAV = conf.getString("job.ec2.availabilityzone", null); + if (jobAV != null) { + LOG.info("Found " + jobAV + " as job-specific preference for availability zone"); + av = jobAV; + } + + if (av != null) { + request.setPlacement(new Placement(av)); + } + // TODO: Make this configurable! final BlockDeviceMapping bdm = new BlockDeviceMapping(); bdm.setVirtualName("ephemeral0"); @@ -640,10 +754,15 @@ private LinkedList allocateCloudInstance(final Configuration conf, final request.setUserData(EC2Utilities.createTaskManagerUserData(jobManagerIPAddress)); // Request instances! - final RunInstancesResult result = ec2client.runInstances(request); + try { + final RunInstancesResult result = ec2client.runInstances(request); - for (Instance i : result.getReservation().getInstances()) { - instanceIDs.add(i.getInstanceId()); + for (Instance i : result.getReservation().getInstances()) { + instanceIDs.add(i.getInstanceId()); + } + } catch (AmazonClientException e) { + // Only log the error here + LOG.error(StringUtils.stringifyException(e)); } return instanceIDs; @@ -655,20 +774,31 @@ private LinkedList allocateCloudInstance(final Configuration conf, final * * @param conf * The configuration provided upon instances request + * @throws InstanceException + * thrown if an error occurs while communicating with Amazon EC2 */ - private void checkAndConvertOrphanedInstances(final Configuration conf) { - if (this.orphanedTMs.size() == 0) { + private void checkAndConvertOrphanedInstances(final Configuration conf) throws InstanceException { + + if (this.orphanedInstances.size() == 0) { return; } - final String awsAccessId = conf.getString("job.cloud.awsaccessid", null); - final String awsSecretKey = conf.getString("job.cloud.awssecretkey", null); + final String awsAccessId = conf.getString(AWS_ACCESS_ID_KEY, null); + final String awsSecretKey = conf.getString(AWS_SECRET_KEY_KEY, null); - LOG.debug("Checking orphaned Instances... " + this.orphanedTMs.size() + " orphaned instances listed."); + if (LOG.isDebugEnabled()) { + LOG.debug("Checking orphaned instances, " + this.orphanedInstances.size() + " orphaned instances listed."); + } final AmazonEC2Client ec2client = EC2ClientFactory.getEC2Client(awsAccessId, awsSecretKey); - final DescribeInstancesRequest request = new DescribeInstancesRequest(); - final DescribeInstancesResult result = ec2client.describeInstances(request); + DescribeInstancesResult result = null; + + try { + final DescribeInstancesRequest request = new DescribeInstancesRequest(); + result = ec2client.describeInstances(request); + } catch (AmazonClientException e) { + throw new InstanceException(StringUtils.stringifyException(e)); + } // Iterate over all Instances for (final Reservation r : result.getReservations()) { @@ -694,23 +824,27 @@ private void checkAndConvertOrphanedInstances(final Configuration conf) { continue; } - final Iterator it = this.orphanedTMs.iterator(); + final Iterator> it = this.orphanedInstances + .entrySet().iterator(); while (it.hasNext()) { - final InstanceConnectionInfo oi = it.next(); + final Map.Entry entry = it.next(); + + final InstanceConnectionInfo oi = entry.getKey(); + final HardwareDescription hd = entry.getValue(); + if (oi.getAddress().equals(inetAddress) && type != null) { - LOG.info("Orphaned Instance " + oi + " converted into floating instance."); + LOG.info("Orphaned instance " + oi + " converted into floating instance."); - // We have found the corresponding orphaned TM.. take the poor lamb back to its nest. - FloatingInstance floatinginstance = new FloatingInstance(t.getInstanceId(), oi, t - .getLaunchTime().getTime(), type, awsAccessId, awsSecretKey); + // We have found the corresponding orphaned TM.. convert it back to a floating instance. + final FloatingInstance floatinginstance = new FloatingInstance(t.getInstanceId(), oi, t + .getLaunchTime().getTime(), this.leasePeriod, type, hd, awsAccessId, awsSecretKey); this.floatingInstances.put(oi, floatinginstance); it.remove(); break; } } - } } } @@ -732,7 +866,7 @@ private void checkAndConvertOrphanedInstances(final Configuration conf) { private LinkedList anyFloatingInstancesAvailable(final String awsAccessId, final String awsSecretKey, final InstanceType type, final int count) throws InstanceException { - LOG.info("Check for floating instance of type" + type.getIdentifier() + " requested count: " + count + "."); + LOG.info("Checking for up to " + count + " floating instance of type " + type.getIdentifier()); final LinkedList foundfloatinginstances = new LinkedList(); @@ -774,14 +908,13 @@ public void run() { while (it.hasNext()) { final Map.Entry entry = it.next(); - // Call lifecycle method for each floating instance. If true, remove from floatinginstances list. - if (entry.getValue().checkIfLifeCycleEnded()) { + // Call life cycle method for each floating instance. If true, remove from floating instances list. + if (entry.getValue().hasLifeCycleEnded()) { it.remove(); - LOG.info("Floating Instance " + entry.getValue().getInstanceID() - + " ended its lifecycle. Terminated"); + LOG.info("Lifecycle of floating instance " + entry.getValue().getInstanceID() + + " has ended, terminating instance..."); } } - } } @@ -793,12 +926,7 @@ public void run() { @Override public InstanceType getDefaultInstanceType() { - final String instanceIdentifier = GlobalConfiguration.getString("cloudmgr.instancetype.defaultInstance", null); - if (instanceIdentifier == null) { - return null; - } - - return getInstanceTypeByName(instanceIdentifier); + return this.defaultInstanceType; } /** @@ -832,17 +960,19 @@ public void shutdown() { } @Override - public NetworkTopology getNetworkTopology(JobID jobID) { + public NetworkTopology getNetworkTopology(final JobID jobID) { - // TODO: Make topology job specific - return this.networkTopology; + synchronized (this.networkTopologies) { + + return this.networkTopologies.get(jobID); + } } /** * {@inheritDoc} */ @Override - public void setInstanceListener(InstanceListener instanceListener) { + public void setInstanceListener(final InstanceListener instanceListener) { this.instanceListener = instanceListener; } @@ -856,7 +986,10 @@ public Map getMapOfAvailableInstanceTypes final Map availableinstances = new SerializableHashMap(); for (final InstanceType t : this.availableInstanceTypes) { - availableinstances.put(t, InstanceTypeDescriptionFactory.construct(t, estimateHardwareDescription(t), -1)); + // TODO: Number of available instances is set to 1000 to improve interaction with PACT layer, must be -1 + // actually according to API + availableinstances + .put(t, InstanceTypeDescriptionFactory.construct(t, estimateHardwareDescription(t), 1000)); } return availableinstances; diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2Utilities.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2Utilities.java similarity index 97% rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2Utilities.java rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2Utilities.java index 6a305f4d6b8c1..c3ea5dd8776ff 100644 --- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/EC2Utilities.java +++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/EC2Utilities.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; /** * This class provides auxiliary methods needed to set up the custom EC2-image. diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/FloatingInstance.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/FloatingInstance.java similarity index 67% rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/FloatingInstance.java rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/FloatingInstance.java index 6e50256d62158..1abe00ac010a0 100644 --- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/FloatingInstance.java +++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/FloatingInstance.java @@ -13,15 +13,17 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import java.util.LinkedList; import com.amazonaws.services.ec2.AmazonEC2Client; import com.amazonaws.services.ec2.model.TerminateInstancesRequest; +import eu.stratosphere.nephele.instance.HardwareDescription; import eu.stratosphere.nephele.instance.InstanceConnectionInfo; import eu.stratosphere.nephele.instance.InstanceType; +import eu.stratosphere.nephele.topology.NetworkNode; /** * A FloatingInstance is an instance in the cloud allocated for a user. It is idle and carries out no task. @@ -29,11 +31,15 @@ */ class FloatingInstance { - /** The user pays fee for his instances every time unit. */ - private static final long TIMEUNIT = 60 * 60 * 1000; // 1 hour in ms. + /** + * Time limit to full next hour when instance is terminate. + **/ + private static final long TIME_THRESHOLD = 2 * 60 * 1000; // 2 mins in ms. - /** Timelimit to full next hour when instance is kicked. */ - private static final long TIMETHRESHOLD = 2 * 60 * 1000; // 2 mins in ms. + /** + * The lease period for this instance of Amazon EC2 in milliseconds. + */ + private final long leasePeriod; /** The instance ID. */ private final String instanceID; @@ -50,9 +56,14 @@ class FloatingInstance { /** The AWS Secret Key to access this machine */ private String awsSecretKey; - /** The instance Type */ + /** The instance type */ private InstanceType type; + /** + * The instance's hardware description. + */ + private final HardwareDescription hardwareDescription; + /** The last received heart beat. */ private long lastHeartBeat; @@ -64,7 +75,9 @@ class FloatingInstance { * @param instanceConnectionInfo * the information required to connect to the instance's task manager * @param launchTime - * the time the instance was allocated + * the time the instance was allocated in milliseconds since January 1st, 1970 + * @param leasePeriod + * the lease period for this floating instances in milliseconds * @param type * The type of this instance. * @param awsAccessKey @@ -72,15 +85,29 @@ class FloatingInstance { * @param awsSecretKey * The AWS Secret Key to access this machine */ - public FloatingInstance(String instanceID, InstanceConnectionInfo instanceConnectionInfo, long launchTime, - InstanceType type, String awsAccessKey, String awsSecretKey) { + public FloatingInstance(final String instanceID, final InstanceConnectionInfo instanceConnectionInfo, + final long launchTime, + final long leasePeriod, final InstanceType type, final HardwareDescription hardwareDescription, + final String awsAccessKey, final String awsSecretKey) { + + if (launchTime < 0) { + throw new IllegalArgumentException("Argument launchTime must be greater than 0"); + } + + if (leasePeriod <= 0) { + throw new IllegalArgumentException("Argument leasePeriod be greater than 0"); + } + this.instanceID = instanceID; this.instanceConnectionInfo = instanceConnectionInfo; this.launchTime = launchTime; + this.leasePeriod = leasePeriod; this.lastHeartBeat = System.currentTimeMillis(); this.awsAccessKey = awsAccessKey; this.awsSecretKey = awsSecretKey; this.type = type; + this.hardwareDescription = hardwareDescription; + } /** @@ -155,22 +182,24 @@ public long getLaunchTime() { * * @return */ - public CloudInstance asCloudInstance() { - return new CloudInstance(this.instanceID, this.type, this.getInstanceConnectionInfo(), this.launchTime, null, - null, null, this.awsAccessKey, this.awsSecretKey); + public EC2CloudInstance asCloudInstance(final NetworkNode parentNode) { + + return new EC2CloudInstance(this.instanceID, this.type, this.getInstanceConnectionInfo(), this.launchTime, + this.leasePeriod, parentNode, parentNode.getNetworkTopology(), this.hardwareDescription, this.awsAccessKey, + this.awsSecretKey); } /** - * This method checks, if this floating instance has reached the end of its lifecycle and - if so - terminates + * This method checks if this floating instance has reached the end of its life cycle and, if so, terminates * itself. */ - public boolean checkIfLifeCycleEnded() { + public boolean hasLifeCycleEnded() { final long currentTime = System.currentTimeMillis(); - final long msremaining = TIMEUNIT - ((currentTime - this.launchTime) % TIMEUNIT); + final long msremaining = this.leasePeriod - ((currentTime - this.launchTime) % this.leasePeriod); - if (msremaining < TIMETHRESHOLD) { - // Destroy this instance. + if (msremaining < TIME_THRESHOLD) { + // Destroy this instance final AmazonEC2Client client = EC2ClientFactory.getEC2Client(this.awsAccessKey, this.awsSecretKey); final TerminateInstancesRequest tr = new TerminateInstancesRequest(); final LinkedList instanceIDlist = new LinkedList(); @@ -178,9 +207,18 @@ public boolean checkIfLifeCycleEnded() { tr.setInstanceIds(instanceIDlist); client.terminateInstances(tr); return true; - } else { - return false; } + + return false; } + /** + * Returns the hardware description of the floating instance. + * + * @return the hardware description of the floating instance + */ + public HardwareDescription getHardwareDescription() { + + return this.hardwareDescription; + } } diff --git a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMapping.java b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMapping.java similarity index 87% rename from nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMapping.java rename to nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMapping.java index 4e4d2a8e96ce9..272a24f6aacbd 100644 --- a/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMapping.java +++ b/nephele/nephele-ec2cloudmanager/src/main/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMapping.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import java.util.ArrayList; import java.util.Iterator; @@ -28,7 +28,7 @@ final class JobToInstancesMapping { /** The list of assigned cloud instances for the job. */ - private final List assignedInstances = new ArrayList(); + private final List assignedInstances = new ArrayList(); /** The access ID into Amazon Web Services. */ private final String awsAccessId; @@ -56,7 +56,7 @@ public JobToInstancesMapping(final String awsAccessId, final String awsSecretKey * @param instance * the cloud instance which will be assigned */ - public void assignInstanceToJob(final CloudInstance instance) { + public void assignInstanceToJob(final EC2CloudInstance instance) { synchronized (this.assignedInstances) { this.assignedInstances.add(instance); @@ -70,7 +70,7 @@ public void assignInstanceToJob(final CloudInstance instance) { * the cloud instance which will be unassigned * @return the unassigned cloud instance */ - public boolean unassignInstanceFromJob(CloudInstance instance) { + public boolean unassignInstanceFromJob(EC2CloudInstance instance) { synchronized (this.assignedInstances) { return this.assignedInstances.remove(instance); @@ -91,7 +91,7 @@ public int getNumberOfAssignedInstances() { * * @return the list of assigned cloud instances for the job */ - public List getAssignedInstances() { + public List getAssignedInstances() { return this.assignedInstances; } @@ -103,7 +103,7 @@ public List getAssignedInstances() { * @return the cloud instance matching the given connection information or null if no matching instance * exists */ - public CloudInstance getInstanceByConnectionInfo(final InstanceConnectionInfo instanceConnectionInfo) { + public EC2CloudInstance getInstanceByConnectionInfo(final InstanceConnectionInfo instanceConnectionInfo) { if (instanceConnectionInfo == null) { return null; @@ -111,11 +111,11 @@ public CloudInstance getInstanceByConnectionInfo(final InstanceConnectionInfo in synchronized (this.assignedInstances) { - final Iterator it = this.assignedInstances.iterator(); + final Iterator it = this.assignedInstances.iterator(); while (it.hasNext()) { - final CloudInstance ci = it.next(); + final EC2CloudInstance ci = it.next(); if (instanceConnectionInfo.equals(ci.getInstanceConnectionInfo())) { return ci; } diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceTest.java similarity index 77% rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceTest.java rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceTest.java index e67f0e36a517b..84712622c81e5 100644 --- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudInstanceTest.java +++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudInstanceTest.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import static org.junit.Assert.*; @@ -25,28 +25,30 @@ import eu.stratosphere.nephele.instance.HardwareDescriptionFactory; import eu.stratosphere.nephele.instance.InstanceConnectionInfo; import eu.stratosphere.nephele.instance.InstanceTypeFactory; -import eu.stratosphere.nephele.instance.cloud.CloudInstance; +import eu.stratosphere.nephele.instance.ec2.EC2CloudInstance; import eu.stratosphere.nephele.topology.NetworkTopology; -public class CloudInstanceTest { +public class EC2CloudInstanceTest { + + private EC2CloudInstance constructSmallCloudInstance() { - private CloudInstance constructSmallCloudInstance() { - final NetworkTopology networkTopology = NetworkTopology.createEmptyTopology(); - final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L*1024L*1024L, 2048L*1024L*1024L); - - final CloudInstance cloudInstance = new CloudInstance("i-1234ABCD", + final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L * 1024L * 1024L, + 2048L * 1024L * 1024L); + + final EC2CloudInstance cloudInstance = new EC2CloudInstance("i-1234ABCD", InstanceTypeFactory.constructFromDescription("m1.small,1,1,2048,40,10"), new InstanceConnectionInfo(new InetSocketAddress("localhost", 6122).getAddress(), 6122, 6121), - 1234567890, networkTopology.getRootNode(), networkTopology, hardwareDescription, null, null); - + 1234567890, EC2CloudManager.DEFAULT_LEASE_PERIOD, networkTopology.getRootNode(), networkTopology, + hardwareDescription, null, null); + return cloudInstance; } - + @Test public void testHeartBeat() { - final CloudInstance ci = constructSmallCloudInstance(); + final EC2CloudInstance ci = constructSmallCloudInstance(); long lastHeartBeat = ci.getLastReceivedHeartBeat(); try { @@ -58,4 +60,4 @@ public void testHeartBeat() { assertTrue(ci.getLastReceivedHeartBeat() - lastHeartBeat > 0); } -} +} \ No newline at end of file diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudManagerTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManagerTest.java similarity index 82% rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudManagerTest.java rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManagerTest.java index c1573c3139fb7..a253f0e528847 100644 --- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/CloudManagerTest.java +++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/EC2CloudManagerTest.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import static org.junit.Assert.*; @@ -49,13 +49,13 @@ import eu.stratosphere.nephele.instance.InstanceRequestMap; import eu.stratosphere.nephele.instance.InstanceType; import eu.stratosphere.nephele.instance.InstanceTypeFactory; -import eu.stratosphere.nephele.instance.cloud.CloudInstance; -import eu.stratosphere.nephele.instance.cloud.CloudManager; -import eu.stratosphere.nephele.instance.cloud.FloatingInstance; -import eu.stratosphere.nephele.instance.cloud.JobToInstancesMapping; +import eu.stratosphere.nephele.instance.ec2.EC2CloudInstance; +import eu.stratosphere.nephele.instance.ec2.EC2CloudManager; +import eu.stratosphere.nephele.instance.ec2.FloatingInstance; +import eu.stratosphere.nephele.instance.ec2.JobToInstancesMapping; import eu.stratosphere.nephele.jobgraph.JobID; -public class CloudManagerTest { +public class EC2CloudManagerTest { private static final class MyInstanceListener implements InstanceListener { @@ -68,12 +68,8 @@ public void allocatedResourcesDied(final JobID jobID, final List resourcesOfJob = this.resourcesOfJobs.get(jobID); assertTrue(resourcesOfJob != null); - - for (final AllocatedResource allocatedResource : allocatedResources) { - assertTrue(resourcesOfJob.contains(allocatedResource)); - resourcesOfJob.remove(allocatedResource); - } - + assertTrue(resourcesOfJob.contains(allocatedResource)); + resourcesOfJob.remove(allocatedResource); if (resourcesOfJob.isEmpty()) { this.resourcesOfJobs.remove(jobID); } @@ -102,13 +98,12 @@ public void testLoadConf() { GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf"); - assertEquals(5, GlobalConfiguration.getInteger("cloudmgr.nrtypes", -1)); - assertEquals("m1.small,1,1,2048,40,10", GlobalConfiguration.getString("cloudmgr.instancetype.1", null)); - assertEquals("c1.medium,2,2,4096,80,20", GlobalConfiguration.getString("cloudmgr.instancetype.2", null)); - assertEquals("m1.large,4,4,6144,160,40", GlobalConfiguration.getString("cloudmgr.instancetype.3", null)); - assertEquals("m1.xlarge,4,4,12288,160,60", GlobalConfiguration.getString("cloudmgr.instancetype.4", null)); - assertEquals("c1.xlarge,8,8,28672,280,80", GlobalConfiguration.getString("cloudmgr.instancetype.5", null)); - assertEquals("m1.small", GlobalConfiguration.getString("cloudmgr.instancetype.defaultInstance", null)); + assertEquals("m1.small,1,1,2048,40,10", GlobalConfiguration.getString("instancemanager.ec2.type.1", null)); + assertEquals("c1.medium,2,2,4096,80,20", GlobalConfiguration.getString("instancemanager.ec2.type.2", null)); + assertEquals("m1.large,4,4,6144,160,40", GlobalConfiguration.getString("instancemanager.ec2.type.3", null)); + assertEquals("m1.xlarge,4,4,12288,160,60", GlobalConfiguration.getString("instancemanager.ec2.type.4", null)); + assertEquals("c1.xlarge,8,8,28672,280,80", GlobalConfiguration.getString("instancemanager.ec2.type.5", null)); + assertEquals(1, GlobalConfiguration.getInteger("instancemanager.ec2.defaulttype", -1)); } @Test @@ -133,7 +128,7 @@ public void testGetDefaultInstance() { GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf"); MyInstanceListener myInstanceListener = new MyInstanceListener(); - CloudManager cm = new CloudManager(); + EC2CloudManager cm = new EC2CloudManager(); cm.setInstanceListener(myInstanceListener); InstanceType defaultIT = cm.getDefaultInstanceType(); @@ -153,7 +148,7 @@ public void testGetSuitableInstanceType() { GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf"); MyInstanceListener myInstanceListener = new MyInstanceListener(); - CloudManager cm = new CloudManager(); + EC2CloudManager cm = new EC2CloudManager(); cm.setInstanceListener(myInstanceListener); InstanceType type1 = cm.getSuitableInstanceType(16, 16, 2048, 40, 80); @@ -185,7 +180,7 @@ public void testGetInstanceTypeByName() { GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf"); MyInstanceListener myInstanceListener = new MyInstanceListener(); - CloudManager cm = new CloudManager(); + EC2CloudManager cm = new EC2CloudManager(); cm.setInstanceListener(myInstanceListener); InstanceType type = cm.getInstanceTypeByName("m1.small"); @@ -205,7 +200,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { GlobalConfiguration.loadConfiguration(System.getProperty("user.dir") + "/correct-conf"); MyInstanceListener myInstanceListener = new MyInstanceListener(); - CloudManager cm = new CloudManager(); + EC2CloudManager cm = new EC2CloudManager(); cm.setInstanceListener(myInstanceListener); JobID jobID = new JobID(); @@ -216,7 +211,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { if (!f.exists()) { System.err.println("Please create an XML file \"ec2-account.xml\" for EC2 account in the folder " + System.getProperty("user.dir") + "/correct-conf\n" - + "Three keys must be included: job.cloud.username, job.cloud.awsaccessid, job.cloud.awssecretkey\n" + + "Three keys must be included: , job.ec2.awsaccessid, job.ec2.awssecretkey\n" + "The format is:\n" + "\n" + " ...\n" + " ...\n" + ""); return; } @@ -245,21 +240,16 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { e.printStackTrace(); } - // check whether all three keys are included - if (conf.getString("job.cloud.username", null) == null) { - System.err.println("Please set the key job.cloud.username in " + System.getProperty("user.dir") - + "/correct-conf/ec2-account.xml"); - return; - } - - if (conf.getString("job.cloud.awsaccessid", null) == null) { - System.err.println("Please set the key job.cloud.awsaccessid in " + System.getProperty("user.dir") + if (conf.getString(EC2CloudManager.AWS_ACCESS_ID_KEY, null) == null) { + System.err.println("Please set the key " + EC2CloudManager.AWS_ACCESS_ID_KEY + " in " + + System.getProperty("user.dir") + "/correct-conf/ec2-account.xml"); return; } - if (conf.getString("job.cloud.awssecretkey", null) == null) { - System.err.println("Please set the key job.cloud.awssecretkey in " + System.getProperty("user.dir") + if (conf.getString(EC2CloudManager.AWS_SECRET_KEY_KEY, null) == null) { + System.err.println("Please set the key " + EC2CloudManager.AWS_SECRET_KEY_KEY + " in " + + System.getProperty("user.dir") + "/correct-conf/ec2-account.xml"); return; } @@ -271,23 +261,23 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { Object jobToInstancesMap = new Object(); try { - Field f1 = CloudManager.class.getDeclaredField("reservedInstances"); + Field f1 = EC2CloudManager.class.getDeclaredField("reservedInstances"); f1.setAccessible(true); reservedInstances = f1.get(cm); - Field f2 = CloudManager.class.getDeclaredField("cloudInstances"); + Field f2 = EC2CloudManager.class.getDeclaredField("cloudInstances"); f2.setAccessible(true); cloudInstances = f2.get(cm); - Field f3 = CloudManager.class.getDeclaredField("floatingInstances"); + Field f3 = EC2CloudManager.class.getDeclaredField("floatingInstances"); f3.setAccessible(true); floatingInstances = f3.get(cm); - Field f4 = CloudManager.class.getDeclaredField("floatingInstanceIDs"); + Field f4 = EC2CloudManager.class.getDeclaredField("floatingInstanceIDs"); f4.setAccessible(true); floatingInstanceIDs = f4.get(cm); - Field f5 = CloudManager.class.getDeclaredField("jobToInstancesMap"); + Field f5 = EC2CloudManager.class.getDeclaredField("jobToInstancesMap"); f5.setAccessible(true); jobToInstancesMap = f5.get(cm); @@ -302,7 +292,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { } assertEquals(0, ((Map) reservedInstances).size()); - assertEquals(0, ((List) cloudInstances).size()); + assertEquals(0, ((List) cloudInstances).size()); assertEquals(0, ((Map) floatingInstances).size()); assertEquals(0, ((Map) floatingInstanceIDs).size()); assertEquals(0, ((Map) jobToInstancesMap).size()); @@ -319,7 +309,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { } assertEquals(1, ((Map) reservedInstances).size()); - assertEquals(0, ((List) cloudInstances).size()); + assertEquals(0, ((List) cloudInstances).size()); assertEquals(0, ((Map) floatingInstances).size()); assertEquals(0, ((Map) floatingInstanceIDs).size()); assertEquals(1, ((Map) jobToInstancesMap).size()); @@ -334,7 +324,7 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { * String.class, String.class }); * m1.setAccessible(true); * Object instanceList = m1.invoke(cm, new Object[] { conf.getString("job.cloud.username", null), - * conf.getString("job.cloud.awsaccessid", null), conf.getString("job.cloud.awssecretkey", null) }); + * conf.getString("job.ec2.awsaccessid", null), conf.getString("job.ec2.awssecretkey", null) }); * assertEquals(1, ((List) instanceList).size()); * com.xerox.amazonws.ec2.ReservationDescription.Instance instance = * ((List) instanceList) @@ -352,17 +342,17 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { } assertEquals(0, ((Map) reservedInstances).size()); - assertEquals(1, ((List) cloudInstances).size()); + assertEquals(1, ((List) cloudInstances).size()); assertEquals(0, ((Map) floatingInstances).size()); assertEquals(0, ((Map) floatingInstanceIDs).size()); assertEquals(1, ((Map) jobToInstancesMap).size()); // release instance - CloudInstance ci = ((List) cloudInstances).get(0); + EC2CloudInstance ci = ((List) cloudInstances).get(0); cm.releaseAllocatedResource(jobID, conf, ci.asAllocatedResource()); assertEquals(0, ((Map) reservedInstances).size()); - assertEquals(0, ((List) cloudInstances).size()); + assertEquals(0, ((List) cloudInstances).size()); assertEquals(1, ((Map) floatingInstances).size()); assertEquals(1, ((Map) floatingInstanceIDs).size()); assertEquals(1, ((Map) jobToInstancesMap).size()); @@ -371,7 +361,8 @@ public void testRequestReleaseDestroyInstanceAndHeartBeat() { assertNotNull(instanceID); try { - Method m2 = CloudManager.class.getDeclaredMethod("destroyCloudInstance", new Class[] { Configuration.class, + Method m2 = EC2CloudManager.class.getDeclaredMethod("destroyCloudInstance", new Class[] { + Configuration.class, String.class }); m2.setAccessible(true); Object terminatedID = m2.invoke(cm, new Object[] { conf, instanceID }); diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/FloatingInstanceTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/FloatingInstanceTest.java similarity index 89% rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/FloatingInstanceTest.java rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/FloatingInstanceTest.java index 522fdc0769cf1..222d5ea59db8d 100644 --- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/FloatingInstanceTest.java +++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/FloatingInstanceTest.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import static org.junit.Assert.*; @@ -22,7 +22,7 @@ import org.junit.Test; import eu.stratosphere.nephele.instance.InstanceConnectionInfo; -import eu.stratosphere.nephele.instance.cloud.FloatingInstance; +import eu.stratosphere.nephele.instance.ec2.FloatingInstance; public class FloatingInstanceTest { @@ -30,7 +30,8 @@ public class FloatingInstanceTest { public void testHeartBeat() { FloatingInstance fi = new FloatingInstance("i-1234ABCD", new InstanceConnectionInfo(new InetSocketAddress( - "localhost", 6122).getAddress(), 6122, 6121), System.currentTimeMillis(),null, null, null); + "localhost", 6122).getAddress(), 6122, 6121), System.currentTimeMillis(), + EC2CloudManager.DEFAULT_LEASE_PERIOD, null, null, null, null); long lastHeartBeat = fi.getLastReceivedHeartBeat(); try { @@ -42,4 +43,4 @@ public void testHeartBeat() { assertTrue(fi.getLastReceivedHeartBeat() - lastHeartBeat > 0); } -} +} \ No newline at end of file diff --git a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMappingTest.java b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMappingTest.java similarity index 76% rename from nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMappingTest.java rename to nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMappingTest.java index e1b415937a39d..0c8ef04616a87 100644 --- a/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/cloud/JobToInstancesMappingTest.java +++ b/nephele/nephele-ec2cloudmanager/src/test/java/eu/stratosphere/nephele/instance/ec2/JobToInstancesMappingTest.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.nephele.instance.cloud; +package eu.stratosphere.nephele.instance.ec2; import static org.junit.Assert.*; @@ -26,8 +26,8 @@ import eu.stratosphere.nephele.instance.HardwareDescriptionFactory; import eu.stratosphere.nephele.instance.InstanceConnectionInfo; import eu.stratosphere.nephele.instance.InstanceTypeFactory; -import eu.stratosphere.nephele.instance.cloud.CloudInstance; -import eu.stratosphere.nephele.instance.cloud.JobToInstancesMapping; +import eu.stratosphere.nephele.instance.ec2.EC2CloudInstance; +import eu.stratosphere.nephele.instance.ec2.JobToInstancesMapping; import eu.stratosphere.nephele.topology.NetworkTopology; public class JobToInstancesMappingTest { @@ -36,15 +36,18 @@ public class JobToInstancesMappingTest { public void testAssignedInstances() { final NetworkTopology networkTopology = NetworkTopology.createEmptyTopology(); - final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L*1024L*1024L, 2048L*1024L*1024L); - + final HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(1, 2048L * 1024L * 1024L, + 2048L * 1024L * 1024L); + JobToInstancesMapping map = new JobToInstancesMapping("1234567", "abcdefg"); - CloudInstance ci = new CloudInstance("i-1234ABCD", InstanceTypeFactory.constructFromDescription("m1.small,1,1,2048,40,10"), + EC2CloudInstance ci = new EC2CloudInstance("i-1234ABCD", + InstanceTypeFactory.constructFromDescription("m1.small,1,1,2048,40,10"), new InstanceConnectionInfo(new InetSocketAddress("localhost", 6122).getAddress(), 6122, 6121), 1234567890, - networkTopology.getRootNode(), networkTopology, hardwareDescription, null, null); + EC2CloudManager.DEFAULT_LEASE_PERIOD, networkTopology.getRootNode(), networkTopology, hardwareDescription, + null, null); assertEquals(0, map.getNumberOfAssignedInstances()); - assertEquals(new ArrayList(), map.getAssignedInstances()); + assertEquals(new ArrayList(), map.getAssignedInstances()); map.assignInstanceToJob(ci); @@ -57,6 +60,6 @@ public void testAssignedInstances() { map.unassignInstanceFromJob(ci); assertEquals(0, map.getNumberOfAssignedInstances()); - assertEquals(new ArrayList(), map.getAssignedInstances()); + assertEquals(new ArrayList(), map.getAssignedInstances()); } -} +} \ No newline at end of file diff --git a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml index 64844a603c3be..56643ea0f2bdd 100644 --- a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml +++ b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-default.xml @@ -9,23 +9,15 @@ 6123 - jobmanager.instancemanager.classname - de.tu_berlin.cit.nephele.instance.cloud.CloudManager + jobmanager.instancemanager.cloud.classname + eu.stratosphere.nephele.instance.ec2.EC2CloudManager - cloud.ec2ws.secure - false + instancemanager.ec2.endpoint + cloud01.cit.tu-berlin.de:8773 - cloud.ec2ws.server - cloud01.cit.tu-berlin.de - - - cloud.ec2ws.port - 8773 - - - ec2.image.id + job.ec2.ami emi-E9B11C08 @@ -52,4 +44,4 @@ fs.hdfs.hdfsdefault /home/wenjun/hadoop-0.19.1/conf/hadoop-default.xml - \ No newline at end of file + diff --git a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml index 8c7821003bc59..33be1a19ceff1 100644 --- a/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml +++ b/nephele/nephele-ec2cloudmanager/src/test/resources/correct-conf/nephele-user.xml @@ -1,38 +1,32 @@ - - - cloudmgr.nrtypes - 5 - - - cloudmgr.instancetype.1 + instancemanager.ec2.type.1 m1.small,1,1,2048,40,10 - cloudmgr.instancetype.2 + instancemanager.ec2.type.2 c1.medium,2,2,4096,80,20 - cloudmgr.instancetype.3 + instancemanager.ec2.type.3 m1.large,4,4,6144,160,40 - cloudmgr.instancetype.4 + instancemanager.ec2.type.4 m1.xlarge,4,4,12288,160,60 - cloudmgr.instancetype.5 + instancemanager.ec2.type.5 c1.xlarge,8,8,28672,280,80 - cloudmgr.instancetype.defaultInstance - m1.small + instancemanager.ec2.defaulttype + 1 - \ No newline at end of file + diff --git a/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java b/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java index 044e4980e59b0..c9fb8d62031bc 100644 --- a/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java +++ b/nephele/nephele-management/src/main/java/eu/stratosphere/nephele/instance/InstanceType.java @@ -159,20 +159,11 @@ public String getIdentifier() { */ @Override public String toString() { - return this.identifier; - } - /** - * Returns a String representation of this instance in the same form as it is parsed by the - * {@link #getTypeFromString(java.lang.String)} method. - * - * @return A String representation of this instance type. - */ - public String toStringRepresentation() { - final StringBuilder bld = new StringBuilder(32); bld.append(this.identifier); - bld.append(','); + bld.append(' '); + bld.append('('); bld.append(this.numberOfComputeUnits); bld.append(','); bld.append(this.numberOfCores); @@ -182,6 +173,7 @@ public String toStringRepresentation() { bld.append(this.diskCapacity); bld.append(','); bld.append(this.pricePerHour); + bld.append(')'); return bld.toString(); } diff --git a/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java b/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java index b0cc78a380947..8b28d6e07019d 100644 --- a/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java +++ b/nephele/nephele-profiling/src/main/java/eu/stratosphere/nephele/profiling/impl/InstanceProfiler.java @@ -98,10 +98,11 @@ InternalInstanceProfilingData generateProfilingData(long timestamp) throws Profi private void updateMemoryUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException { + BufferedReader in = null; + try { - final FileReader memReader = new FileReader(PROC_MEMINFO); - final BufferedReader in = new BufferedReader(memReader); + in = new BufferedReader(new FileReader(PROC_MEMINFO)); long freeMemory = 0; long totalMemory = 0; @@ -145,6 +146,13 @@ private void updateMemoryUtilization(InternalInstanceProfilingData profilingData } catch (IOException ioe) { throw new ProfilingException("Error while reading network utilization: " + StringUtils.stringifyException(ioe)); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + } + } } } @@ -161,9 +169,11 @@ private long extractMemoryValue(String line) throws ProfilingException { private void updateNetworkUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException { + BufferedReader in = null; + try { - final BufferedReader in = new BufferedReader(new FileReader(PROC_NET_DEV)); + in = new BufferedReader(new FileReader(PROC_NET_DEV)); long receivedSum = 0; long transmittedSum = 0; @@ -184,6 +194,7 @@ private void updateNetworkUtilization(InternalInstanceProfilingData profilingDat } in.close(); + in = null; profilingData.setReceivedBytes(receivedSum - this.lastReceivedBytes); profilingData.setTransmittedBytes(transmittedSum - this.lastTramsmittedBytes); @@ -198,20 +209,30 @@ private void updateNetworkUtilization(InternalInstanceProfilingData profilingDat } catch (NumberFormatException nfe) { throw new ProfilingException("Error while reading network utilization: " + StringUtils.stringifyException(nfe)); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + } + } } } private void updateCPUUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException { + BufferedReader in = null; + try { - final BufferedReader in = new BufferedReader(new FileReader(PROC_STAT)); + in = new BufferedReader(new FileReader(PROC_STAT)); final String output = in.readLine(); if (output == null) { throw new ProfilingException("Cannot read CPU utilization, return value is null"); } in.close(); + in = null; final Matcher cpuMatcher = CPU_PATTERN.matcher(output); if (!cpuMatcher.matches()) { @@ -282,6 +303,13 @@ private void updateCPUUtilization(InternalInstanceProfilingData profilingData) t throw new ProfilingException("Error while reading CPU utilization: " + StringUtils.stringifyException(ioe)); } catch (NumberFormatException nfe) { throw new ProfilingException("Error while reading CPU utilization: " + StringUtils.stringifyException(nfe)); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + } + } } } diff --git a/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java b/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java index 11bf02a7a1809..24a5aef584943 100644 --- a/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java +++ b/nephele/nephele-queuescheduler/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueScheduler.java @@ -152,15 +152,17 @@ public void schedulJob(final ExecutionGraph executionGraph) throws SchedulingExc // Add job to the job queue (important to add job to queue before requesting instances) synchronized (this.jobQueue) { this.jobQueue.add(executionGraph); - } - // Request resources for the first stage of the job - final ExecutionStage executionStage = executionGraph.getCurrentExecutionStage(); - try { - requestInstances(executionStage); - } catch (InstanceException e) { - // TODO: Handle this error correctly - LOG.error(StringUtils.stringifyException(e)); + // Request resources for the first stage of the job + final ExecutionStage executionStage = executionGraph.getCurrentExecutionStage(); + try { + requestInstances(executionStage); + } catch (InstanceException e) { + final String exceptionMessage = StringUtils.stringifyException(e); + LOG.error(exceptionMessage); + this.jobQueue.remove(executionGraph); + throw new SchedulingException(exceptionMessage); + } } } diff --git a/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java b/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java index b09c124963082..c0c5c34a76095 100644 --- a/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java +++ b/nephele/nephele-queuescheduler/src/test/java/eu/stratosphere/nephele/jobmanager/scheduler/queue/QueueSchedulerTest.java @@ -178,7 +178,7 @@ public void testResourceAllocated() throws Exception { final DeploymentManager deploymentManager = new TestDeploymentManager(); final QueueScheduler toTest = spy(new QueueScheduler(deploymentManager, this.instanceManager)); - final JobID jobid = mock(JobID.class); + final JobID jobid = new JobID(); final AllocatedResource resource = mock(AllocatedResource.class); final List resources = new ArrayList(); resources.add(resource); diff --git a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java index f1a3c2acd82e6..2f83ec3224622 100644 --- a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java +++ b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/executiongraph/ExecutionGraph.java @@ -41,12 +41,12 @@ import eu.stratosphere.nephele.io.channels.ChannelType; import eu.stratosphere.nephele.io.channels.bytebuffered.NetworkOutputChannel; import eu.stratosphere.nephele.io.compression.CompressionLevel; +import eu.stratosphere.nephele.jobgraph.AbstractJobInputVertex; import eu.stratosphere.nephele.jobgraph.AbstractJobVertex; import eu.stratosphere.nephele.jobgraph.JobEdge; import eu.stratosphere.nephele.jobgraph.JobFileOutputVertex; import eu.stratosphere.nephele.jobgraph.JobGraph; import eu.stratosphere.nephele.jobgraph.JobID; -import eu.stratosphere.nephele.jobgraph.JobInputVertex; import eu.stratosphere.nephele.template.AbstractInputTask; import eu.stratosphere.nephele.template.AbstractInvokable; import eu.stratosphere.nephele.template.IllegalConfigurationException; @@ -546,8 +546,8 @@ private ExecutionVertex createVertex(AbstractJobVertex jobVertex, InstanceManage try { ev = new ExecutionVertex(jobVertex.getJobGraph().getJobID(), invokableClass, this, groupVertex); - } catch (Exception e) { - throw new GraphConversionException(StringUtils.stringifyException(e)); + } catch (Throwable t) { + throw new GraphConversionException(StringUtils.stringifyException(t)); } // Run the configuration check the user has provided for the vertex @@ -593,7 +593,8 @@ private ExecutionVertex createVertex(AbstractJobVertex jobVertex, InstanceManage null)); // Register input and output vertices separately - if (jobVertex instanceof JobInputVertex) { + if (jobVertex instanceof AbstractJobInputVertex) { + final InputSplit[] inputSplits; // let the task code compute the input splits @@ -611,6 +612,13 @@ private ExecutionVertex createVertex(AbstractJobVertex jobVertex, InstanceManage "BUG: JobInputVertex contained a task class which was not an input task."); } + if (inputSplits == null) { + LOG.info("Job input vertex " + jobVertex.getName() + " generated 0 input splits"); + } else { + LOG.info("Job input vertex " + jobVertex.getName() + " generated " + inputSplits.length + + " input splits"); + } + // assign input splits groupVertex.setInputSplits(inputSplits); } diff --git a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java index 4b11d65481fbb..a8a5b73629c3a 100644 --- a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java +++ b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/instance/local/LocalInstanceManager.java @@ -16,7 +16,6 @@ package eu.stratosphere.nephele.instance.local; import java.io.File; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; diff --git a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java index 3a9150dc0414d..1abfe86afd4c1 100644 --- a/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java +++ b/nephele/nephele-server/src/main/java/eu/stratosphere/nephele/jobmanager/scheduler/local/LocalScheduler.java @@ -103,125 +103,4 @@ public void schedulJob(ExecutionGraph executionGraph) throws SchedulingException final ExecutionStage stage = executionGraph.getStage(i); stage.collectRequiredInstanceTypes(instanceRequestMap, ExecutionState.CREATED); - final Iterator> it = instanceRequestMap.getMinimumIterator(); - while (it.hasNext()) { - - final Map.Entry entry = it.next(); - final InstanceTypeDescription descr = availableInstances.get(entry.getKey()); - if (descr == null) { - throw new SchedulingException("Unable to schedule job: No instance of type " + entry.getKey() - + " available"); - } - - if (descr.getMaximumNumberOfAvailableInstances() != -1 - && descr.getMaximumNumberOfAvailableInstances() < entry.getValue().intValue()) { - throw new SchedulingException("Unable to schedule job: " + entry.getValue().intValue() - + " instances of type " + entry.getKey() + " required, but only " - + descr.getMaximumNumberOfAvailableInstances() + " are available"); - } - } - } - - // Subscribe to job status notifications - executionGraph.registerJobStatusListener(this); - - final ExecutionGraphIterator it2 = new ExecutionGraphIterator(executionGraph, true); - while (it2.hasNext()) { - - final ExecutionVertex vertex = it2.next(); - vertex.getEnvironment().registerExecutionListener(new LocalExecutionListener(this, vertex)); - } - - // Register the scheduler as an execution stage listener - executionGraph.registerExecutionStageListener(this); - - // Add job to the job queue (important to add job to queue before requesting instances) - synchronized (this.jobQueue) { - this.jobQueue.add(executionGraph); - } - - // Request resources for the first stage of the job - final ExecutionStage executionStage = executionGraph.getCurrentExecutionStage(); - try { - requestInstances(executionStage); - } catch (InstanceException e) { - // TODO: Handle this error correctly - LOG.error(StringUtils.stringifyException(e)); - } - } - - /** - * {@inheritDoc} - */ - @Override - public ExecutionGraph getExecutionGraphByID(JobID jobID) { - - synchronized (this.jobQueue) { - - final Iterator it = this.jobQueue.iterator(); - while (it.hasNext()) { - - final ExecutionGraph executionGraph = it.next(); - if (executionGraph.getJobID().equals(jobID)) { - return executionGraph; - } - } - } - - return null; - } - - - - /** - * {@inheritDoc} - */ - @Override - public void allocatedResourcesDied(final JobID jobID, final List allocatedResource) { - // TODO Auto-generated method stub - - } - - /** - * {@inheritDoc} - */ - @Override - public void shutdown() { - - synchronized (this.jobQueue) { - this.jobQueue.clear(); - } - - } - - /** - * {@inheritDoc} - */ - @Override - public void jobStatusHasChanged(final ExecutionGraph executionGraph, final InternalJobStatus newJobStatus, - final String optionalMessage) { - - if (newJobStatus == InternalJobStatus.FAILED || newJobStatus == InternalJobStatus.FINISHED - || newJobStatus == InternalJobStatus.CANCELED) { - removeJobFromSchedule(executionGraph); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void nextExecutionStageEntered(final JobID jobID, final ExecutionStage executionStage) { - - // Request new instances if necessary - try { - requestInstances(executionStage); - } catch (InstanceException e) { - // TODO: Handle this error correctly - LOG.error(StringUtils.stringifyException(e)); - } - - // Deploy the assigned vertices - deployAssignedVertices(executionStage.getExecutionGraph()); - } -} + final Iterator getInputSplitType() @Override public void open(GenericInputSplit split) throws IOException { - this.partitionNumber = split.getPartitionNumber(); + this.partitionNumber = split.getSplitNumber(); } /* (non-Javadoc) diff --git a/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java b/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java index 3741bbe89c47f..648a9e672b583 100644 --- a/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java +++ b/pact/pact-compiler/src/main/java/eu/stratosphere/pact/compiler/jobgen/JobGraphGenerator.java @@ -27,12 +27,12 @@ import eu.stratosphere.nephele.io.channels.ChannelType; import eu.stratosphere.nephele.io.compression.CompressionLevel; import eu.stratosphere.nephele.jobgraph.AbstractJobVertex; -import eu.stratosphere.nephele.jobgraph.JobGenericInputVertex; -import eu.stratosphere.nephele.jobgraph.JobGenericOutputVertex; +import eu.stratosphere.nephele.jobgraph.JobInputVertex; +import eu.stratosphere.nephele.jobgraph.JobOutputVertex; import eu.stratosphere.nephele.jobgraph.JobGraph; import eu.stratosphere.nephele.jobgraph.JobGraphDefinitionException; import eu.stratosphere.nephele.jobgraph.JobInputVertex; -import eu.stratosphere.nephele.jobgraph.JobOutputVertex; +import eu.stratosphere.nephele.jobgraph.AbstractJobOutputVertex; import eu.stratosphere.nephele.jobgraph.JobTaskVertex; import eu.stratosphere.pact.common.contract.GenericDataSink; import eu.stratosphere.pact.common.contract.GenericDataSource; @@ -579,7 +579,7 @@ private JobInputVertex generateDataSourceVertex(OptimizerNode sourceNode) throws GenericDataSource contract = dsn.getPactContract(); // create task vertex - JobGenericInputVertex sourceVertex = new JobGenericInputVertex(contract.getName(), this.jobGraph); + JobInputVertex sourceVertex = new JobInputVertex(contract.getName(), this.jobGraph); // set task class sourceVertex.setInputClass(DataSourceTask.class); @@ -608,13 +608,13 @@ private JobInputVertex generateDataSourceVertex(OptimizerNode sourceNode) throws * @return * @throws CompilerException */ - private JobOutputVertex generateDataSinkVertex(OptimizerNode sinkNode) throws CompilerException + private AbstractJobOutputVertex generateDataSinkVertex(OptimizerNode sinkNode) throws CompilerException { DataSinkNode sNode = (DataSinkNode) sinkNode; GenericDataSink sinkContract = sNode.getPactContract(); // create task vertex - JobGenericOutputVertex sinkVertex = new JobGenericOutputVertex(sinkNode.getPactContract().getName(), this.jobGraph); + JobOutputVertex sinkVertex = new JobOutputVertex(sinkNode.getPactContract().getName(), this.jobGraph); // set task class sinkVertex.setOutputClass(DataSinkTask.class); diff --git a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java index 05396b2174fe6..6f99a22f1773e 100644 --- a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java +++ b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/TPCHQuery3.java @@ -48,17 +48,16 @@ * The TPC-H is a decision support benchmark on relational data. * Its documentation and the data generator (DBGEN) can be found * on http://www.tpc.org/tpch/ .This implementation is tested with - * the DB2 data format. - * THe PACT program implements a modified version of the query 3 of + * the DB2 data format. + * THe PACT program implements a modified version of the query 3 of * the TPC-H benchmark including one join, some filtering and an * aggregation. - * * SELECT l_orderkey, o_shippriority, sum(l_extendedprice) as revenue - * FROM orders, lineitem - * WHERE l_orderkey = o_orderkey - * AND o_orderstatus = "X" - * AND YEAR(o_orderdate) > Y - * AND o_orderpriority LIKE "Z%" + * FROM orders, lineitem + * WHERE l_orderkey = o_orderkey + * AND o_orderstatus = "X" + * AND YEAR(o_orderdate) > Y + * AND o_orderpriority LIKE "Z%" * GROUP BY l_orderkey, o_shippriority; */ public class TPCHQuery3 implements PlanAssembler, PlanAssemblerDescription { @@ -81,8 +80,10 @@ public N_IntStringPair() { /** * Initializes the concatenation of integer and string. * - * @param first Integer value for concatenating - * @param second String value for concatenating + * @param first + * Integer value for concatenating + * @param second + * String value for concatenating */ public N_IntStringPair(PactInteger first, PactString second) { super(first, second); @@ -93,14 +94,14 @@ public N_IntStringPair(PactInteger first, PactString second) { * Map PACT implements the filter on the orders table. The SameKey * OutputContract is annotated because the key does not change during * filtering. - * */ @SameKey public static class FilterO extends MapStub { private int yearFilter; + private String prioFilter; - + @Override public void configure(Configuration parameters) { this.yearFilter = parameters.getInteger("YEAR_FILTER", 1990); @@ -109,14 +110,12 @@ public void configure(Configuration parameters) { /** * Filters the orders table by year, orderstatus and orderpriority - * - * o_orderstatus = "X" - * AND YEAR(o_orderdate) > Y - * AND o_orderpriority LIKE "Z" - * - * Output Schema: - * Key: ORDERKEY - * Value: 0:ORDERKEY, 1:SHIPPRIORITY + * o_orderstatus = "X" + * AND YEAR(o_orderdate) > Y + * AND o_orderpriority LIKE "Z" + * Output Schema: + * Key: ORDERKEY + * Value: 0:ORDERKEY, 1:SHIPPRIORITY */ @Override public void map(final PactInteger oKey, final Tuple value, final Collector out) { @@ -143,22 +142,19 @@ public void map(final PactInteger oKey, final Tuple value, final Collector { /** - * Does the projection on the LineItem table - * + * Does the projection on the LineItem table * Output Schema: - * Key: ORDERKEY - * Value: 0:ORDERKEY, 1:EXTENDEDPRICE + * Key: ORDERKEY + * Value: 0:ORDERKEY, 1:EXTENDEDPRICE */ @Override public void map(PactInteger oKey, Tuple value, Collector out) { @@ -168,23 +164,20 @@ public void map(PactInteger oKey, Tuple value, Collector out } /** - * Match PACT realizes the join between LineItem and Order table. The + * Match PACT realizes the join between LineItem and Order table. The * SuperKey OutputContract is annotated because the new key is * built of the keys of the inputs. - * */ @SuperKey public static class JoinLiO extends MatchStub { /** - * Implements the join between LineItem and Order table on the + * Implements the join between LineItem and Order table on the * order key. - * * WHERE l_orderkey = o_orderkey - * * Output Schema: - * Key: ORDERKEY, SHIPPRIORITY - * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICE + * Key: ORDERKEY, SHIPPRIORITY + * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICE */ @Override public void match(PactInteger oKey, Tuple oVal, Tuple liVal, Collector out) { @@ -199,24 +192,20 @@ public void match(PactInteger oKey, Tuple oVal, Tuple liVal, Collector { /** - * Does the aggregation of the query. - * + * Does the aggregation of the query. * sum(l_extendedprice) as revenue * GROUP BY l_orderkey, o_shippriority; - * * Output Schema: - * Key: ORDERKEY - * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICESUM - * + * Key: ORDERKEY + * Value: 0:ORDERKEY, 1:SHIPPRIORITY, 2:EXTENDEDPRICESUM */ @Override public void reduce(N_IntStringPair oKeyShipPrio, Iterator values, Collector out) { @@ -269,10 +258,16 @@ public void combine(N_IntStringPair oKeyShipPrio, Iterator values, Collec public Plan getPlan(final String... args) { // parse program parameters - int noSubtasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1); - String ordersPath = (args.length > 1 ? args[1] : ""); + int noSubtasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1); + String ordersPath = (args.length > 1 ? args[1] : ""); String lineitemsPath = (args.length > 2 ? args[2] : ""); - String output = (args.length > 3 ? args[3] : ""); + String output = (args.length > 3 ? args[3] : ""); + + // optional parameters to run this job on Amazon EC2 + String awsAccessID = (args.length > 4 ? args[4] : null); + String awsSecretKey = (args.length > 5 ? args[5] : null); + String awsImageID = (args.length > 6 ? args[6] : null); + String sshKeyPair = (args.length > 7 ? args[7] : null); // create DataSourceContract for Orders input FileDataSourceContract orders = new FileDataSourceContract( @@ -342,8 +337,22 @@ public Plan getPlan(final String... args) { joinLiO.setSecondInput(projectLi); projectLi.setInput(lineitems); + Plan plan = new Plan(result, "TPCH Q3"); + if (awsAccessID != null) { + plan.getPlanConfiguration().setNepheleString("job.ec2.awsaccessid", awsAccessID); + } + if (awsSecretKey != null) { + plan.getPlanConfiguration().setNepheleString("job.ec2.awssecretkey", awsSecretKey); + } + if (awsImageID != null) { + plan.getPlanConfiguration().setNepheleString("job.ec2.ami", awsImageID); + } + if (sshKeyPair != null) { + plan.getPlanConfiguration().setNepheleString("job.ec2.sshkeypair", sshKeyPair); + } + // return the PACT plan - return new Plan(result, "TPCH Q3"); + return plan; } /** diff --git a/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh b/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh index 59b5a8d0a2078..4f5fe1a5ae25d 100755 --- a/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh +++ b/stratosphere-dist/src/main/stratosphere-bin/bin/nephele-jobmanager.sh @@ -67,8 +67,6 @@ constructJobManagerClassPath() { add=1 elif [[ "$jarfile" =~ 'commons-httpclient' ]]; then add=1 - elif [[ "$jarfile" =~ 'typica' ]]; then - add=1 elif [[ "$jarfile" =~ 'pact-common' ]]; then add=1 elif [[ "$jarfile" =~ 'pact-runtime' ]]; then diff --git a/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml b/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml index ae6e119b6ec68..7c06fa65bf323 100644 --- a/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml +++ b/stratosphere-dist/src/main/stratosphere-bin/conf/nephele-user.xml @@ -63,7 +63,28 @@ instancemanager.cluster.defaulttype 1 - + + + + + instancemanager.ec2.type.1 + t1.micro,2,1,613,8,3 + + + + instancemanager.ec2.type.2 + m1.large,4,2,7680,850,38 + + + + instancemanager.ec2.type.3 + m1.xlarge,8,4,15360,1690,76 + + + + instancemanager.ec2.defaulttype + 1 + jobmanager.instancemanager.cloud.classname - eu.stratosphere.nephele.instance.cloud.CloudManager + eu.stratosphere.nephele.instance.ec2.EC2CloudManager