Merge pull request #151 from sanity/regressionTrees

Regression trees

pom.xml

@@ -32,7 +32,7 @@
  be accompanied by a bump in version number, regardless of how minor the change.
  0.10.1 -->
 
- <version> 0.10.9</version>
+ <version> 0.10.10</version>
 
  <properties>
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -56,6 +56,11 @@
  <artifactId>super-csv</artifactId>
  <version>2.2.0</version>
  </dependency>
+ <dependency>
+ <groupId>com.googlecode.efficient-java-matrix-library</groupId>
+ <artifactId>ejml</artifactId>
+ <version>0.23</version>
+ </dependency>
  <dependency>
  <groupId>net.sf.opencsv</groupId>
  <artifactId>opencsv</artifactId>

src/main/java/quickml/data/AttributesMap.java

@@ -15,7 +15,8 @@ public class AttributesMap implements Map<String, Serializable>, Serializable {
  private final Map<String, Serializable> map;
 
  public AttributesMap(Map<String, Serializable> map) {
- this.map = map;
+ this.map = new HashMap<String, Serializable>();
+ this.map.putAll(map);
  }
 
  public AttributesMap() {

src/main/java/quickml/data/OnespotDateTimeExtractor.java

@@ -3,10 +3,11 @@
 
 import org.joda.time.DateTime;
 import quickml.data.instances.ClassifierInstance;
+import quickml.data.instances.InstanceWithAttributesMap;
 import quickml.supervised.crossValidation.utils.DateTimeExtractor;
 
 
-public class OnespotDateTimeExtractor<T extends ClassifierInstance> implements DateTimeExtractor<T> {
+public class OnespotDateTimeExtractor<T extends InstanceWithAttributesMap> implements DateTimeExtractor<T> {
 
  @Override
  public DateTime extractDateTime(T instance) {

src/main/java/quickml/data/instances/RegressionInstance.java

@@ -0,0 +1,25 @@
+package quickml.data.instances;
+
+import quickml.data.AttributesMap;
+
+import java.io.Serializable;
+
+/**
+ * Created by alexanderhawk on 4/14/15.
+ */
+public class RegressionInstance extends InstanceWithAttributesMap<Double> {
+ public RegressionInstance(AttributesMap attributes, Double label) {
+ super(attributes, label, 1.0);
+ }
+ public RegressionInstance(AttributesMap attributes, Double label, double weight) {
+ super(attributes, label, weight);
+ }
+ public RegressionInstance(AttributesMap attributes, Double label, double weight, double alternativeTarget) {
+ super(attributes, label, weight);
+ this.alternativeTarget = alternativeTarget;
+ }
+ public double alternativeTarget;
+ public long id;
+
+}
+

src/main/java/quickml/data/instances/SparseRegressionInstance.java

@@ -0,0 +1,84 @@
+package quickml.data.instances;
+
+import org.javatuples.Pair;
+import quickml.data.AttributesMap;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Created by alexanderhawk on 10/12/15.
+ */
+public class SparseRegressionInstance extends RegressionInstance {
+ private int[] indicesOfCorrespondingWeights;
+ private double[] values;
+
+ public SparseRegressionInstance(AttributesMap attributes, Double label, Map<String, Integer> nameToValueIndexMap) {
+ super(attributes, label);
+ setIndicesAndValues(attributes, nameToValueIndexMap);
+ }
+
+ public SparseRegressionInstance(AttributesMap attributes, Double label, double weight, Map<String, Integer> nameToValueIndexMap) {
+ super(attributes, label, weight);
+ setIndicesAndValues(attributes, nameToValueIndexMap);
+ }
+
+ private void setIndicesAndValues(AttributesMap attributes, Map<String, Integer> nameToIndexMap) {
+ indicesOfCorrespondingWeights = new int[attributes.size()+1];
+ values = new double[attributes.size()+1];
+ //add bias term
+ indicesOfCorrespondingWeights[0] = 0;
+ values[0] = 1.0;
+ //add non bias terms
+ int i = 1;
+ for (Map.Entry<String, Serializable> entry : attributes.entrySet()) {
+ if (!(entry.getValue() instanceof Double)) {
+ throw new RuntimeException("wrong type of values in attributes");
+ }
+ int valueIndex = nameToIndexMap.get(entry.getKey());
+ indicesOfCorrespondingWeights[i] = valueIndex;
+ values[i] = (Double)entry.getValue();
+ i++;
+ }
+ }
+
+ public static double[] getArrayOfValues(RegressionInstance regressionInstance, Map<String, Integer> nameToIndexMap, boolean useBias){
+ int numAttributes = regressionInstance.getAttributes().size();
+ AttributesMap attributesMap = regressionInstance.getAttributes();
+ double[] valuesArray;
+ int attributeIndex = 0;
+
+ if (useBias) {
+ valuesArray = new double[numAttributes + 1];
+ valuesArray[0] = 1.0;
+ attributeIndex++;
+ } else {
+ valuesArray = new double[numAttributes];
+ }
+ for (Map.Entry<String, Serializable> attributeEntry : attributesMap.entrySet()) {
+ attributeIndex = nameToIndexMap.get(attributeEntry.getKey());
+ valuesArray[attributeIndex] = (Double)attributeEntry.getValue();
+ }
+ return valuesArray;
+ }
+
+ @Override
+ public AttributesMap getAttributes() {
+ return super.getAttributes();
+ }
+
+ public Pair<int[], double[]> getSparseAttributes(){
+ return new Pair<>(indicesOfCorrespondingWeights, values);
+ }
+
+ public double dotProduct(double[] omega) {
+ double result = 0;
+ for (int i = 0; i< indicesOfCorrespondingWeights.length; i++) {
+ int indexOfFeature = indicesOfCorrespondingWeights[i];
+ double valueOfFeature = values[i];
+ result+= omega[indexOfFeature]* valueOfFeature;
+ }
+ return result;
+ }
+}

src/main/java/quickml/experiments/GeoDistance.java

@@ -0,0 +1,76 @@
+package quickml.experiments;
+
+/**
+ * Created by alexanderhawk on 4/9/16.
+ *
+ import java.util.*;
+ import java.lang.*;
+ import java.io.*;
+ */
+public class GeoDistance {
+
+/*::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::*/
+/*:: :*/
+/*:: This routine calculates the distance between two points (given the :*/
+/*:: latitude/longitude of those points). It is being used to calculate :*/
+/*:: the distance between two locations using GeoDataSource (TM) prodducts :*/
+/*:: :*/
+/*:: Definitions: :*/
+/*:: South latitudes are negative, east longitudes are positive :*/
+/*:: :*/
+/*:: Passed to function: :*/
+/*:: lat1, lon1 = Latitude and Longitude of point 1 (in decimal degrees) :*/
+/*:: lat2, lon2 = Latitude and Longitude of point 2 (in decimal degrees) :*/
+/*:: unit = the unit you desire for results :*/
+/*:: where: 'M' is statute miles (default) :*/
+/*:: 'K' is kilometers :*/
+/*:: 'N' is nautical miles :*/
+/*:: Worldwide cities and other features databases with latitude longitude :*/
+/*:: are available at http:https://www.geodatasource.com :*/
+/*:: :*/
+/*:: For enquiries, please contact [email protected] :*/
+/*:: :*/
+/*:: Official Web site: http:https://www.geodatasource.com :*/
+/*:: :*/
+/*:: GeoDataSource.com (C) All Rights Reserved 2015 :*/
+/*:: :*/
+/*::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::*/
+
+
+ public static void main (String[] args) throws java.lang.Exception
+ {
+ System.out.println(distance(32.9697, -96.80322, 29.46786, -98.53506, "M") + " Miles\n");
+ System.out.println(distance(32.9697, -96.80322, 29.46786, -98.53506, "K") + " Kilometers\n");
+ System.out.println(distance(32.9697, -96.80322, 29.46786, -98.53506, "N") + " Nautical Miles\n");
+ }
+
+ public static double distance(double lat1, double lon1, double lat2, double lon2, String unit) {
+ double theta = lon1 - lon2;
+ double dist = Math.sin(deg2rad(lat1)) * Math.sin(deg2rad(lat2)) + Math.cos(deg2rad(lat1)) * Math.cos(deg2rad(lat2)) * Math.cos(deg2rad(theta));
+ dist = Math.acos(dist);
+ dist = rad2deg(dist);
+ dist = dist * 60 * 1.1515;
+ if (unit == "K") {
+ dist = dist * 1.609344;
+ } else if (unit == "N") {
+ dist = dist * 0.8684;
+ }
+
+ return (dist);
+ }
+
+ /*:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::*/
+ /*:: This function converts decimal degrees to radians :*/
+ /*:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::*/
+ private static double deg2rad(double deg) {
+ return (deg * Math.PI / 180.0);
+ }
+
+ /*:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::*/
+ /*:: This function converts radians to decimal degrees :*/
+ /*:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::*/
+ private static double rad2deg(double rad) {
+ return (rad * 180 / Math.PI);
+ }
+ }
+

src/main/java/quickml/experiments/kin88nm.java

@@ -1,11 +1,26 @@
 package quickml.experiments;
 
+import org.javatuples.Pair;
 import quickml.data.instances.ClassifierInstance;
+import quickml.data.instances.RegressionInstance;
+import quickml.supervised.crossValidation.RegressionLossChecker;
+import quickml.supervised.crossValidation.SimpleCrossValidator;
+import quickml.supervised.crossValidation.data.FoldedData;
+import quickml.supervised.crossValidation.lossfunctions.regressionLossFunctions.RegressionRMSELossFunction;
+import quickml.supervised.ensembles.randomForest.randomDecisionForest.RandomDecisionForest;
+import quickml.supervised.ensembles.randomForest.randomRegressionForest.RandomRegressionForest;
+import quickml.supervised.ensembles.randomForest.randomRegressionForest.RandomRegressionForestBuilder;
+import quickml.supervised.tree.attributeIgnoringStrategies.IgnoreAttributesWithConstantProbability;
+import quickml.supervised.tree.regressionTree.OptimizedRegressionForests;
+import quickml.supervised.tree.regressionTree.RegressionTree;
+import quickml.supervised.tree.regressionTree.RegressionTreeBuilder;
 import quickml.utlities.CSVToInstanceReader;
 import quickml.utlities.CSVToInstanceReaderBuilder;
 import quickml.utlities.selectors.NumericSelector;
 
+import java.io.Serializable;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Created by alexanderhawk on 9/16/15.
@@ -23,10 +38,43 @@ public boolean isNumeric(String columnName) {
  public String cleanValue(String value) {
  return value;
  }
- }).delimiter(',').collumnNameForLabel("y");
+ }).delimiter(',').collumnNameForLabel("x8").hasHeader(false);
  CSVToInstanceReader csvToInstanceReader =csvToInstanceReaderBuilder.buildCsvReader();
  try {
- List<ClassifierInstance> allTrainingData = csvToInstanceReader.readCsv("");
+ List<RegressionInstance> allTrainingData = csvToInstanceReader.readRegressionInstancesFromCsv("uci-20070111-kin8nm.csv");
+ List<RegressionInstance> trainData = csvToInstanceReader.readRegressionInstancesFromCsv("/Users/alexanderhawk/msda-denoising/spearmint/data/kin8nm_train.csv");
+ List<RegressionInstance> valData = csvToInstanceReader.readRegressionInstancesFromCsv("/Users/alexanderhawk/msda-denoising/spearmint/data/kin8nm_test.csv");
+ RegressionTreeBuilder<RegressionInstance> regressionTreeBuilder
+ = new RegressionTreeBuilder<>()
+ .degreeOfGainRatioPenalty(1.0)
+ .attributeIgnoringStrategy(new IgnoreAttributesWithConstantProbability(0.5))
+ .maxDepth(18)
+ .minLeafInstances(2)
+ .minSplitFraction(0.1)
+ .numNumericBins(10)
+ .numSamplesPerNumericBin(20);
+ RandomRegressionForestBuilder<RegressionInstance> regressionForestBuilder = new RandomRegressionForestBuilder<>(regressionTreeBuilder).numTrees(400);
+ //RegressionTree regressionTree = regressionTreeBuilder.buildPredictiveModel(trainData);
+
+
+ RandomRegressionForest randomRegressionForest = regressionForestBuilder.buildPredictiveModel(trainData);
+ //Pair<Map<String, Serializable>, RandomRegressionForest> randomForestPair = OptimizedRegressionForests.<RegressionInstance>getOptimizedRandomForest(trainData);
+ //RandomRegressionForest randomRegressionForest = randomForestPair.getValue1();
+
+
+ double loss =0;
+ for (RegressionInstance instance: valData) {
+ loss+=(instance.getLabel() - randomRegressionForest.predict(instance.getAttributes()))
+ *(instance.getLabel() - randomRegressionForest.predict(instance.getAttributes()));
+ }
+ loss=Math.sqrt(loss/valData.size());
+ System.out.println("loss " + loss);
+
+ SimpleCrossValidator simpleCrossValidator = new SimpleCrossValidator(regressionTreeBuilder,
+ new RegressionLossChecker(new RegressionRMSELossFunction()), new FoldedData(allTrainingData, 8, 8));
+ // double loss=simpleCrossValidator.getLossForModel();
+
+ System.out.println("here");
  } catch (Exception e) {
  e.printStackTrace();
  throw new RuntimeException(e);

src/main/java/quickml/supervised/PredictiveModelsFromPreviousVersionsToBenchMarkAgainst/oldTree/OldClassificationCounter.java

@@ -6,6 +6,7 @@
 import org.javatuples.Pair;
 import quickml.collections.ValueSummingMap;
 import quickml.data.instances.ClassifierInstance;
+import quickml.supervised.tree.decisionTree.valueCounters.ClassificationCounter;
 
 import java.io.Serializable;
 import java.util.*;
@@ -25,7 +26,7 @@ public OldClassificationCounter(OldClassificationCounter classificationCounter)
  this.counts.putAll(classificationCounter.counts);
  }
 
- public OldClassificationCounter(quickml.supervised.tree.decisionTree.valueCounters.ClassificationCounter classificationCounter) {
+ public OldClassificationCounter(ClassificationCounter classificationCounter) {
  this.counts.putAll(classificationCounter.getCounts());
  }
 

src/main/java/quickml/supervised/PredictiveModelsFromPreviousVersionsToBenchMarkAgainst/oldTree/oldAttributeIgnoringStrategies/IgnoreAttributesWithConstantProbability.java

@@ -29,6 +29,10 @@ public boolean ignoreAttribute(String attribute, OldBranch parent) {
  return false;
  }
 
+ public double getIgnoreAttributeProbability() {
+ return ignoreAttributeProbability;
+ }
+
  @Override
  public String toString(){
  return "ignoreAttributeProbability = " + ignoreAttributeProbability;

src/main/java/quickml/supervised/Utils.java

@@ -9,6 +9,7 @@
 import quickml.data.instances.Instance;
 import quickml.data.instances.InstanceWithAttributesMap;
 import quickml.data.PredictionMap;
+import quickml.data.instances.RegressionInstance;
 import quickml.supervised.classifier.Classifier;
 import quickml.supervised.classifier.logisticRegression.SparseClassifierInstance;
 import quickml.supervised.crossValidation.PredictionMapResult;
@@ -22,8 +23,11 @@
 import quickml.supervised.tree.nodes.Node;
 import quickml.supervised.tree.summaryStatistics.ValueCounter;
 
+import java.io.BufferedWriter;
+import java.io.IOException;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.nio.DoubleBuffer;
 import java.util.*;
 
 /**
@@ -69,6 +73,21 @@ public static List<LabelPredictionWeight<Double, Double>> getRegLabelsPrediction
  return results;
  }
 
+ public static List<LabelPredictionWeight<Double, Double>> getRegLabelsPredictionsWeights(PredictiveModel<AttributesMap, Double> predictiveModel, List<? extends Instance<AttributesMap, Double>> validationSet, BufferedWriter bw) {
+ ArrayList<LabelPredictionWeight<Double, Double>> results = new ArrayList<>();
+ for (Instance<AttributesMap, Double> instance : validationSet) {
+ Double prediction = predictiveModel.predict(instance.getAttributes());
+ Long id = ((RegressionInstance)instance).id;
+ results.add(new LabelPredictionWeight<Double, Double>(instance.getLabel(), prediction, instance.getWeight()));
+ try {
+ bw.write(""+id + "," + instance.getLabel() + "," + prediction + "\n");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return results;
+ }
+
  public static PredictionMapResults calcResultPredictions(Classifier predictiveModel, List<? extends InstanceWithAttributesMap<?>> validationSet) {
  ArrayList<PredictionMapResult> results = new ArrayList<>();
  for (InstanceWithAttributesMap<?> instance : validationSet) {
@@ -86,6 +105,15 @@ public static PredictionMapResults calcResultpredictionsWithoutAttrs(Classifier
  return new PredictionMapResults(results);
  }
 
+ public static List<LabelPredictionWeight<Double, Double>> calcLabelPredictionsWeightsWithoutAttrs(PredictiveModel<AttributesMap, Double> predictiveModel, List<? extends RegressionInstance> validationSet, Set<String> attributesToIgnore) {
+ ArrayList<LabelPredictionWeight<Double, Double>> results = new ArrayList<>();
+ for (RegressionInstance instance : validationSet) {
+ Double prediction = predictiveModel.predictWithoutAttributes(instance.getAttributes(), attributesToIgnore);
+ results.add(new LabelPredictionWeight<Double, Double>(prediction, instance.getLabel(), instance.getWeight()));
+ }
+ return results;
+ }
+
  public static <T extends InstanceWithAttributesMap<?>> void sortTrainingInstancesByTime(List<T> trainingData, final DateTimeExtractor<T> dateTimeExtractor) {
  Collections.sort(trainingData, new Comparator<T>() {
  @Override