GP-3765 Updated AutoVersionTracking script and task to process implie…

…d matches when running from the script if user wants them.

Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java

@@ -23,6 +23,7 @@
 import ghidra.feature.vt.api.util.VTOptions;
 import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask;
 import ghidra.feature.vt.gui.plugin.VTController;
+import ghidra.feature.vt.gui.util.VTOptionDefines;
 import ghidra.framework.model.DomainFolder;
 import ghidra.framework.options.ToolOptions;
 import ghidra.framework.plugintool.PluginTool;
@@ -68,6 +69,9 @@ public void run() throws Exception {
  return;
  }
 
+ boolean autoCreateImpliedMatches = askYesNo("Implied Matches?",
+ "Would you like the script to figure out implied matches from any matches it creates?");
+
  // Need to end the script transaction or it interferes with vt things that need locks
  end(true);
 
@@ -76,12 +80,32 @@ public void run() throws Exception {
 
  folder.createFile(name, session, monitor);
 
+ ToolOptions options = getOptions();
+
+ boolean originalImpliedMatchSetting =
+ options.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
+ options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, autoCreateImpliedMatches);
+
  AutoVersionTrackingTask autoVtTask =
- new AutoVersionTrackingTask(session, getOptions(), 1.0, 10.0);
+ new AutoVersionTrackingTask(session, options, 0.95, 10.0);
+
 
  TaskLauncher.launch(autoVtTask);
+
+
+ // if not running headless user can decide whether to save or not
+ // if running headless - must save here or nothing that was done in this script will be
+ // accessible later.
+ if (isRunningHeadless()) {
+ session.save();
+ }
+ options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, originalImpliedMatchSetting);
+
+ println(autoVtTask.getStatusMsg());
  }
 
+
  private ToolOptions getOptions() {
  ToolOptions vtOptions = new VTOptions("Dummy");
  PluginTool tool = state.getTool();

Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java

@@ -15,28 +15,61 @@
  */
 package ghidra.feature.vt.gui.actions;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 
 import javax.swing.SwingConstants;
 
-import ghidra.feature.vt.api.correlator.program.*;
-import ghidra.feature.vt.api.main.*;
+import ghidra.feature.vt.api.correlator.program.CombinedFunctionAndDataReferenceProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.DataReferenceProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.DuplicateFunctionMatchProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactDataMatchProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactMatchBytesProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactMatchInstructionsProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactMatchMnemonicsProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.FunctionReferenceProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.SymbolNameProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.VTAbstractReferenceProgramCorrelatorFactory;
+import ghidra.feature.vt.api.main.VTAssociation;
+import ghidra.feature.vt.api.main.VTAssociationManager;
+import ghidra.feature.vt.api.main.VTAssociationStatus;
+import ghidra.feature.vt.api.main.VTAssociationType;
+import ghidra.feature.vt.api.main.VTMarkupItem;
+import ghidra.feature.vt.api.main.VTMatch;
+import ghidra.feature.vt.api.main.VTMatchSet;
+import ghidra.feature.vt.api.main.VTProgramCorrelator;
+import ghidra.feature.vt.api.main.VTProgramCorrelatorFactory;
+import ghidra.feature.vt.api.main.VTSession;
 import ghidra.feature.vt.api.util.VTAssociationStatusException;
 import ghidra.feature.vt.api.util.VTOptions;
 import ghidra.feature.vt.gui.plugin.AddressCorrelatorManager;
 import ghidra.feature.vt.gui.task.ApplyMarkupItemTask;
+import ghidra.feature.vt.gui.util.ImpliedMatchUtils;
 import ghidra.feature.vt.gui.util.MatchInfo;
 import ghidra.feature.vt.gui.util.MatchInfoFactory;
+import ghidra.feature.vt.gui.util.VTOptionDefines;
 import ghidra.framework.options.ToolOptions;
 import ghidra.program.model.address.Address;
 import ghidra.program.model.address.AddressSetView;
 import ghidra.program.model.lang.OperandType;
-import ghidra.program.model.listing.*;
+import ghidra.program.model.listing.CodeUnit;
+import ghidra.program.model.listing.CodeUnitIterator;
+import ghidra.program.model.listing.Function;
+import ghidra.program.model.listing.FunctionManager;
+import ghidra.program.model.listing.Instruction;
+import ghidra.program.model.listing.InstructionIterator;
+import ghidra.program.model.listing.Program;
 import ghidra.program.util.ListingDiff;
 import ghidra.util.Msg;
 import ghidra.util.exception.AssertException;
 import ghidra.util.exception.CancelledException;
-import ghidra.util.task.*;
+import ghidra.util.task.Task;
+import ghidra.util.task.TaskMonitor;
+import ghidra.util.task.WrappingTaskMonitor;
 import util.CollectionUtils;
 
 /**
@@ -142,9 +175,24 @@ private void doRun(TaskMonitor realMonitor) throws CancelledException {
  int count = 0;
  monitor.doInitialize(NUM_CORRELATORS);
 
+ // save user option and use to determine whether to handle implied matches at all later
+ boolean autoCreateImpliedMatches =
+ applyOptions.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
+ // Turn off auto implied matches and handle later if user had that option set
+ // This is because when run from the VT GUI action implied matches are created automatically
+ // by the VT controller when the option is set but they are not created when called from a 
+ // script since there is no VT controller in that case. If allowed to happen in 
+ // GUI then they will happen twice when called later in this task and the implied match 
+ // votes will be wrong. This Task doesn't know if called from GUI or script so this is 
+ // klunky but will make sure they are only processed once and will make sure the user option
+ // is put back the way the user had it. 
+ applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
  // Use default options for all of the "exact" correlators; passed in options for the others
  VTOptions options;
 
+
  // Run the correlators in the following order:
  // Do this one first because we don't want it to find ones that get markup applied by later
  // correlators
@@ -266,6 +314,63 @@ private void doRun(TaskMonitor realMonitor) throws CancelledException {
  " with some apply markup errors. See the log or the markup table for more details";
  }
  statusMsg = NAME + " completed successfully" + applyMarkupStatus;
+
+ // if user had implied match option chosen then figure out implied matches now
+ if (autoCreateImpliedMatches) {
+ processImpliedMatches(monitor);
+ }
+
+ // reset auto implied match option to user choice
+ applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH,
+ autoCreateImpliedMatches);
+
+ }
+
+ private void processImpliedMatches(TaskMonitor monitor) throws CancelledException {
+
+ List<VTAssociation> processedSrcDestPairs = new ArrayList<>();
+ List<VTMatchSet> matchSets = session.getMatchSets();
+
+ monitor.setMessage("Processing Implied Matches...");
+ monitor.initialize(matchSets.size());
+
+ for (VTMatchSet matchSet : matchSets) {
+ monitor.checkCancelled();
+
+ Collection<VTMatch> matches = matchSet.getMatches();
+ for (VTMatch match : matches) {
+ monitor.checkCancelled();
+
+ VTAssociation association = match.getAssociation();
+
+ // Implied matches currently only created for functions so skip matches that are
+ // data matches
+ if (association.getType() == VTAssociationType.DATA) {
+ continue;
+ }
+
+ // Implied matches should only be created for matches that user has accepted as 
+ // good matches
+ if (association.getStatus() != VTAssociationStatus.ACCEPTED) {
+ continue;
+ }
+ // only process the same match pair once so implied vote counts are not overinflated
+ if (processedSrcDestPairs.contains(association)) {
+ continue;
+ }
+
+ MatchInfo matchInfo = matchInfoFactory.getMatchInfo(match, addressCorrelator);
+
+ ImpliedMatchUtils.updateImpliedMatchForAcceptedAssocation(
+ matchInfo.getSourceFunction(),
+ matchInfo.getDestinationFunction(), session,
+ addressCorrelator, monitor);
+
+ processedSrcDestPairs.add(association);
+ }
+ monitor.incrementProgress();
+ }
+
  }
 
  private int getNumberOfDataMatches(TaskMonitor monitor) throws CancelledException {

Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java

@@ -15,27 +15,48 @@
  */
 package ghidra.feature.vt.api;
 
-import static ghidra.feature.vt.db.VTTestUtils.*;
-import static org.junit.Assert.*;
+import static ghidra.feature.vt.db.VTTestUtils.addr;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
-import org.junit.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 
 import ghidra.app.cmd.disassemble.DisassembleCommand;
 import ghidra.feature.vt.api.db.VTSessionDB;
-import ghidra.feature.vt.api.main.*;
+import ghidra.feature.vt.api.main.VTAssociationStatus;
+import ghidra.feature.vt.api.main.VTAssociationType;
+import ghidra.feature.vt.api.main.VTMatch;
+import ghidra.feature.vt.api.main.VTMatchInfo;
+import ghidra.feature.vt.api.main.VTMatchSet;
+import ghidra.feature.vt.api.main.VTProgramCorrelator;
+import ghidra.feature.vt.api.main.VTScore;
+import ghidra.feature.vt.api.main.VTSession;
 import ghidra.feature.vt.api.util.VTAssociationStatusException;
 import ghidra.feature.vt.db.VTTestUtils;
 import ghidra.feature.vt.gui.VTTestEnv;
 import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask;
 import ghidra.feature.vt.gui.plugin.VTController;
+import ghidra.feature.vt.gui.util.VTOptionDefines;
 import ghidra.framework.options.Options;
+import ghidra.framework.options.ToolOptions;
 import ghidra.program.database.ProgramDB;
 import ghidra.program.database.function.OverlappingFunctionException;
 import ghidra.program.model.address.Address;
 import ghidra.program.model.address.AddressSet;
-import ghidra.program.model.listing.*;
+import ghidra.program.model.listing.CodeUnit;
+import ghidra.program.model.listing.CodeUnitIterator;
+import ghidra.program.model.listing.Function;
+import ghidra.program.model.listing.FunctionManager;
+import ghidra.program.model.listing.Listing;
+import ghidra.program.model.listing.Program;
 import ghidra.program.model.mem.MemoryAccessException;
 import ghidra.program.model.symbol.SourceType;
 import ghidra.test.AbstractGhidraHeadedIntegrationTest;
@@ -778,6 +799,50 @@ public void testMarkup_MissingDestinationAddresses() throws Exception {
  }
  }
 
+ /*
+ * This tests auto version tracking with auto implied matches option set
+ */
+ @Test
+ public void testRunAutoVT_impliedMatches() throws Exception {
+
+ sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME);
+ destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME);
+
+ session = env.createSession(sourceProgram, destinationProgram);
+
+ env.showTool();
+ controller = env.getVTController();
+ ToolOptions options = controller.getOptions();
+ options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, true);
+
+ // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up
+ runAutoVTCommand(0.999999999, 10.0);
+
+ assertTrue(session.getImpliedMatchSet().getMatchCount() > 0);
+ }
+
+ /*
+ * This tests auto version tracking with auto implied matches option not set
+ */
+ @Test
+ public void testRunAutoVT_noImpliedMatches() throws Exception {
+
+ sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME);
+ destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME);
+
+ session = env.createSession(sourceProgram, destinationProgram);
+
+ env.showTool();
+ controller = env.getVTController();
+ ToolOptions options = controller.getOptions();
+ options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
+ // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up
+ runAutoVTCommand(0.999999999, 10.0);
+
+ assertTrue(session.getImpliedMatchSet().getMatchCount() == 0);
+ }
+
  private VTMatch createMatch(Address sourceAddress, Address destinationAddress,
  boolean setAccepted) throws VTAssociationStatusException {
  VTProgramCorrelator correlator =