From 7f5eb65e55fb9253af475e0c528df67b864d788f Mon Sep 17 00:00:00 2001 From: ghidra007 Date: Tue, 19 Sep 2023 16:39:44 +0000 Subject: [PATCH] GP-3765 Updated AutoVersionTracking script and task to process implied matches when running from the script if user wants them. --- .../AutoVersionTrackingScript.java | 26 +++- .../gui/actions/AutoVersionTrackingTask.java | 115 +++++++++++++++++- .../vt/api/VTAutoVersionTrackingTest.java | 77 +++++++++++- 3 files changed, 206 insertions(+), 12 deletions(-) diff --git a/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java b/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java index 142f80bfc17..fd7d02584d1 100644 --- a/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java +++ b/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java @@ -23,6 +23,7 @@ import ghidra.feature.vt.api.util.VTOptions; import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask; import ghidra.feature.vt.gui.plugin.VTController; +import ghidra.feature.vt.gui.util.VTOptionDefines; import ghidra.framework.model.DomainFolder; import ghidra.framework.options.ToolOptions; import ghidra.framework.plugintool.PluginTool; @@ -68,6 +69,9 @@ public void run() throws Exception { return; } + boolean autoCreateImpliedMatches = askYesNo("Implied Matches?", + "Would you like the script to figure out implied matches from any matches it creates?"); + // Need to end the script transaction or it interferes with vt things that need locks end(true); @@ -76,12 +80,32 @@ public void run() throws Exception { folder.createFile(name, session, monitor); + ToolOptions options = getOptions(); + + boolean originalImpliedMatchSetting = + options.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + + options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, autoCreateImpliedMatches); + AutoVersionTrackingTask autoVtTask = - new AutoVersionTrackingTask(session, getOptions(), 1.0, 10.0); + new AutoVersionTrackingTask(session, options, 0.95, 10.0); + TaskLauncher.launch(autoVtTask); + + + // if not running headless user can decide whether to save or not + // if running headless - must save here or nothing that was done in this script will be + // accessible later. + if (isRunningHeadless()) { + session.save(); + } + options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, originalImpliedMatchSetting); + + println(autoVtTask.getStatusMsg()); } + private ToolOptions getOptions() { ToolOptions vtOptions = new VTOptions("Dummy"); PluginTool tool = state.getTool(); diff --git a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java index 2426da1573d..41b5ac53411 100644 --- a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java +++ b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java @@ -15,28 +15,61 @@ */ package ghidra.feature.vt.gui.actions; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; import javax.swing.SwingConstants; -import ghidra.feature.vt.api.correlator.program.*; -import ghidra.feature.vt.api.main.*; +import ghidra.feature.vt.api.correlator.program.CombinedFunctionAndDataReferenceProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.DataReferenceProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.DuplicateFunctionMatchProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.ExactDataMatchProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.ExactMatchBytesProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.ExactMatchInstructionsProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.ExactMatchMnemonicsProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.FunctionReferenceProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.SymbolNameProgramCorrelatorFactory; +import ghidra.feature.vt.api.correlator.program.VTAbstractReferenceProgramCorrelatorFactory; +import ghidra.feature.vt.api.main.VTAssociation; +import ghidra.feature.vt.api.main.VTAssociationManager; +import ghidra.feature.vt.api.main.VTAssociationStatus; +import ghidra.feature.vt.api.main.VTAssociationType; +import ghidra.feature.vt.api.main.VTMarkupItem; +import ghidra.feature.vt.api.main.VTMatch; +import ghidra.feature.vt.api.main.VTMatchSet; +import ghidra.feature.vt.api.main.VTProgramCorrelator; +import ghidra.feature.vt.api.main.VTProgramCorrelatorFactory; +import ghidra.feature.vt.api.main.VTSession; import ghidra.feature.vt.api.util.VTAssociationStatusException; import ghidra.feature.vt.api.util.VTOptions; import ghidra.feature.vt.gui.plugin.AddressCorrelatorManager; import ghidra.feature.vt.gui.task.ApplyMarkupItemTask; +import ghidra.feature.vt.gui.util.ImpliedMatchUtils; import ghidra.feature.vt.gui.util.MatchInfo; import ghidra.feature.vt.gui.util.MatchInfoFactory; +import ghidra.feature.vt.gui.util.VTOptionDefines; import ghidra.framework.options.ToolOptions; import ghidra.program.model.address.Address; import ghidra.program.model.address.AddressSetView; import ghidra.program.model.lang.OperandType; -import ghidra.program.model.listing.*; +import ghidra.program.model.listing.CodeUnit; +import ghidra.program.model.listing.CodeUnitIterator; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionManager; +import ghidra.program.model.listing.Instruction; +import ghidra.program.model.listing.InstructionIterator; +import ghidra.program.model.listing.Program; import ghidra.program.util.ListingDiff; import ghidra.util.Msg; import ghidra.util.exception.AssertException; import ghidra.util.exception.CancelledException; -import ghidra.util.task.*; +import ghidra.util.task.Task; +import ghidra.util.task.TaskMonitor; +import ghidra.util.task.WrappingTaskMonitor; import util.CollectionUtils; /** @@ -142,9 +175,24 @@ private void doRun(TaskMonitor realMonitor) throws CancelledException { int count = 0; monitor.doInitialize(NUM_CORRELATORS); + // save user option and use to determine whether to handle implied matches at all later + boolean autoCreateImpliedMatches = + applyOptions.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + + // Turn off auto implied matches and handle later if user had that option set + // This is because when run from the VT GUI action implied matches are created automatically + // by the VT controller when the option is set but they are not created when called from a + // script since there is no VT controller in that case. If allowed to happen in + // GUI then they will happen twice when called later in this task and the implied match + // votes will be wrong. This Task doesn't know if called from GUI or script so this is + // klunky but will make sure they are only processed once and will make sure the user option + // is put back the way the user had it. + applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + // Use default options for all of the "exact" correlators; passed in options for the others VTOptions options; + // Run the correlators in the following order: // Do this one first because we don't want it to find ones that get markup applied by later // correlators @@ -266,6 +314,63 @@ private void doRun(TaskMonitor realMonitor) throws CancelledException { " with some apply markup errors. See the log or the markup table for more details"; } statusMsg = NAME + " completed successfully" + applyMarkupStatus; + + // if user had implied match option chosen then figure out implied matches now + if (autoCreateImpliedMatches) { + processImpliedMatches(monitor); + } + + // reset auto implied match option to user choice + applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, + autoCreateImpliedMatches); + + } + + private void processImpliedMatches(TaskMonitor monitor) throws CancelledException { + + List processedSrcDestPairs = new ArrayList<>(); + List matchSets = session.getMatchSets(); + + monitor.setMessage("Processing Implied Matches..."); + monitor.initialize(matchSets.size()); + + for (VTMatchSet matchSet : matchSets) { + monitor.checkCancelled(); + + Collection matches = matchSet.getMatches(); + for (VTMatch match : matches) { + monitor.checkCancelled(); + + VTAssociation association = match.getAssociation(); + + // Implied matches currently only created for functions so skip matches that are + // data matches + if (association.getType() == VTAssociationType.DATA) { + continue; + } + + // Implied matches should only be created for matches that user has accepted as + // good matches + if (association.getStatus() != VTAssociationStatus.ACCEPTED) { + continue; + } + // only process the same match pair once so implied vote counts are not overinflated + if (processedSrcDestPairs.contains(association)) { + continue; + } + + MatchInfo matchInfo = matchInfoFactory.getMatchInfo(match, addressCorrelator); + + ImpliedMatchUtils.updateImpliedMatchForAcceptedAssocation( + matchInfo.getSourceFunction(), + matchInfo.getDestinationFunction(), session, + addressCorrelator, monitor); + + processedSrcDestPairs.add(association); + } + monitor.incrementProgress(); + } + } private int getNumberOfDataMatches(TaskMonitor monitor) throws CancelledException { diff --git a/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java b/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java index cad69eb18a6..e2bfb9de585 100644 --- a/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java +++ b/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java @@ -15,27 +15,48 @@ */ package ghidra.feature.vt.api; -import static ghidra.feature.vt.db.VTTestUtils.*; -import static org.junit.Assert.*; +import static ghidra.feature.vt.db.VTTestUtils.addr; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import java.util.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; -import org.junit.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import ghidra.app.cmd.disassemble.DisassembleCommand; import ghidra.feature.vt.api.db.VTSessionDB; -import ghidra.feature.vt.api.main.*; +import ghidra.feature.vt.api.main.VTAssociationStatus; +import ghidra.feature.vt.api.main.VTAssociationType; +import ghidra.feature.vt.api.main.VTMatch; +import ghidra.feature.vt.api.main.VTMatchInfo; +import ghidra.feature.vt.api.main.VTMatchSet; +import ghidra.feature.vt.api.main.VTProgramCorrelator; +import ghidra.feature.vt.api.main.VTScore; +import ghidra.feature.vt.api.main.VTSession; import ghidra.feature.vt.api.util.VTAssociationStatusException; import ghidra.feature.vt.db.VTTestUtils; import ghidra.feature.vt.gui.VTTestEnv; import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask; import ghidra.feature.vt.gui.plugin.VTController; +import ghidra.feature.vt.gui.util.VTOptionDefines; import ghidra.framework.options.Options; +import ghidra.framework.options.ToolOptions; import ghidra.program.database.ProgramDB; import ghidra.program.database.function.OverlappingFunctionException; import ghidra.program.model.address.Address; import ghidra.program.model.address.AddressSet; -import ghidra.program.model.listing.*; +import ghidra.program.model.listing.CodeUnit; +import ghidra.program.model.listing.CodeUnitIterator; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionManager; +import ghidra.program.model.listing.Listing; +import ghidra.program.model.listing.Program; import ghidra.program.model.mem.MemoryAccessException; import ghidra.program.model.symbol.SourceType; import ghidra.test.AbstractGhidraHeadedIntegrationTest; @@ -778,6 +799,50 @@ public void testMarkup_MissingDestinationAddresses() throws Exception { } } + /* + * This tests auto version tracking with auto implied matches option set + */ + @Test + public void testRunAutoVT_impliedMatches() throws Exception { + + sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME); + destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME); + + session = env.createSession(sourceProgram, destinationProgram); + + env.showTool(); + controller = env.getVTController(); + ToolOptions options = controller.getOptions(); + options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, true); + + // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up + runAutoVTCommand(0.999999999, 10.0); + + assertTrue(session.getImpliedMatchSet().getMatchCount() > 0); + } + + /* + * This tests auto version tracking with auto implied matches option not set + */ + @Test + public void testRunAutoVT_noImpliedMatches() throws Exception { + + sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME); + destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME); + + session = env.createSession(sourceProgram, destinationProgram); + + env.showTool(); + controller = env.getVTController(); + ToolOptions options = controller.getOptions(); + options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + + // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up + runAutoVTCommand(0.999999999, 10.0); + + assertTrue(session.getImpliedMatchSet().getMatchCount() == 0); + } + private VTMatch createMatch(Address sourceAddress, Address destinationAddress, boolean setAccepted) throws VTAssociationStatusException { VTProgramCorrelator correlator =