diff --git a/app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java b/app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java index 3daa4a4ebc4..4c0839bc2f8 100644 --- a/app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java +++ b/app/server/appsmith-plugins/mysqlPlugin/src/main/java/com/external/plugins/MySqlPlugin.java @@ -1,8 +1,12 @@ package com.external.plugins; +import com.appsmith.external.constants.DataType; +import com.appsmith.external.dtos.ExecuteActionDTO; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginError; import com.appsmith.external.exceptions.pluginExceptions.AppsmithPluginException; import com.appsmith.external.exceptions.pluginExceptions.StaleConnectionException; +import com.appsmith.external.helpers.MustacheHelper; +import com.appsmith.external.helpers.SqlStringUtils; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.ActionExecutionRequest; import com.appsmith.external.models.ActionExecutionResult; @@ -11,6 +15,7 @@ import com.appsmith.external.models.DatasourceStructure; import com.appsmith.external.models.DatasourceTestResult; import com.appsmith.external.models.Endpoint; +import com.appsmith.external.models.Param; import com.appsmith.external.models.Property; import com.appsmith.external.plugins.BasePlugin; import com.appsmith.external.plugins.PluginExecutor; @@ -21,6 +26,7 @@ import io.r2dbc.spi.Result; import io.r2dbc.spi.Row; import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; import io.r2dbc.spi.ValidationDepth; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.ObjectUtils; @@ -45,9 +51,13 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; + public class MySqlPlugin extends BasePlugin { private static final String DATE_COLUMN_TYPE_NAME = "date"; @@ -113,115 +123,132 @@ public MySqlPlugin(PluginWrapper wrapper) { @Slf4j @Extension public static class MySqlPluginExecutor implements PluginExecutor { + private final Scheduler scheduler = Schedulers.elastic(); + private static final int PREPARED_STATEMENT_INDEX = 0; + /** - * 1. Parse the actual row objects returned by r2dbc driver for mysql statements. - * 2. Return the row as a map {column_name -> column_value}. + * Instead of using the default executeParametrized provided by pluginExecutor, this implementation affords an opportunity + * to use PreparedStatement (if configured) which requires the variable substitution, etc. to happen in a particular format + * supported by PreparedStatement. In case of PreparedStatement turned off, the action and datasource configurations are + * prepared (binding replacement) using PluginExecutor.variableSubstitution + * + * @param connection : This is the connection that is established to the data source. This connection is according + * to the parameters in Datasource Configuration + * @param executeActionDTO : This is the data structure sent by the client during execute. This contains the params + * which would be used for substitution + * @param datasourceConfiguration : These are the configurations which have been used to create a Datasource from a Plugin + * @param actionConfiguration : These are the configurations which have been used to create an Action from a Datasource. + * @return */ - private Map getRow(Row row, RowMetadata meta) { - Iterator iterator = (Iterator) meta.getColumnMetadatas().iterator(); - Map processedRow = new LinkedHashMap<>(); + @Override + public Mono executeParameterized(Connection connection, + ExecuteActionDTO executeActionDTO, + DatasourceConfiguration datasourceConfiguration, + ActionConfiguration actionConfiguration) { + + final Map requestData = new HashMap<>(); + + Boolean isPreparedStatement; + + final List properties = actionConfiguration.getPluginSpecifiedTemplates(); + if (properties == null || properties.get(PREPARED_STATEMENT_INDEX) == null) { + /** + * TODO : + * In case the prepared statement configuration is missing, default to true once PreparedStatement + * is no longer in beta. + */ + isPreparedStatement = false; + } else { + isPreparedStatement = Boolean.parseBoolean(properties.get(PREPARED_STATEMENT_INDEX).getValue()); + } + requestData.put("preparedStatement", TRUE.equals(isPreparedStatement) ? true : false); - while (iterator.hasNext()) { - ColumnMetadata metaData = iterator.next(); - String columnName = metaData.getName(); - String typeName = metaData.getJavaType().toString(); - Object columnValue = row.get(columnName); + String query = actionConfiguration.getBody(); + // Check for query parameter before performing the probably expensive fetch connection from the pool op. + if (query == null) { + ActionExecutionResult errorResult = new ActionExecutionResult(); + errorResult.setStatusCode(AppsmithPluginError.PLUGIN_ERROR.getAppErrorCode().toString()); + errorResult.setIsExecutionSuccess(false); + errorResult.setBody(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR.getMessage("Missing required " + + "parameter: Query.")); + ActionExecutionRequest actionExecutionRequest = new ActionExecutionRequest(); + actionExecutionRequest.setProperties(requestData); + errorResult.setRequest(actionExecutionRequest); + return Mono.just(errorResult); + } - if (java.time.LocalDate.class.toString().equalsIgnoreCase(typeName) - && columnValue != null) { - columnValue = DateTimeFormatter.ISO_DATE.format(row.get(columnName, - LocalDate.class)); - } else if ((java.time.LocalDateTime.class.toString().equalsIgnoreCase(typeName)) - && columnValue != null) { - columnValue = DateTimeFormatter.ISO_DATE_TIME.format( - LocalDateTime.of( - row.get(columnName, LocalDateTime.class).toLocalDate(), - row.get(columnName, LocalDateTime.class).toLocalTime() - ) - ) + "Z"; - } else if (java.time.LocalTime.class.toString().equalsIgnoreCase(typeName) - && columnValue != null) { - columnValue = DateTimeFormatter.ISO_TIME.format(row.get(columnName, - LocalTime.class)); - } else if (java.time.Year.class.toString().equalsIgnoreCase(typeName) - && columnValue != null) { - columnValue = row.get(columnName, LocalDate.class).getYear(); - } else { - columnValue = row.get(columnName); - } + actionConfiguration.setBody(query.trim()); - processedRow.put(columnName, columnValue); + // In case of non prepared statement, simply do binding replacement and execute + if (FALSE.equals(isPreparedStatement)) { + prepareConfigurationsForExecution(executeActionDTO, actionConfiguration, datasourceConfiguration); + return executeCommon(connection, actionConfiguration, FALSE, null, null, requestData); } - return processedRow; + //This has to be executed as Prepared Statement + // First extract all the bindings in order + List mustacheKeysInOrder = MustacheHelper.extractMustacheKeysInOrder(query); + // Replace all the bindings with a ? as expected in a prepared statement. + String updatedQuery = SqlStringUtils.replaceMustacheWithQuestionMark(query, mustacheKeysInOrder); + // Set the query with bindings extracted and replaced with '?' back in config + actionConfiguration.setBody(updatedQuery); + return executeCommon(connection, actionConfiguration, TRUE, mustacheKeysInOrder, executeActionDTO, requestData); } - /** - * 1. Check the type of sql query - i.e Select ... or Insert/Update/Drop - * 2. In case sql queries are chained together, then decide the type based on the last query. i.e In case of - * query "select * from test; updated test ..." the type of query will be based on the update statement. - * 3. This is used because the output returned to client is based on the type of the query. In case of a - * select query rows are returned, whereas, in case of any other query the number of updated rows is - * returned. - */ - private boolean getIsSelectOrShowQuery(String query) { - String[] queries = query.split(";"); - return (queries[queries.length - 1].trim().split(" ")[0].equalsIgnoreCase("select") - || queries[queries.length - 1].trim().split(" ")[0].equalsIgnoreCase("show")); - } - - @Override - public Mono execute(Connection connection, - DatasourceConfiguration datasourceConfiguration, - ActionConfiguration actionConfiguration) { - - String query = actionConfiguration.getBody().trim(); + public Mono executeCommon(Connection connection, + ActionConfiguration actionConfiguration, + Boolean preparedStatement, + List mustacheValuesInOrder, + ExecuteActionDTO executeActionDTO, + Map requestData) { - if (query == null) { - return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR, "Missing required " + - "parameter: Query.")); - } + String query = actionConfiguration.getBody(); boolean isSelectOrShowQuery = getIsSelectOrShowQuery(query); + final List> rowsList = new ArrayList<>(50); + Flux resultFlux = Mono.from(connection.validate(ValidationDepth.REMOTE)) .flatMapMany(isValid -> { if (isValid) { - return connection.createStatement(query).execute(); - } else { - return Flux.error(new StaleConnectionException()); + return createAndExecuteQueryFromConnection(query, + connection, + preparedStatement, + mustacheValuesInOrder, + executeActionDTO, + requestData); } + return Flux.error(new StaleConnectionException()); }); - Mono>> resultMono = null; + Mono>> resultMono; if (isSelectOrShowQuery) { resultMono = resultFlux - .flatMap(result -> { - return result.map((row, meta) -> { - rowsList.add(getRow(row, meta)); - return result; - }); - }) + .flatMap(result -> + result.map((row, meta) -> { + rowsList.add(getRow(row, meta)); + return result; + } + ) + ) .collectList() - .flatMap(execResult -> { - return Mono.just(rowsList); - }); + .thenReturn(rowsList); } else { resultMono = resultFlux .flatMap(result -> result.getRowsUpdated()) .collectList() .flatMap(list -> Mono.just(list.get(list.size() - 1))) - .flatMap(rowsUpdated -> { + .map(rowsUpdated -> { rowsList.add( Map.of( "affectedRows", ObjectUtils.defaultIfNull(rowsUpdated, 0) ) ); - return Mono.just(rowsList); + return rowsList; }); } @@ -234,7 +261,7 @@ public Mono execute(Connection connection, "execution result"); return result; }) - .onErrorResume(AppsmithPluginException.class, error -> { + .onErrorResume(AppsmithPluginException.class, error -> { ActionExecutionResult result = new ActionExecutionResult(); result.setIsExecutionSuccess(false); result.setStatusCode(error.getAppErrorCode().toString()); @@ -245,6 +272,7 @@ public Mono execute(Connection connection, .map(actionExecutionResult -> { ActionExecutionRequest request = new ActionExecutionRequest(); request.setQuery(query); + request.setProperties(requestData); ActionExecutionResult result = actionExecutionResult; result.setRequest(request); return result; @@ -253,6 +281,111 @@ public Mono execute(Connection connection, } + private Flux createAndExecuteQueryFromConnection(String query, + Connection connection, + Boolean preparedStatement, + List mustacheValuesInOrder, + ExecuteActionDTO executeActionDTO, + Map requestData) { + + Statement connectionStatement = connection.createStatement(query); + if (FALSE.equals(preparedStatement) || mustacheValuesInOrder == null || mustacheValuesInOrder.isEmpty()) { + return Flux.from(connectionStatement.execute()); + } + + System.out.println("Query : " + query); + + List params = executeActionDTO.getParams(); + List parameters = new ArrayList<>(); + + for (int i = 0; i < mustacheValuesInOrder.size(); i++) { + String key = mustacheValuesInOrder.get(i); + Optional matchingParam = params + .stream() + .filter(param -> param.getKey().trim().equals(key)) + .findFirst(); + if (matchingParam.isPresent()) { + String value = matchingParam.get().getValue(); + parameters.add(value); + DataType valueType = SqlStringUtils.stringToKnownDataTypeConverter(value); + if (DataType.NULL.equals(valueType)) { + try { + connectionStatement.bindNull(i, Object.class); + } catch (UnsupportedOperationException e) { + // Do nothing. Move on + } + } else { + connectionStatement.bind(i, value); + } + } + } + requestData.put("parameters", parameters); + + return Flux.from(connectionStatement.execute()); + + } + + /** + * 1. Parse the actual row objects returned by r2dbc driver for mysql statements. + * 2. Return the row as a map {column_name -> column_value}. + */ + private Map getRow(Row row, RowMetadata meta) { + Iterator iterator = (Iterator) meta.getColumnMetadatas().iterator(); + Map processedRow = new LinkedHashMap<>(); + + while (iterator.hasNext()) { + ColumnMetadata metaData = iterator.next(); + String columnName = metaData.getName(); + String typeName = metaData.getJavaType().toString(); + Object columnValue = row.get(columnName); + + if (java.time.LocalDate.class.toString().equalsIgnoreCase(typeName) && columnValue != null) { + columnValue = DateTimeFormatter.ISO_DATE.format(row.get(columnName, LocalDate.class)); + } else if ((java.time.LocalDateTime.class.toString().equalsIgnoreCase(typeName)) && columnValue != null) { + columnValue = DateTimeFormatter.ISO_DATE_TIME.format( + LocalDateTime.of( + row.get(columnName, LocalDateTime.class).toLocalDate(), + row.get(columnName, LocalDateTime.class).toLocalTime() + ) + ) + "Z"; + } else if (java.time.LocalTime.class.toString().equalsIgnoreCase(typeName) && columnValue != null) { + columnValue = DateTimeFormatter.ISO_TIME.format(row.get(columnName, + LocalTime.class)); + } else if (java.time.Year.class.toString().equalsIgnoreCase(typeName) && columnValue != null) { + columnValue = row.get(columnName, LocalDate.class).getYear(); + } else { + columnValue = row.get(columnName); + } + + processedRow.put(columnName, columnValue); + } + + return processedRow; + } + + /** + * 1. Check the type of sql query - i.e Select ... or Insert/Update/Drop + * 2. In case sql queries are chained together, then decide the type based on the last query. i.e In case of + * query "select * from test; updated test ..." the type of query will be based on the update statement. + * 3. This is used because the output returned to client is based on the type of the query. In case of a + * select query rows are returned, whereas, in case of any other query the number of updated rows is + * returned. + */ + private boolean getIsSelectOrShowQuery(String query) { + String[] queries = query.split(";"); + + String lastQuery = queries[queries.length - 1].trim(); + + return (lastQuery.trim().split(" ")[0].equalsIgnoreCase("select") + || lastQuery.trim().split(" ")[0].equalsIgnoreCase("show")); + } + + @Override + public Mono execute(Connection connection, DatasourceConfiguration datasourceConfiguration, ActionConfiguration actionConfiguration) { + // Unused function + return Mono.error(new AppsmithPluginException(AppsmithPluginError.PLUGIN_ERROR, "Unsupported Operation")); + } + @Override public Mono datasourceCreate(DatasourceConfiguration datasourceConfiguration) { DBAuth authentication = (DBAuth) datasourceConfiguration.getAuthentication(); @@ -294,12 +427,10 @@ public Mono datasourceCreate(DatasourceConfiguration datasourceConfi ob = ob.option(ConnectionFactoryOptions.PASSWORD, authentication.getPassword()); return (Mono) Mono.from(ConnectionFactories.get(ob.build()).create()) - .onErrorResume(exception -> { - return Mono.error(new AppsmithPluginException( - AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, - exception - )); - }) + .onErrorResume(exception -> Mono.error(new AppsmithPluginException( + AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, + exception + ))) .subscribeOn(scheduler); } diff --git a/app/server/appsmith-plugins/mysqlPlugin/src/main/resources/setting.json b/app/server/appsmith-plugins/mysqlPlugin/src/main/resources/setting.json new file mode 100644 index 00000000000..4c433b6bc88 --- /dev/null +++ b/app/server/appsmith-plugins/mysqlPlugin/src/main/resources/setting.json @@ -0,0 +1,36 @@ +{ + "setting": [ + { + "sectionName": "", + "id": 1, + "children": [ + { + "label": "Run query on page load", + "configProperty": "executeOnLoad", + "controlType": "SWITCH", + "info": "Will refresh data each time the page is loaded" + }, + { + "label": "Request confirmation before running query", + "configProperty": "confirmBeforeExecute", + "controlType": "SWITCH", + "info": "Ask confirmation from the user each time before refreshing data" + }, + { + "label": "[Beta] Use Prepared Statement", + "info": "Turning on Prepared Statement makes the query parametrized. This in turn makes it resilient against SQL injections", + "configProperty": "actionConfiguration.pluginSpecifiedTemplates[0].value", + "controlType": "SWITCH", + "initialValue": false + }, + { + "label": "Query timeout (in milliseconds)", + "info": "Maximum time after which the query will return", + "configProperty": "actionConfiguration.timeoutInMillisecond", + "controlType": "INPUT_TEXT", + "dataType": "NUMBER" + } + ] + } + ] +} diff --git a/app/server/appsmith-plugins/mysqlPlugin/src/test/java/com/external/plugins/MySqlPluginTest.java b/app/server/appsmith-plugins/mysqlPlugin/src/test/java/com/external/plugins/MySqlPluginTest.java index db176dcf8b2..86e0a6d1352 100644 --- a/app/server/appsmith-plugins/mysqlPlugin/src/test/java/com/external/plugins/MySqlPluginTest.java +++ b/app/server/appsmith-plugins/mysqlPlugin/src/test/java/com/external/plugins/MySqlPluginTest.java @@ -1,5 +1,7 @@ package com.external.plugins; +import com.appsmith.external.dtos.ExecuteActionDTO; +import com.appsmith.external.exceptions.pluginExceptions.StaleConnectionException; import com.appsmith.external.models.ActionConfiguration; import com.appsmith.external.models.ActionExecutionResult; import com.appsmith.external.models.DBAuth; @@ -7,14 +9,12 @@ import com.appsmith.external.models.DatasourceStructure; import com.appsmith.external.models.Endpoint; import com.appsmith.external.models.Property; -import com.appsmith.external.exceptions.pluginExceptions.StaleConnectionException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; - -import io.r2dbc.spi.ConnectionFactoryOptions; import io.r2dbc.spi.Connection; import io.r2dbc.spi.ConnectionFactories; +import io.r2dbc.spi.ConnectionFactoryOptions; import lombok.extern.log4j.Log4j; import org.junit.Assert; import org.junit.BeforeClass; @@ -30,11 +30,11 @@ import java.util.List; import java.util.Set; -import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertArrayEquals; @Log4j public class MySqlPluginTest { @@ -116,6 +116,14 @@ public static void setUp() { " '15:45:30'," + " '2019-11-30 23:59:59', '2019-11-30 23:59:59'" + ")" + ) + .add("INSERT INTO users VALUES (" + + "3, 'MiniJackJill', 'jaji', 'jaji@exemplars.com', NULL, '2021-01-31'," + + " '15:45:30', '04:05:06 PST'," + + " TIMESTAMP '2021-01-31 23:59:59', TIMESTAMP WITH TIME ZONE '2021-01-31 23:59:59 CET'," + + " '0 years'," + + " '{1, 2, 3}', '{\"a\", \"b\"}'" + + ")" ); }) .flatMap(batch -> Mono.from(batch.execute())) @@ -205,7 +213,7 @@ public void testExecute() { ActionConfiguration actionConfiguration = new ActionConfiguration(); actionConfiguration.setBody("show databases"); - Mono executeMono = dsConnectionMono.flatMap(conn -> pluginExecutor.execute(conn, dsConfig, actionConfiguration)); + Mono executeMono = dsConnectionMono.flatMap(conn -> pluginExecutor.executeParameterized(conn, new ExecuteActionDTO(), dsConfig, actionConfiguration)); StepVerifier.create(executeMono) .assertNext(obj -> { ActionExecutionResult result = (ActionExecutionResult) obj; @@ -223,7 +231,7 @@ public void testStaleConnectionCheck() { Connection connection = pluginExecutor.datasourceCreate(dsConfig).block(); Flux resultFlux = Mono.from(connection.close()) - .thenMany(pluginExecutor.execute(connection, dsConfig, actionConfiguration)); + .thenMany(pluginExecutor.executeParameterized(connection, new ExecuteActionDTO(), dsConfig, actionConfiguration)); StepVerifier.create(resultFlux) .expectErrorMatches(throwable -> throwable instanceof StaleConnectionException) @@ -287,7 +295,7 @@ public void testAliasColumnNames() { actionConfiguration.setBody("SELECT id as user_id FROM users WHERE id = 1"); Mono executeMono = dsConnectionMono - .flatMap(conn -> pluginExecutor.execute(conn, dsConfig, actionConfiguration)); + .flatMap(conn -> pluginExecutor.executeParameterized(conn, new ExecuteActionDTO(), dsConfig, actionConfiguration)); StepVerifier.create(executeMono) .assertNext(result -> { @@ -316,7 +324,7 @@ public void testExecuteDataTypes() { actionConfiguration.setBody("SELECT * FROM users WHERE id = 1"); Mono executeMono = dsConnectionMono - .flatMap(conn -> pluginExecutor.execute(conn, dsConfig, actionConfiguration)); + .flatMap(conn -> pluginExecutor.executeParameterized(conn, new ExecuteActionDTO(), dsConfig, actionConfiguration)); StepVerifier.create(executeMono) .assertNext(result -> { @@ -433,7 +441,7 @@ private void testExecute(String query) { Mono dsConnectionMono = pluginExecutor.datasourceCreate(dsConfig); ActionConfiguration actionConfiguration = new ActionConfiguration(); actionConfiguration.setBody(query); - Mono executeMono = dsConnectionMono.flatMap(conn -> pluginExecutor.execute(conn, dsConfig, actionConfiguration)); + Mono executeMono = dsConnectionMono.flatMap(conn -> pluginExecutor.executeParameterized(conn, new ExecuteActionDTO(), dsConfig, actionConfiguration)); StepVerifier.create(executeMono) .assertNext(obj -> { ActionExecutionResult result = (ActionExecutionResult) obj;