From 221c3f7c5926b3c4251d67f09563f49763923557 Mon Sep 17 00:00:00 2001 From: Stefan Borchert Date: Sat, 1 Jul 2017 16:30:06 +0200 Subject: [PATCH] Provide basic set of migration including examples. Signed-off-by: Stefan Borchert --- README.md | 60 ++++- .../Derivative/MigrationGroupMenuLink.php | 65 ++++++ src/Plugin/migrate/id_map/FastSql.php | 45 ++++ src/Plugin/migrate/process/StringReplace.php | 40 ++++ src/Plugin/migrate/source/SqlBase.php | 208 ++++++++++++++++++ up_migrate.drush.inc | 197 +++++++++++++++++ up_migrate.info.yml | 9 + up_migrate.links.menu.yml | 3 + up_migrate.module | 22 ++ ...grate_plus.migration.upm_example__user.yml | 47 ++++ ...grate_plus.migration_group.up_examples.yml | 12 + .../Plugin/migrate/source/UserAccounts.php | 73 ++++++ .../up_migrate_examples.info.yml | 7 + 13 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 src/Plugin/Derivative/MigrationGroupMenuLink.php create mode 100644 src/Plugin/migrate/id_map/FastSql.php create mode 100644 src/Plugin/migrate/process/StringReplace.php create mode 100644 src/Plugin/migrate/source/SqlBase.php create mode 100644 up_migrate.drush.inc create mode 100644 up_migrate.info.yml create mode 100644 up_migrate.links.menu.yml create mode 100644 up_migrate.module create mode 100644 up_migrate_examples/config/install/migrate_plus.migration.upm_example__user.yml create mode 100644 up_migrate_examples/config/install/migrate_plus.migration_group.up_examples.yml create mode 100644 up_migrate_examples/src/Plugin/migrate/source/UserAccounts.php create mode 100644 up_migrate_examples/up_migrate_examples.info.yml diff --git a/README.md b/README.md index 2698c2a..36e8eb1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Migration Starter +# undpaul Migrate This project aims to give you a jump-start on building migrations. @@ -7,6 +7,64 @@ This project aims to give you a jump-start on building migrations. Clone the project into `modules/custom` of your Drupal installation and run `composer install` within the modules directory. +### Adding additional databases as migration source + +`drush upm-da {key} {database name} --username={name} --password={password}` + ## Usage +_up_migrate_ comes with some base classes usefull for building migrations. + +### ID maps + +#### FastSql + +Since the mapping tables create by _Migrate_ do not have proper unique indexes, +the ID map "fastsql" alters the definitions of the map tables and add proper +indexes. This increases migration performance significant. +_up_migrate_ sets the ID map for **all** migrations if they do not specify the +property `idMap` for themself. + +### Source plugins + +_up_migrate_ comes with a set of pre-defined source plugins. + +#### SqlBase + +The abstract class `\Drupal\up_migrate\Plugin\migrate\source\SqlBase` is meant +to be the parent class for all custom migration sources. It allows configuring +sources from directly within a migration yml file. + +Example for definition in _migrate_plus.migration.{id}.yml_: + + source: + table: + name: users + alias: u + ids: + uid: integer + +This sets the base table of the migration source to a table named "users". The +table can be referenced by using the alias "u" and have one unique key named +"uid". + +Example for definition in custom source plugin (must extend `SqlBase`) + + /** + * @MigrateSource( + * id = "upm_examples__user", + * table = { + * "name": "users", + * "alias": "u", + * "ids": { + * "uid": "integer" + * } + * } + * ) + */ + +For more complex ID definitions simply override the function `getIds()`. + +#### UserMerge + @todo diff --git a/src/Plugin/Derivative/MigrationGroupMenuLink.php b/src/Plugin/Derivative/MigrationGroupMenuLink.php new file mode 100644 index 0000000..46d60e0 --- /dev/null +++ b/src/Plugin/Derivative/MigrationGroupMenuLink.php @@ -0,0 +1,65 @@ +configStorage = $config_storage; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container, $base_plugin_id) { + return new static( + $container->get('entity.manager')->getStorage('migration_group') + ); + } + + /** + * {@inheritdoc} + */ + public function getDerivativeDefinitions($base_plugin_definition) { + $links = []; + + $groups = $this->configStorage->loadMultiple(); + /* @var $group \Drupal\migrate_plus\Entity\MigrationGroupInterface */ + foreach ($groups as $group) { + $menu_link_id = "migration_group.{$group->id()}"; + $links[$menu_link_id] = [ + 'id' => $menu_link_id, + 'title' => $group->label(), + 'description' => $group->get('description'), + 'route_name' => 'entity.migration.list', + 'route_parameters' => [ + 'migration_group' => $group->id(), + ], + ] + $base_plugin_definition; + } + + return $links; + } + +} diff --git a/src/Plugin/migrate/id_map/FastSql.php b/src/Plugin/migrate/id_map/FastSql.php new file mode 100644 index 0000000..09fbb6a --- /dev/null +++ b/src/Plugin/migrate/id_map/FastSql.php @@ -0,0 +1,45 @@ +getDatabase()->schema()->indexExists($this->mapTableName, 'source')) { + return; + } + // Add unique index over all source columns. + $count = 1; + $string_fields = 0; + $unique_fields = []; + foreach ($this->migration->getSourcePlugin()->getIds() as $id_definition) { + $unique_fields[] = 'sourceid' . $count++; + if (isset($id_definition['type']) && ('string' === $id_definition['type'])) { + $string_fields++; + } + } + + if ($string_fields > 2) { + // Do not create unique index over more than 2 string fields. Otherwise + // there is a great chance the key is too long (>767 bytes). + return; + } + + $this->getDatabase()->schema()->addUniqueKey($this->mapTableName, 'source', $unique_fields); + } + +} diff --git a/src/Plugin/migrate/process/StringReplace.php b/src/Plugin/migrate/process/StringReplace.php new file mode 100644 index 0000000..00ad8ba --- /dev/null +++ b/src/Plugin/migrate/process/StringReplace.php @@ -0,0 +1,40 @@ + + * process: + * file_destination: + * plugin: up_string_replace + * # Remove unused parts in path. + * 'field/image': '' + * # Move all files into sub-directory. + * 'public://': 'public://imported' + * + * + * @MigrateProcessPlugin( + * id = "up_string_replace" + * ) + */ +class StringReplace extends ProcessPluginBase { + + /** + * {@inheritdoc} + */ + public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) { + $replacements = $this->configuration['replacements'] ?: []; + if (!is_array($replacements)) { + $replacements = [$replacements]; + } + return strtr($value, $replacements); + } + +} diff --git a/src/Plugin/migrate/source/SqlBase.php b/src/Plugin/migrate/source/SqlBase.php new file mode 100644 index 0000000..c696989 --- /dev/null +++ b/src/Plugin/migrate/source/SqlBase.php @@ -0,0 +1,208 @@ +state = $state; + + // Initialize databases. + $this->initDatabases(); + + if (empty($configuration['key']) && empty($configuration['target']) && empty($configuration['database_state_key'])) { + // Use first database as default. + $configuration['key'] = reset(array_keys($this->state->get('up_migrate.databases', []))); + } + + // Now we can safely call the parent constructor. + parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $state); + + if (($this->table = $this->getConfig('table')) === NULL || empty($this->table['name'])) { + // No table defined as source for user data. + throw new MigrateException('No source table defined. Set this in the migration yml file or in the source plugin.'); + } + } + + /** + * {@inheritdoc} + */ + public function query() { + $query = $this->select($this->getTableName(), $this->getTableAlias()) + ->fields($this->getTableAlias()); + + // Extending classes should alter the query. + $this->alterQuery($query); + + return $query; + } + + /** + * {@inheritdoc} + */ + public function fields() { + // Define base fields for user accounts. + $fields = []; + + // Add required fields from extending classes. + $this->alterFields($fields); + + return $fields; + } + + /** + * {@inheritdoc} + */ + public function getIds() { + if (empty($this->table['ids'])) { + throw new MigrateException('No source table ids defined. Set this in the migration yml file or in the source plugin or override the function getIds() in the source plugin.'); + } + $ids = []; + foreach ($this->table['ids'] as $key => $type) { + $ids[$key] = [ + 'type' => $type, + 'alias' => $this->getTableAlias(), + ]; + } + return $ids; + } + + /** + * Alter the base query. + * + * @param \Drupal\Core\Database\Query\SelectInterface $query + * The query to alter. + */ + protected function alterQuery(SelectInterface $query) { + // Give extending classes the possibility to alter the source query. + } + + /** + * Alter the list of available fields. + * + * @param array $fields + * List of fields available for migration. + */ + protected function alterFields(array $fields = []) { + // Give extending classes the possibility to alter or add fields. + } + + /** + * Get configuration values for the current migration. + * + * Note: Configuration made in the source plugin definition may be overriden + * by the migration itself. + * + * @param string|null $name + * The configuration value or NULL if there is no configuration with that + * name. + * @param mixed $default + * Default value. + * + * @return mixed + * The configuration value. + */ + public function getConfig($name, $default = NULL) { + $plugin_definition = $this->pluginDefinition; + $source_configuration = $this->migration->getSourceConfiguration(); + + $value = $default; + if (isset($plugin_definition[$name])) { + // Empty value is allowed. + $value = $plugin_definition[$name]; + } + if (isset($source_configuration[$name])) { + // Empty value is allowed. + $value = is_array($value) ? array_merge($value, $source_configuration[$name]) : $source_configuration[$name]; + } + + return $value; + } + + /** + * Initialize databases used as migration sources. + */ + protected function initDatabases() { + if (($databases = $this->state->get('up_migrate.databases')) === NULL) { + throw new MigrateException('No source databases defined. Run `drush upm-database-add` to add at a database.'); + } + foreach ($databases as $key => $info) { + // No need to check if the connection has been added before, since + // addConnectionInfo() does this. + Database::addConnectionInfo($key, 'default', $this->databaseConfigAddDefaults($info)); + } + } + + /** + * Helper function to expand database configuration with default values. + * + * @param array $config + * The given database configuration. + * + * @return array + * Full database configuration array. + */ + protected function databaseConfigAddDefaults(array $config) { + // Add default configuration. + return array_filter($config) + [ + 'host' => 'localhost', + 'port' => '3306', + 'username' => '', + 'password' => '', + 'driver' => 'mysql', + 'namespace' => 'Drupal\Core\Database\Driver\mysql', + 'init_commands' => [ + // Use custom sql_mode so we disable "ONLY_FULL_GROUP_BY". + 'sql_mode' => "SET sql_mode = 'ANSI,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER'", + ], + ]; + } + + /** + * Get the base table name for the migration query. + * + * @return string + * Name of base table. + */ + protected function getTableName() { + return $this->table['name']; + } + + /** + * Get the base table alias for the migration query. + * + * @return string + * Alias of base table. + */ + protected function getTableAlias() { + return empty($this->table['alias']) ? $this->table['name'] : $this->table['alias']; + } + +} diff --git a/up_migrate.drush.inc b/up_migrate.drush.inc new file mode 100644 index 0000000..7706c7d --- /dev/null +++ b/up_migrate.drush.inc @@ -0,0 +1,197 @@ + 'Refresh default configuration of an extension without the need to reinstall.', + 'core' => ['8+'], + 'aliases' => ['cf5'], + 'arguments' => [ + 'module' => 'The name of the extension needing a configuration refresh.', + 'type' => "The type of the extension (one of 'module', 'theme', 'profile'). Defaults to 'module'.", + ], + 'examples' => [ + 'drush config-refresh mymodule' => "Refresh default configuration of a module named 'mymodule'.", + 'drush config-refresh myprofile profile' => "Refresh default configuration of a profile named 'myprofile'.", + ], + ]; + + // Add migration source database. + $items['upm-database-add'] = [ + 'description' => 'Add a database as migration source', + 'arguments' => [ + 'key' => 'Unique database key. This is used in migrations to switch to a different database', + 'database' => 'Name of database to add', + ], + 'options' => [ + 'username' => 'Username for database connection', + 'password' => 'Password for database connection', + 'host' => 'Host of database server (defaults to localhost)', + 'port' => 'Port of database server (defaults to 3306)', + ], + 'examples' => [ + 'upm-database-add phpbb forum_old' => 'Add database "forum_old" with key "phpbb"', + ], + 'drupal dependencies' => ['up_migrate'], + 'aliases' => ['upm-da'], + ]; + + return $items; +} + +/** + * Implements hook_drush_command_alter(). + * + * If any other module already defined "config-refresh" we override it here. + */ +function up_migrate_drush_command_alter(&$command) { + if (!in_array($command['command'], ['config-refresh'])) { + // Command is not defined yet. + return; + } + // Replace previously declared command "config-refresh". + $command['commandfile'] = 'up_migrate'; + switch ($command['command']) { + case 'config-refresh': + $command['command'] = 'up-migrate-config-refresh'; + $command['command-hook'] = 'up-migrate-config-refresh'; + break; + } + // Strange, but necessary. + drush_set_context('command', $command); +} + +/** + * Config refresh command callback. + * + * @param string $name + * The extension name. + * @param string $type + * (optional) The extension type. + * + * @see ConfigInstaller::installDefaultConfig() + */ +function drush_up_migrate_config_refresh($name, $type = 'module') { + if (!in_array($type, ['module', 'theme', 'profile'])) { + $type = 'module'; + } + $config_installer = Drupal::service('config.installer'); + // Find default configuration of the extension. + $default_install_path = drupal_get_path($type, $name) . '/' . InstallStorage::CONFIG_INSTALL_DIRECTORY; + if (is_dir($default_install_path)) { + if (!$config_installer->isSyncing()) { + $storage = new FileStorage($default_install_path, StorageInterface::DEFAULT_COLLECTION); + $prefix = ''; + } + else { + $storage = $config_installer->getSourceStorage(); + $prefix = $name . '.'; + } + + // Gets profile storages to search for overrides if necessary. + $profile = Settings::get('install_profile'); + $profile_storages = []; + if ($profile && $profile != $name) { + $profile_path = drupal_get_path('module', $profile); + foreach ([InstallStorage::CONFIG_INSTALL_DIRECTORY, InstallStorage::CONFIG_OPTIONAL_DIRECTORY] as $directory) { + if (is_dir($profile_path . '/' . $directory)) { + $profile_storages[] = new FileStorage($profile_path . '/' . $directory, StorageInterface::DEFAULT_COLLECTION); + } + } + } + + $config_factory = Drupal::service('config.factory'); + $collection_info = Drupal::service('config.manager')->getConfigCollectionInfo(); + foreach ($collection_info->getCollectionNames() as $collection) { + $config_to_refresh = _up_migrate_config_refresh_get_config($storage, $collection, $prefix, $profile_storages); + // Remove existing configuration. + foreach (array_keys($config_to_refresh) as $config_name) { + $config_factory->getEditable($config_name)->delete(); + } + } + + // Re-install default config. + $config_installer->installDefaultConfig($type, $name); + Drupal::service('router.builder')->setRebuildNeeded(); + if ('theme' === $type) { + Drupal::moduleHandler()->invokeAll('themes_installed', [[$name]]); + } + else { + Drupal::moduleHandler()->invokeAll('modules_installed', [[$name]]); + } + + drush_log(sprintf('Default configuration refreshed for %s "%s".', $type, $name), LogLevel::OK); + } +} + +/** + * Gets configuration data from the provided storage. + * + * @param StorageInterface $storage + * The configuration storage to read configuration from. + * @param string $collection + * The configuration collection to use. + * @param string $prefix + * (optional) Limit to configuration starting with the provided string. + * @param StorageInterface[] $profile_storages + * An array of storage interfaces containing profile configuration to check + * for overrides. + * + * @return array + * An array of configuration data read from the source storage keyed by the + * configuration object name. + */ +function _up_migrate_config_refresh_get_config(StorageInterface $storage, $collection, $prefix = '', array $profile_storages = []) { + if ($storage->getCollectionName() != $collection) { + $storage = $storage->createCollection($collection); + } + $data = $storage->readMultiple($storage->listAll($prefix)); + + // Check to see if the corresponding override storage has any overrides. + foreach ($profile_storages as $profile_storage) { + if ($profile_storage->getCollectionName() != $collection) { + $profile_storage = $profile_storage->createCollection($collection); + } + $data = $profile_storage->readMultiple(array_keys($data)) + $data; + } + return $data; +} + +/** + * Drush callback to add a database as migration source. + * + * @param string $key + * Database key to use in migrations. + * @param string $database + * Name of database to add. + */ +function drush_up_migrate_upm_database_add($key, $database) { + $state = \Drupal::state(); + // Load existing databases. + $databases = $state->get('up_migrate.databases', []); + // Create new database information. + $databases[$key] = [ + 'database' => $database, + ]; + foreach (['username', 'password', 'host', 'port'] as $option) { + if (drush_get_option($option)) { + $databases[$key][$option] = drush_get_option($option); + } + } + $state->set('up_migrate.databases', $databases); +} diff --git a/up_migrate.info.yml b/up_migrate.info.yml new file mode 100644 index 0000000..df04b78 --- /dev/null +++ b/up_migrate.info.yml @@ -0,0 +1,9 @@ +type: module +name: undpaul Migrate +description: 'Base migration tools' +package: Migration +core: 8.x +dependencies: + - drupal:migrate (>=8.3) + - migrate_plus:migrate_plus + - migrate_tools:migrate_tools diff --git a/up_migrate.links.menu.yml b/up_migrate.links.menu.yml new file mode 100644 index 0000000..0420103 --- /dev/null +++ b/up_migrate.links.menu.yml @@ -0,0 +1,3 @@ +up_migration.migration_groups: + parent: migrate_tools.menu + deriver: \Drupal\up_migrate\Plugin\Derivative\MigrationGroupMenuLink diff --git a/up_migrate.module b/up_migrate.module new file mode 100644 index 0000000..c24fc4e --- /dev/null +++ b/up_migrate.module @@ -0,0 +1,22 @@ + $configuration) { + if (!empty($definitions[$id]['idMap'])) { + // Do not override existing values. + continue; + } + // Override id map for each migration. + $definitions[$id]['idMap'] = [ + 'plugin' => 'fastsql', + ]; + } +} diff --git a/up_migrate_examples/config/install/migrate_plus.migration.upm_example__user.yml b/up_migrate_examples/config/install/migrate_plus.migration.upm_example__user.yml new file mode 100644 index 0000000..1887a97 --- /dev/null +++ b/up_migrate_examples/config/install/migrate_plus.migration.upm_example__user.yml @@ -0,0 +1,47 @@ +id: upm_example__user +label: User accounts +migration_group: up_examples +destination: + plugin: entity:user +source: + plugin: 'upm_examples__user' + # Use database with key "upm_example". + # To register the database use "drush upm-da upm_example {database-name} ...". + key: upm_example + # Define base table for user data. This could be done in the source plugin + # also. + # @see \Drupal\up_migrate_examples\Plugin\migrate\source\UserAccounts + table: + name: users + alias: u + ids: + uid: integer + track_changes: true + constants: + langcode: 'de' +process: + uid: uid + name: name + pass: pass + mail: mail + created: created + access: access + login: login + status: status + timezone: timezone + langcode: 'constants/langcode' + preferred_langcode: 'constants/langcode' + preferred_admin_langcode: 'constants/langcode' + init: init + roles: + plugin: static_map + source: roles + map: + 1: anonymous + 2: authenticated + 3: administrator + default_value: authenticated +dependencies: + enforced: + module: + - up_migrate_examples diff --git a/up_migrate_examples/config/install/migrate_plus.migration_group.up_examples.yml b/up_migrate_examples/config/install/migrate_plus.migration_group.up_examples.yml new file mode 100644 index 0000000..47353ab --- /dev/null +++ b/up_migrate_examples/config/install/migrate_plus.migration_group.up_examples.yml @@ -0,0 +1,12 @@ +id: up_examples +label: undpaul Migration examples +description: Some examples for custom migrations. +source_type: Custom tables +shared_configuration: + source: + # Set default idMap plugin. + id_map: fast_sql +dependencies: + enforced: + module: + - up_migrate_examples diff --git a/up_migrate_examples/src/Plugin/migrate/source/UserAccounts.php b/up_migrate_examples/src/Plugin/migrate/source/UserAccounts.php new file mode 100644 index 0000000..41f6da7 --- /dev/null +++ b/up_migrate_examples/src/Plugin/migrate/source/UserAccounts.php @@ -0,0 +1,73 @@ +condition('u.uid', 0, '>'); + } + + /** + * {@inheritdoc} + */ + protected function alterFields(array $fields = []) { + $fields['uid'] = $this->t('User ID'); + $fields['name'] = $this->t('Username'); + $fields['pass'] = $this->t('Password'); + $fields['mail'] = $this->t('Email address'); + $fields['signature'] = $this->t('Signature'); + $fields['signature_format'] = $this->t('Signature format'); + $fields['created'] = $this->t('Registered timestamp'); + $fields['access'] = $this->t('Last access timestamp'); + $fields['login'] = $this->t('Last login timestamp'); + $fields['status'] = $this->t('Status'); + $fields['timezone'] = $this->t('Timezone'); + $fields['language'] = $this->t('Language'); + $fields['picture'] = $this->t('Picture'); + $fields['init'] = $this->t('Init'); + $fields['data'] = $this->t('User data'); + $fields['roles'] = $this->t('Roles'); + } + + /** + * {@inheritdoc} + */ + public function prepareRow(Row $row) { + if (!parent::prepareRow($row)) { + return FALSE; + } + $roles = $this->select('users_roles', 'ur') + ->fields('ur', ['rid']) + ->condition('ur.uid', $row->getSourceProperty('uid')) + ->execute() + ->fetchCol(); + $row->setSourceProperty('roles', $roles); + + $row->setSourceProperty('data', unserialize($row->getSourceProperty('data'))); + return TRUE; + } + +} diff --git a/up_migrate_examples/up_migrate_examples.info.yml b/up_migrate_examples/up_migrate_examples.info.yml new file mode 100644 index 0000000..c5ecb33 --- /dev/null +++ b/up_migrate_examples/up_migrate_examples.info.yml @@ -0,0 +1,7 @@ +type: module +name: 'undpaul Migrate Examples' +description: 'Migration examples' +package: Migration +core: 8.x +dependencies: + - up_migrate:up_migrate