d2b_migrate.module

<?php

require_once('PHPSQLParserAutoloader.php');
\PHPSQLParser\PHPSQLParserAutoloader::register();

define('D2B_MIGRATE_UPDATE_DEFAULT_URL', 'https://updates.backdropcms.org/release-history');

/**
* Implements hook_menu().
*/
function d2b_migrate_menu() {
  $items['d2b-migrate'] = array(
    'page callback' => 'backdrop_goto',
    'page arguments' => array('d2b-migrate/start'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_CALLBACK,
  );
  $items['d2b-migrate/start'] = array(
    'title' => 'Start',
    'page callback' => 'd2b_migrate_main_page',
    'page arguments' => array('start'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_CALLBACK,
  );
  $items['admin/config/system/d2b-migrate'] = array(
    'title' => 'Drupal to Backdrop Migrate',
    'page callback' => 'd2b_migrate_main_page',
    'page arguments' => array('start'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_NORMAL_ITEM,
  );
  $items['d2b-migrate/source'] = array(
    'title' => 'Source database setup',
    'page callback' => 'd2b_migrate_main_page',
    'page arguments' => array('source'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_CALLBACK,
  );
  $items['d2b-migrate/analysis'] = array(
    'title' => 'Source database analysis',
    'page callback' => 'd2b_migrate_main_page',
    'page arguments' => array('analysis'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_CALLBACK,
  );
  $items['d2b-migrate/backup'] = array(
    'title' => 'Set up database backup',
    'page callback' => 'd2b_migrate_main_page',
    'page arguments' => array('backup'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_CALLBACK,
  );
  $items['d2b-migrate/files'] = array(
    'title' => 'Files backup',
    'page callback' => 'd2b_migrate_main_page',
    'page arguments' => array('files'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_CALLBACK,
  );
  $items['d2b-migrate/restore'] = array(
    'title' => 'Migrate database',
    'page callback' => 'd2b_migrate_main_page',
    'page arguments' => array('restore'),
    'access arguments' => array('execute d2b'),
    'type' => MENU_CALLBACK,
  );

  return $items;
}

/**
 * Implements hook_permission().
 */
function d2b_migrate_permission() {
  return array(
    'execute d2b' => array(
      'title' => t('Execute Drupal to Backdrop migration'),
      'description' => t('Access and run the <a href="!url">Drupal to Backdrop migration</a>.', array('!url' => url('d2b-migrate'))),
      'restrict access' => TRUE,
      'warning' => t('Allows completely replacing the current site database with a migrated Drupal 7 site. Running a D2B migration will destroy all current content and replace it with the migrated content.'),
    ),
  );
}

/**
 * Implements hook_theme().
 */
function d2b_migrate_theme($existing, $type, $theme, $path) {
  return array(
    'd2b_migrate_page' => array(
      'variables' => array('current_task' => NULL, 'main_content' => NULL),
      'template' => 'migrate-page',
    ),
  );
}

/**
 * Add some variables for the task list.
 *
 * @param $variables
 *   An associative array containing:
 *     - current_task : the current task.
 */
function template_preprocess_d2b_migrate_page(&$variables) {
  $variables['task_list'] = d2b_migrate_task_list($variables['current_task']);
}

/**
 * Get a task list to the sidebar area.
 *
 * This will need to be called from every page of the install process.
 *
 * @param $active
 *   (Optional) Set the active task by key.
 *
 * @return string
 *   The rendered HTML for the installation task list.
 */
function d2b_migrate_task_list($active = NULL) {
  // Default list of tasks.
  $tasks = array(
    'start' => l(t('Start'), 'd2b-migrate/start'),
    'source' => l(t('Source database'), 'd2b-migrate/source'),
    'analysis' => l(t('Database analysis'), 'd2b-migrate/analysis'),
    'files' => l(t('Backup files'), 'd2b-migrate/files'),
    'restore' => l(t('Final upgrade'), 'd2b-migrate/restore'),
  );

  require_once BACKDROP_ROOT . '/core/includes/theme.maintenance.inc';

  return theme('task_list', array('items' => $tasks, 'active' => $active));
}

/**
 * Menu callback; the main D2B Migrate wizard.
 *
 * @param $op
 *   Operation to preform.
 *
 * @return string
 *   The rendered wizard page.
 */
function d2b_migrate_main_page($op) {
  backdrop_add_css(backdrop_get_path('module', 'd2b_migrate') . '/d2b_migrate.css', array('preprocess' => FALSE));

  $private_files = config_get('system.core','file_private_path');
  if (empty($private_files)) {
    $op = 'start';
    backdrop_set_message(t('You must specify a private file system path in the !settings to continue.', array('!settings' => l(t('file system settings'), 'admin/config/media/file-system'))), 'warning');
  }

  switch ($op) {
    case 'start':
      backdrop_set_title(t("Start"));
      $content = 'd2b_migrate_start_form';
      break;
    case 'source':
      backdrop_set_title(t("Source database"));
      $content = 'd2b_migrate_source_database_form';
      break;
    case 'analysis':
      backdrop_set_title(t("Source analysis"));
      $content = 'd2b_migrate_analysis_form';
      break;
    case 'files':
      backdrop_set_title(t("Files backup"));
      $content = 'd2b_migrate_files_form';
      break;
    case 'restore':
      backdrop_set_title(t("Migrate database"));
      $content = 'd2b_migrate_restore_form';
      break;
    default:
      return MENU_NOT_FOUND;
  }
  $form = backdrop_get_form($content);
  return theme('d2b_migrate_page', array('current_task' => $op, 'main_content' => backdrop_render($form)));
}

/**
 * Implements hook_admin_paths().
 */
function d2b_migrate_admin_paths() {
  $paths = array(
    'd2b-migrate' => TRUE,
    'd2b-migrate/*' => TRUE,
  );
  return $paths;
}

/**
 * Returns the active source and sets a page message.
 */
function d2b_migrate_get_active_database() {
  $source = array();
  $source_data = d2b_migrate_get_source();
  if (empty($source_data)) {
    backdrop_set_message(t('Source database not set.'), 'warning');
  }
  elseif ($source_data['type'] == 'db') {
    backup_migrate_include('sources');
    $database = $source_data['source'];
    $sources = backup_migrate_get_sources();
    if (isset($sources[$database])) {
      $source = $sources[$database]->dest_url;
      backdrop_set_message(t('Current source database: path %path with user %user.', array('%path' => $source['path'], '%user' => $source['user'])), 'info');
    }
    else {
      backdrop_set_message(t('The saved database source is invalid. Please re-submit the Source form.'), 'warning');
    }
  }
  elseif ($source_data['type'] == 'file') {
    if (file_exists($source_data['source'])) {
      $path_parts = pathinfo($source_data['source']);
      $file = $path_parts['basename'];
      backdrop_set_message(t('Current source database: file %file.', array('%file' => $file)), 'info');
    }
    else {
      backdrop_set_message(t('The saved file source is invalid. Please re-submit the Source form.'), 'warning');
    }
  }
  return $source_data;
}


/**
 * Returns the start form.
 */
function d2b_migrate_start_form($form, &$form_state) {
  $source = d2b_migrate_get_active_database();

  $private_files = config_get('system.core','file_private_path');

  $form['help'] = array(
    '#type' => 'help',
    '#markup' => t('This form will require access to the source database. This will be required in the next step.'),
  );
  $form['actions'] = array(
    '#type' => 'actions',
  );
  $form['actions']['submit'] = array(
    '#type' => 'submit',
    '#value' => $source ? t('Next: modify source database') : t('Next: set source database'),
    '#disabled' => empty($private_files),
  );
  return $form;
}

/**
 * Submit handler for the start form.
 */
function d2b_migrate_start_form_submit($form, &$form_state) {
  $form_state['redirect'] = 'd2b-migrate/source';
}

/**
 * Returns the source database form.
 */
function d2b_migrate_source_database_form($form, &$form_state) {
  backup_migrate_include('crud');

  $form += backup_migrate_crud_ui_create('source', 'mysql');
  $form['db_prefix'] = array(
    '#type' => 'textfield',
    '#title' => t('Database prefix'),
    '#description' => t('Enter the Drupal database table prefix (including any underscore), or leave blank if no prefix. If there are multiple prefixed tables in this database, all tables will be migrated, not only those with this prefix.'),
    '#weight' => 80,
  );
  $form['#validate'] = array('d2b_migrate_source_database_form_validate');
  $form['#submit'] = array('d2b_migrate_source_database_form_submit');
  $form['upload_wrapper'] = array(
    '#type' => 'fieldset',
    '#title' => t('Or upload a backup file'),
    '#collapsible' => TRUE,
    '#collapsed' => TRUE,
    '#description' => t("This <em>must</em> be a backup file created by Backup and Migrate module."),
    '#weight' => 90,
  );
  $form['upload_wrapper']['upload'] = array(
    '#type' => 'file',
    '#title' => t('Choose a backup file'),
    '#size' => 22,
  );
  return $form;
}

/**
 * Implements hook_form_FORM_ID_alter().
 */
function d2b_migrate_form_backup_migrate_crud_edit_form_alter(&$form, &$form_state, $form_id) {
  // todo: add a disconect button.
  // todo: prevent this from altering except for D2B Migrate.
  $value = uniqid();

  $source_data = d2b_migrate_get_active_database();
  if (isset($source_data['type']) && $source_data['type'] == 'db') {
    backup_migrate_include('sources');
    $database = $source_data['source'];
    $sources = backup_migrate_get_sources();
    if (isset($sources[$database])) {
      $source = $sources[$database]->dest_url;

      $form['host']['#default_value'] = $source['host'];
      $form['user']['#default_value'] = $source['user'];
      $form['path']['#default_value'] = $source['path'];
      $form['pass']['#default_value'] = $source['pass'];
      $form['port']['#default_value'] = isset($source['port']) ? $source['port'] : '';
    }
  }

  $form['help'] = array(
    '#type' => 'help',
    '#markup' => t('Enter the connection details for the source Drupal 7 site.'),
  );
  $form['name']['#type'] = 'value';
  $form['name']['#value'] = $value;
  $form['machine_name']['#type'] = 'value';
  $form['machine_name']['#value'] = $value;
  $form['actions']['submit']['#value'] = t('Connect');
}

/**
 * Validate handler for the source database form.
 */
function d2b_migrate_source_database_form_validate($form, &$form_state) {
  $destination = 'private://backup_migrate/manual';
  // $destination = 'public://';
  backup_migrate_include('profiles');
  backup_migrate_include('destinations');
  $validators = array('file_validate_extensions' => array('gz zip sql mysql bz bz2'));

  // Prepare the destination; if this fails, dont set a form error yet, as the
  // use may not have tried file upload at all, if using DB connection.
  $file_prepare_directory = file_prepare_directory($destination, FILE_CREATE_DIRECTORY);
  $file = file_save_upload('upload', $validators, $destination, FILE_EXISTS_REPLACE);

  // File upload was attempted.
  if (isset($file)) {
    if (!$file_prepare_directory) {
      form_set_error('upload', t('The upload directory could not be configured.'));
    }
    elseif ($file) {
      $filename = $destination . '/' . $file->filename . '.info';
      $info_file = array(
        'filename' => $filename,
        'generator' => 'Backup and Migrate for Backdrop (https://github.com/backdrop-contrib/backup_migrate)',
      );
      $data = _backup_migrate_array_to_ini($info_file);
      file_put_contents($filename, $data);
      $file = d2b_migrate_archive_extract($file->uri, $destination);
      if (!$file) {
        form_set_error('upload', t('The uploaded file is invalid, or it is not a Drupal database file.'));
        // todo: validate that this is a D7 db.
      }
      else {
        $form_state['file'] = $file;
      }
    }
    else {
      // File upload failed.
      form_set_error('upload', t('The backup file could not be uploaded.'));
    }
  }
  else {
    // If a file wasn't uploaded, connect to the Drupal database to check that
    // it is valid.
    try {
      $form_state['values'] = array_merge($form_state['values'], $form_state['input']);
      $source = $form_state['values'];

      $d7_database = array(
        'database' => $source['path'],
        'username' => $source['user'],
        'password' => $source['pass'],
        'host' => $source['host'],
        'port' => $source['port'],
        'driver' => $source['scheme']
      );

      Database::addConnectionInfo('migrateKeys', 'default', $d7_database);
      db_set_active('migrateKeys');

      // Query a table which does not exist in Backdrop. Use the DB prefix if
      // provided.
      $role = $form_state['values']['db_prefix'] . 'role';
      $sql = "SELECT * FROM $role";
      $result = db_query($sql);
      db_set_active();
    }
    catch (\PDOException $e) {
      form_set_error('path', t('Could not connect to the database, or it is not a Drupal database.'));
      db_set_active();
    }
  }
}

/**
 * Unpacks an uploaded backup file and moves it to the manual destination.
 */
function d2b_migrate_archive_extract($file, $directory) {
  backup_migrate_include('sources');
  backup_migrate_include('destinations');
  backup_migrate_include('profiles');
  backup_migrate_include('filters');
  backup_migrate_get_filters();

  $archiver = new backup_migrate_filter_compression();
  $settings = _backup_migrate_profile_saved_default_profile();
  if (!$archiver) {
    throw new Exception(t('Cannot extract %file, not a valid archive.', array('%file' => $file)));
  }

  $out = new backup_file(array('filepath' => $file));
  $new_file = $archiver->_backup_migrate_file_decompress($out, $settings);
  $ext = implode('.', $new_file->ext);
  $name = $new_file->name . '.' . $ext;
  $return = file_unmanaged_copy($new_file->path, $directory . '/' . $name, FILE_EXISTS_RENAME);

  return $return;
}

/**
 * Submit handler for the source database form.
 */
function d2b_migrate_source_database_form_submit($form, &$form_state) {
  if (isset($form_state['file'])) {
    // Save the BAM database name in state so we can use it later.
    d2b_migrate_set_source($form_state['file'], 'file');
  }
  elseif (isset($form_state['values']['item'])) {
    $item = $form_state['values']['item'];
    // For some reason, the info is not in form_state values. Use input.
    // This should be OK since we validate the input above.
    $form_state['values'] = array_merge($form_state['values'], $form_state['input']);

    $item->edit_form_submit($form, $form_state);

    // Save the BAM database name in state so we can use it later.
    d2b_migrate_set_source($form_state['values']['machine_name'], 'db');
  }
  backdrop_set_message(t('Source database has been saved.'));
  _update_cache_clear('d2b_migrate_get_cached_projects');

  $form_state['redirect'] = 'd2b-migrate/analysis';
}

/**
 * Returns the database analysis form.
 */
function d2b_migrate_analysis_form($form, &$form_state) {
  $source = d2b_migrate_get_active_database();
  if (empty($source)) {
    _update_cache_clear('d2b_migrate_get_cached_projects');
  }

  backdrop_add_js(backdrop_get_path('module', 'd2b_migrate') .'/d2b_migrate.js');
  $projects = _update_cache_get('d2b_migrate_get_cached_projects');

  $project_data = !empty($projects->data) ? $projects->data : array();
  // Try and build a real name for the project.
  foreach ($project_data as $key => $project) {
    if (backdrop_get_filename('module', $key)) {
      $location = t('Already downloaded');
      $project_data[$key]['location'] = 'already_downloaded';
    }
    if (isset($project['includes'][$key])) {
      $project_data[$key]['realname'] = $project['includes'][$key];
    }
    else {
      $project_data[$key]['realname'] = ucfirst($key);
    }
  }
  backdrop_sort($project_data, array('location' => SORT_STRING));

  $header = array(
    'project_name' => array('data' => t('Project name')),
    'type' => array('data' => t('Type')),
    'location' => t('Location'),
  );
  $rows = array();

  $downloadables = array();
  $default_values = array();
  foreach ($project_data as $project_name => $project) {
    $default_values[$project_name] = 1;
    $form_state['redirect'] = 'admin/installer/install/select_versions';
    $row = array();
    if (isset($project['in_core_since'])) {
      $location = t('In core');
    }
    elseif ($project['location'] == 'already_downloaded') {
      $location = t('Already downloaded');
    }
    elseif ($project['location'] == 'backdrop') {
      $downloadables[$project_name] = $project;
      $location = t('Available for download');
    }
    elseif ($project['location'] == 'github') {
      $default_values[$project_name] = 0;
      $location = l(t('Download from GitHub'), 'https://github.com/backdrop-contrib/' . $project_name, array('external' => TRUE, 'attributes' => array('target'=>'_blank')));
    }
    elseif ($project['location'] == 'not_found') {
      $default_values[$project_name] = 0;
      $location = t('Not yet ported');
    }
    if (trim($project['project_status'], "'") != 1) {
      $default_values[$project_name] = 0;
      $location .= t(' <em>(Disabled in source)</em>');
    }

    $row['project_name'] = $project['realname'];
    $row['type'] = t('Module');
    $row['location'] = $location;
    $rows[$project_name] = $row;
  }

  $form['help'] = array(
    '#type' => 'help',
    '#markup' => t('Click the "Update project data" button to fetch the list of modules installed on the source Drupal 7 site and see their availability in Backdrop. Click the "Download available projects" button to download all modules listed as "Available for download". Modules available on GitHub must be downloaded manually.'),
  );

  $form['downloadables'] = array(
    '#type' => 'value',
    '#value' => $downloadables,
  );

  $form['projects'] = array(
    '#type' => 'tableselect',
    '#default_value' => $default_values,
    '#header' => $header,
    '#options' => $rows,
    '#empty' => t('No source module data was found.'),
  );

  $form['fetch'] = array(
    '#type' => 'submit',
    '#submit' => array('d2b_migrate_analysis_fetch_projects'),
    '#disabled' => empty($source),
    '#value' => t('Update project data'),
  );
  $form['installer'] = array(
    '#type' => 'submit',
    '#submit' => array('d2b_migrate_analysis_download_projects'),
    '#disabled' => empty($source),
    '#value' => t('Download available projects'),
  );
  $form['actions'] = array(
    '#type' => 'actions',
  );
  $form['actions']['submit'] = array(
    '#disabled' => empty($source),
    '#type' => 'submit',
    '#value' => t('Next: backup files'),
  );

  return $form;
}

/**
 * Handler for the "download available projects" button on the database
 * analysis form.
 */
function d2b_migrate_analysis_download_projects($form, &$form_state) {
  module_load_include('inc', 'installer', 'installer.browser');

  $modules = installer_browser_fetch_results(array('items_per_page' => 999, 'type' => 'module'));
  $all_projects = $modules['projects'];

  $batch_projects = array();
  foreach ($form_state['values']['downloadables'] as $project_name => $downloadable) {
    if ($form_state['values']['projects'][$project_name] && isset($all_projects[$project_name])) {
      installer_browser_install_queue_add($all_projects[$project_name]);
      $batch_projects[$project_name] = $all_projects[$project_name];
    }
  }

  unset($_SESSION['installer_browser_install_releases_list']);

  d2b_migrate_analysis_batch_download($batch_projects);
}

/**
 * Batch API utility.
 */
function d2b_migrate_analysis_batch_download($projects) {
  foreach ($projects as $project) {
    if ($project) {
      // Get the available releases for this project.
      $release_data = installer_browser_get_project_release_data($project);
      if (!$release_data) {
        backdrop_set_message(t('Could not fetch releases for project %project.', array('%project' => $project['title'])), 'warning');
        watchdog('installer', 'Could not fetch releases for project %project.', array('%project' => $project['title']), WATCHDOG_ERROR);
        installer_browser_install_queue_remove($project['name']);
        continue;
      }

      // We use the update module to calculate the recommended version.
      $project_data = array(
        'existing_major' => 0,
        'existing_version' => 0,
        'install_type' => '',
      );
      module_load_include('inc', 'update', 'update.compare');
      update_calculate_project_update_status($project_data, $release_data);

      $releases[$project['name']]['release_name'] = isset($project_data['recommended']) ? $project_data['recommended'] : key($release_data);
      $releases[$project['name']]['project'] = $project;
    }
  }

  module_load_include('inc', 'installer', 'installer.browser');
  // Store maintenance_mode setting so we can restore it when done.
  $_SESSION['maintenance_mode'] = state_get('maintenance_mode', FALSE);

  foreach ($releases as $item) {
    // Load the selected release.
    if ($release = installer_browser_get_release($item['release_name'], $item['project'])) {
      // Add the release to a session variable.
      $_SESSION['installer_browser_install_releases_list'][$item['release_name']] = $item['project'];
    }
  }

  // Install the projects with batch.
  module_load_include('inc', 'installer', 'installer.manager');

  $queued_releases = installer_browser_get_queued_releases();
  $operations = array();
  foreach ($queued_releases as $release_name => $project) {
    $operations[] = array('_installer_browser_batch_install_release', array($release_name, $project));
  }
  $batch = array(
    'operations' => $operations,
    'finished' => '_d2b_migrate_analysis_batch_download_finished',
    'title' => t('Installing projects'),
    'init_message' => t('Installing projects...'),
    'progress_message' => t('Installed @current out of @total.'),
    'error_message' => t('Installation has encountered an error.'),
    'file' => backdrop_get_path('module', 'installer') . '/installer.browser.inc',
  );
  batch_set($batch);
}

/**
 * Batch API callback: shows a message and finishes up the batch.
 */
function _d2b_migrate_analysis_batch_download_finished($success, $results, $operations) {
  // Although we turn off maintenance mode immediately, the message that the
  // site is offline displays until a page reload. We wipe the messages cache
  // here to avoid this.
  backdrop_get_messages();

  // Restore the maintenance mode to what it was at the start.
  if (isset($_SESSION['maintenance_mode'])) {
    state_set('maintenance_mode', $_SESSION['maintenance_mode']);
    unset($_SESSION['maintenance_mode']);
  }

  unset($_SESSION['installer_browser_install_releases_list']);
  if ($success) {
    if (!empty($results)) {
      if (!empty($results['failures'])) {
        backdrop_set_message(format_plural(count($results['failures']), 'Failed to install one project.', 'Failed to install @count projects.'), 'error');
        foreach($results['failures'] as $message){
          backdrop_set_message($message, 'error');
        }
      }
    }
  }
  else {
    backdrop_set_message(t('Error installing projects.'), 'error');
    backdrop_goto('d2b-migrate/analysis');
  }

  $projects = installer_browser_get_installed_projects();
  $missing = installer_browser_get_missing_dependencies($projects);
  // If there are missing dependencies, go to install dependencies.
  if (count($missing) > 0) {
    // @Todo: since projects come from an existing site, youd expect all dependencies
    // to be in the list, but we could probaby figure some action for this.
    backdrop_goto('d2b-migrate/analysis');
  }
  else {
    backdrop_goto('d2b-migrate/analysis');
  }
}

/**
 * Submit handler for the database analysis form.
 */
function d2b_migrate_analysis_form_submit($form, &$form_state) {
  $form_state['redirect'] = 'd2b-migrate/files';
}

/**
 * Handler for the "Update project data" button of the database analysis form.
 */
function d2b_migrate_analysis_fetch_projects($form, &$form_state) {
  _update_cache_clear('d2b_migrate_get_cached_projects');
  backup_migrate_include('sources');
  // $database = state_get('d2b_migrate_source_db_name', '');
  $db_source = d2b_migrate_get_source();
  if (empty($db_source)) {
    return;
  }
  $projects = array();

  if ($db_source['type'] == 'db') {
    $database = $db_source['source'];
    $sources = backup_migrate_get_sources();
    $source = $sources[$database]->dest_url;

    $d7_database = array(
      'database' => $source['path'],
      'username' => $source['user'],
      'password' => $source['pass'],
      'host' => $source['host'],
      'port' => $source['port'],
      'driver' => $source['scheme']
    );

    Database::addConnectionInfo('migrateKey', 'default', $d7_database);
    db_set_active('migrateKey');

    try {
      $result = db_query('SELECT * FROM {system}');
    }
    catch (Exception $exception) {
      db_set_active();
      backdrop_set_message(t('Could not connect to the database: "@error".', array('@error' => $exception->getMessage())), 'error');
      $form_state['redirect'] = 'd2b-migrate/analysis';
      return;
    }

    db_set_active();

    $system_list = array();

    foreach ($result as $record) {
      $record->info = unserialize($record->info);
      // Only include visible contrib modules.
      // @Todo: contrib themes would still be in this; need to exclude them.
      if (!empty($record->info['hidden'])) {
        continue;
      }
      if (strpos($record->filename, 'modules/') === 0) {
        continue;
      }
      if (strpos($record->filename, 'themes/') === 0) {
        continue;
      }
      $record->info['_info_file_ctime'] = 1;
      $system_list[$record->name] = $record;
    }
    module_load_include('inc', 'update', 'update.compare');
    d2b_migrate_process_info_list($projects, $system_list, 'module', TRUE);
  }
  elseif ($db_source['type'] == 'file') {
    $projects = d2b_migrate_parse_sql_system_table($db_source['source']);
  }

  // After we fetch the project list, now we're going to use batch API to
  // see if there are Backdrop versions of each and save this in the project
  // metadata. This is quite intensive so we save in upgrade cache once fetched.
  $cached_projects = _update_cache_get('d2b_migrate_get_cached_projects');
  $cached_project_data = array();

  // If we haven't fetched before, then build the batch operation to fetch.
  if (!empty($cached_projects->data)) {
    $cached_project_data = $cached_projects->data;
  }

  module_load_include('inc', 'd2b_migrate', 'd2b_migrate.core');

  // Check for projects that are already in Backdrop core. We don't need to
  // check for contrib versions of those of course.
  foreach ($projects as $project) {
    if (!isset($cached_project_data[$project['name']])) {
      $core = d2b_migrate_core($projects, $project['name']);
    }
  }

  $operations = array();
  foreach ($projects as $project) {
    if (!isset($cached_project_data[$project['name']])) {
      $operations[] = array('d2b_migrate_analysis_fetch_project', array($project));
    }
  }
  if (!empty($operations)) {
    $batch = array(
      'operations' => $operations,
      'finished' => 'd2b_migrate_analysis_fetch_project_finished',
      'title' => t('Processing project'),
      'init_message' => t('Fetching data about projects.'),
      'progress_message' => t('Processed @current out of @total.'),
      'error_message' => t('Batch project fetch encountered an error.'),
    );
    batch_set($batch);
    batch_process('d2b-migrate/analysis');
  }
  else {
    backdrop_set_message(t('All projects updated'));
    $form_state['redirect'] = 'd2b-migrate/analysis';
  }
}

/**
 * Batch API handler for the database analysis form. Fetches data about
 * available projects.
 */
function d2b_migrate_analysis_fetch_project($project, &$context) {
  $project_name = $project['name'];
  if (!empty($project['in_core_since'])) {
    $project['location'] = 'in_core';
  }
  else {
    $url = D2B_MIGRATE_UPDATE_DEFAULT_URL . '/' . $project_name . '/1.x';
    $xml = backdrop_http_request($url);
    module_load_include('inc', 'update', 'update.fetch');
    if (!isset($xml->error) && isset($xml->data) && $data = update_parse_xml($xml->data)) {
      $project['location'] = 'backdrop';
    }
    else {
      $github = 'https://github.com/backdrop-contrib/' . $project_name;
      $response = backdrop_http_request($github);
      if ($response->code != '404') {
        $project['location'] = 'github';
      }
      else {
        $project['location'] = 'not_found';
      }
    }
  }

  $context['results']['projects'][$project_name] = $project;
  $context['message'] = t('Processing project "@title"', array('@title' => $project['info']['name']));
}

/**
 * Batch API callback: shows a message and finishes up the batch.
 */
function d2b_migrate_analysis_fetch_project_finished($success, $results, $operations) {
  if ($success) {
    $cached_data = _update_cache_get('d2b_migrate_get_cached_projects');
    if (!empty($cached_data->data)) {
      $data = array_merge($cached_data->data, $results['projects']);
    }
    else {
      $data = $results['projects'];
    }
    _update_cache_set('d2b_migrate_get_cached_projects', $data, REQUEST_TIME + 3600);
    // Here we could do something meaningful with the results.
    // We just display the number of nodes we processed...
    backdrop_set_message(t('@count projects  processed.', array('@count' => count($results['projects']))));
  } else {
    // An error occurred.
    // $operations contains the operations that remained unprocessed.
    $error_operation = reset($operations);
    backdrop_set_message(t('An error occurred while processing @operation with arguments : @args', array('@operation' => $error_operation[0], '@args' => print_r($error_operation[0], TRUE))));
  }
}

/**
 * Returns the download files form.
 */
function d2b_migrate_files_form($form, &$form_state) {
  $source = d2b_migrate_get_active_database();
  $form['help'] = array(
    '#type' => 'help',
    '#markup' => t('Copy "managed" files from the source site. Managed files are files with a database record. Many files in the public files directory may not have a database record, such as default images and image styles. This method of migrating files requires the file URL be known and accessible over http, and is unreliable. The recommended option for migrating files is to manually copy the directory contents from the server.'),
  );
  $form['source_url'] = array(
    '#type' => 'textfield',
    '#title' => t('Source site URL'),
    '#default_value' => '',
    '#required' => TRUE,
    '#description' => t('Enter the URL of the existing Drupal 7 site.')
  );
  $form['backup'] = array(
    '#type' => 'submit',
    '#submit' => array('d2b_migrate_files_form_backup'),
    '#value' => t('Copy files'),
    '#disabled' => empty($source),
  );
  $form['actions'] = array(
    '#type' => 'actions',
  );
  $form['actions']['submit'] = array(
    '#disabled' => empty($source),
    '#type' => 'submit',
    '#value' => t('Next: prepare for upgrade'),
    '#submit' => array('d2b_migrate_files_form_submit'),
    '#limit_validation_errors' => array(),
  );
  return $form;
}

/**
 * Submit handler for the "copy files" button on the download files form.
 */
function d2b_migrate_files_form_backup($form, &$form_state) {
  $source_base_path = $form_state['values']['source_url'];
  $data_source = d2b_migrate_get_source();

  if ($data_source['type'] == 'db') {
    $source_database = $data_source['source'];
    backup_migrate_include('sources');

    $sources = backup_migrate_get_sources();
    $source = $sources[$source_database]->dest_url;

    $d7_database = array(
      'database' => $source['path'],
      'username' => $source['user'],
      'password' => $source['pass'],
      'host' => $source['host'],
      'driver' => $source['scheme']
    );

    Database::addConnectionInfo('migrateKey', 'default', $d7_database);
    db_set_active('migrateKey');

    $result = db_query('SELECT uri FROM {file_managed}')->fetchCol();
    $public_set = db_query('SELECT value FROM {variable} WHERE name = :file_public_path', [
        ':file_public_path' => 'file_public_path',
    ])->fetchField();

    $file_public_path = $public_set ? unserialize($public_set) : 'sites/default/files';

    db_set_active();
  }
  elseif ($data_source['type'] == 'file') {
    $result = d2b_migrate_parse_sql_file_managed_table($data_source['source']);
    $file_public_path = d2b_migrate_parse_sql_variable_table($data_source['source']);
  }

  foreach ($result as $record) {
    $path_info = pathinfo($record);
    if (strpos($record, 'public://') === 0) {
      $dir_path = substr($path_info['dirname'], 9);
      $stream = backdrop_realpath('public://');
    }
    elseif (strpos($record, 'private://') === 0) {
      continue;
    }
    $source_file = $source_base_path . $file_public_path . '/' . $dir_path . '/' . $path_info['basename'];
    $new_dir = $stream . '/' . $dir_path;
    file_prepare_directory($new_dir, FILE_CREATE_DIRECTORY);
    $new_file = $new_dir . '/' . $path_info['basename'];

    if (copy($source_file, $new_file) ) {
        // dpm( "Copy success!");
    }else{
        // dpm( "Copy no success!");
    }

  }
}

/**
 * Submit handler for the download files form.
 */
function d2b_migrate_files_form_submit($form, &$form_state) {
  $form_state['redirect'] = 'd2b-migrate/restore';
}


/**
 * Returns the restore form.
 */
function d2b_migrate_restore_form($form, &$form_state) {
  $source = d2b_migrate_get_active_database();

  $form = array();
  $form['help'] = array(
    '#type' => 'help',
    '#markup' => t('This step will import the database from the Drupal site, and then, after this is complete, you will be redirected to run database updates to upgrade this Drupal database to Backdrop. <strong>This is the final step and cannot be reversed.</strong>'),
  );


  $form['actions'] = array(
    '#type' => 'actions',
  );
  $form['actions']['submit'] = array(
    '#type' => 'submit',
    '#disabled' => empty($source),
    '#value' => t('Upgrade Drupal 7 database'),
  );

  return $form;
}

/**
 * Submit handler for the restore form.
 */
function d2b_migrate_restore_form_submit($form, &$form_state) {
  backup_migrate_include('sources');
  backup_migrate_include('destinations');
  backup_migrate_include('profiles');

  $source = d2b_migrate_get_source();

  // Create a backup file. Either we use Backup and Migrates backup functions to
  // create one from a connected database, or prepare an uploaded backup file.
  if ($source['type'] == 'db') {
    // Backup file
    $destinations = backup_migrate_get_destinations();

    $settings = _backup_migrate_profile_saved_default_profile();

    $settings->destination_id = array('manual');

    $database = $source['source'];
    $settings->source_id = $database;

    $backup_file = backup_migrate_perform_backup($settings);
    $latest_file = backup_migrate_destination_get_latest_file('manual');
    $file = $latest_file->file_info['file_id'];
  }
  elseif ($source['type'] == 'file') {
    $path_parts = pathinfo($source['source']);
    $file = $path_parts['basename'];
  }

  // Now we restore from the backup file.
  $values = array();
  $values['source_id'] = 'db';
  $values['from'] = 'saved';

  d2b_migrate_perform_restore('manual', $file, $values);
}

/**
 * Restore from a file in the given destination.
 */
function d2b_migrate_perform_restore($destination_id, $file, $settings = array()) {
  backup_migrate_include('files', 'filters');
  timer_start('backup_migrate_restore');

  // If not in 'safe mode', increase the maximum execution time:
  if (!ini_get('safe_mode') && ini_get('max_execution_time') < config_get('backup_migrate.settings','backup_migrate_backup_max_time')) {
    set_time_limit(config_get('backup_migrate.settings','backup_migrate_restore_max_time'));
  }

  // Make the settings into a default profile.
  if (!is_object($settings)) {
    $settings = backup_migrate_crud_create_item('profile', $settings);
    $settings->source_id = empty($settings->source_id) ? 'db' : $settings->source_id;
  }

  // Register shutdown callback.
  register_shutdown_function('d2b_migrate_shutdown', $settings);

  if (!is_object($file)) {
    // Load the file from the destination.
    $file = backup_migrate_destination_get_file($destination_id, $file);
    if (!$file) {
      _backup_migrate_message("Could not restore because the file could not be loaded from the destination.", array(), 'error');
      backup_migrate_cleanup();
      return FALSE;
    }
  }

  // Filter the file and perform the restore.
  $file = backup_migrate_filters_restore($file, $settings);
  if (!$file) {
    if (_backup_migrate_check_timeout()) {
      backup_migrate_restore_fail('Could not perform the restore because the script timed out. Try increasing your PHP <a href="!url">max_execution_time setting</a>.', array('!url' => 'http://www.php.net/manual/en/info.configuration.php#ini.max-execution-time'), 'error');
    }
    else {
      backup_migrate_restore_fail("Could not perform the restore.", array(), 'error');
    }
    backup_migrate_cleanup();
    return FALSE;
  }

  state_set('database_utf8mb4_active', FALSE);

  $time = timer_stop('backup_migrate_restore');

  // Delete any temp files we've created.
  backup_migrate_cleanup();

  // No errors. Return the file.
  return $file;
}

/**
 * Shutdown callback when the script terminates even if the script timed out.
 */
function d2b_migrate_shutdown($settings) {
  // If we ran out of time, set an error so the user knows what happened
  if (_backup_migrate_check_timeout()) {
    backup_migrate_cleanup();
    backup_migrate_backup_fail('The operation timed out. Try increasing your PHP <a href="!url">max_execution_time setting</a>.', array('!url' => 'http://www.php.net/manual/en/info.configuration.php#ini.max-execution-time'), $settings);
  }

  // The session will have already been written and closed, so we need to write any changes directly.
  _backdrop_session_write(session_id(), session_encode());

  // backdrop_goto() calls t() at some point, on whats now still essentially a
  // D7 database, which is therefore  missing a locale column and thus causes
  // an exception. So we fake backdrop_goto().
  d2b_migrate_goto();
}

/**
 * Copy of backdrop_goto() that doesn't crash.
 */
function d2b_migrate_goto() {
  $url = base_path() . 'core/update.php';

  header('Location: ' . $url, TRUE, 302);
  backdrop_exit($url);
}

/**
 * Returns all currently cached data about available releases for all projects.
 *
 * @return
 *   Array of data about available releases, keyed by project shortname.
 */
function d2b_migrate_get_cached_projects() {
  $data = array();
  $cache_items = _update_get_cache_multiple('d2b_migrate_get_cached_projects');
  foreach ($cache_items as $cid => $cache) {
    $cache->data['last_fetch'] = $cache->created;
    if ($cache->expire < REQUEST_TIME) {
      $cache->data['fetch_status'] = UPDATE_FETCH_PENDING;
    }
    // The project shortname is embedded in the cache ID, even if there's no
    // data for this project in the DB at all, so use that for the indexes in
    // the array.
    $parts = explode('::', $cid, 2);
    $data[$parts[1]] = $cache->data;
  }
  return $data;
}

/**
 * Implements hook_autoload_info().
 */
function d2b_migrate_autoload_info() {
  return array(
    'SqlParser' => 'SqlParser.php',
  );
}

/**
 * Parses an uploaded sql file to get the saved file public path.
 */
function d2b_migrate_parse_sql_variable_table($sql_path) {
  $sqlLists = SqlParser::parse(file_get_contents($sql_path));
  $string = 'INSERT INTO `variable`';
  $file_public_path = 'sites/default/files';
  foreach ($sqlLists as $sqlList) {
    if (strpos($sqlList, $string) === 0) {
      $parser = new PHPSQLParser\PHPSQLParser($sqlList);
      $data = $parser->parsed['VALUES'];
      foreach ($data as $line) {
        if ($line['data'][0]['base_expr'] == "'file_public_path'") {
          $file_public_path = hex2bin(ltrim($line['data'][0]['base_expr'], '0x'));
          break;
        }
      }
    }
  }

  return $file_public_path;
}

/**
 * Parses an uploaded sql file to get all managed file paths.
 */
function d2b_migrate_parse_sql_file_managed_table($sql_path) {
  $sqlLists = SqlParser::parse(file_get_contents($sql_path));
  $string = 'INSERT INTO `file_managed`';
  $files = array();
  foreach ($sqlLists as $sqlList) {
    if (strpos($sqlList, $string) === 0) {
      $parser = new PHPSQLParser\PHPSQLParser($sqlList);
      $data = $parser->parsed['VALUES'];
      foreach ($data as $line) {
        $uri = trim($line['data'][3]['base_expr'], "'");
        if (strpos($uri, 'private://') === 0) {
          continue;
        }
        $files[] = $uri;
      }
    }
  }

  return $files;
}

/**
 * Parses an uploaded sql file to get the all site modules.
 */
function d2b_migrate_parse_sql_system_table($sql_path) {
  $sqlLists = SqlParser::parse(file_get_contents($sql_path));
  $string = 'INSERT INTO `system`';
  $modules = array();
  $system_array = array(
    'filename',
    'name',
    'type',
    'owner',
    'status',
    'bootstrap',
  );
  foreach ($sqlLists as $sqlList) {
    if (strpos($sqlList, $string) === 0) {
      $parser = new PHPSQLParser\PHPSQLParser($sqlList);
      $data = $parser->parsed['VALUES'];
      foreach ($data as $line) {
        if (strpos($line['data'][0]['base_expr'], 'modules/') === 1) {
          continue;
        }
        
        $name = str_replace("'", "", $line['data'][1]['base_expr']);
        $modules[$name] = new stdClass();
        foreach ($system_array as $key => $field) {
          $modules[$name]->{$field} = trim($line['data'][$key]['base_expr'], '\'"');
        }
        // PHPSQLParser messes up with negative (e.g. -1) schema and weight
        // values, and we dont need them anyway so just set as 0.
        $modules[$name]->schema_version = 0;
        $modules[$name]->weight = 0;

        // The PHPSQLParser messes up on quotes and splits the serialized info
        // field. We try and concatenate them here.
        $blob_source = '';
        $data_count = count($line['data']);
        //if ($data_count > 9) {
          for ($i = 8; $i < $data_count; $i++) {
            $blob_source .= $line['data'][$i]['base_expr'];
          }
        //}
        $blob_source = str_replace("''", "'", $blob_source);
        $blob_data = trim($blob_source, "'");
        // suppress unserialize errors because cant fix all of them.
        $modules[$name]->info = @unserialize($blob_data);
      }
    }
  }
  $projects = array();
  module_load_include('inc', 'update', 'update.compare');
  $projects = array();
  d2b_migrate_process_info_list($projects, $modules, 'module', TRUE);

  return $projects;
}

/**
 * Copy of _update_process_info_list().
 */
function d2b_migrate_process_info_list(&$projects, $list, $project_type, $status) {
  // todo: most of this code is verbatim and highly unnecessary. This function
  // is just needed to group modules by the "project" key in their info files.

  foreach ($list as $name_key => $file) {
    // Skip if the .info file is broken.
    if (empty($file->info)) {
      continue;
    }

    // Skip if it's a hidden module, theme, or layout.
    if (!empty($file->info['hidden'])) {
      continue;
    }

    // If the .info doesn't define the 'project', try to figure it out.
    if (!isset($file->info['project'])) {
      $file->info['project'] = update_get_project_name($file);
    }

    if (empty($file->info['project'])) {
      $project_name = $name_key;
    }
    else {
      $project_name = $file->info['project'];
    }

    if (!isset($file->info['_info_file_ctime'])) {
      $file->info['_info_file_ctime'] = 1525159574;
    }

    if (!isset($file->info['datestamp'])) {
      $file->info['datestamp'] = 0;
    }

    if ($project_name == 'drupal') {
      continue;
    }

    $project_display_type = $project_type;

    // The file info arrays for modules and themes have the human-readable name
    // of the project in a 'name' key, whereas layout templates in a 'title'
    // key.
    // - the 'name' key holds the machine name for layouts
    // - the 'title' key is not present in modules/themes
    // @todo: Fix all that in https://github.com/backdrop/backdrop-issues/issues/2548
    $includes_name = isset($file->info['title']) ? $file->info['title'] : $file->info['name'];
    if (!isset($projects[$project_name])) {
      // Only process this if we haven't done this project, since a single
      // project can have multiple modules, themes, or layout templates.
      $projects[$project_name] = array(
        'name' => $project_name,
        // Only save attributes from the .info file we care about, so we do not
        // bloat our RAM usage needlessly.
        'info' => update_filter_project_info($file->info),
        'datestamp' => $file->info['datestamp'],
        'includes' => array($file->name => $includes_name),
        'project_type' => $project_name == 'backdrop' ? 'core' : $project_display_type,
        'project_status' => $file->status,
        'sub_themes' => 'subtheme',
        'base_themes' => 'subtheme',
      );
    }
    elseif ($projects[$project_name]['project_type'] == $project_display_type) {
      // Only add the file we're processing to the 'includes' array for this
      // project if it is of the same type and status (which is encoded in the
      // $project_display_type). This prevents listing all the disabled modules
      // included within an enabled project, if we happen to be checking for
      // disabled modules too.
      $projects[$project_name]['includes'][$file->name] = $includes_name;
      $projects[$project_name]['info']['_info_file_ctime'] = max($projects[$project_name]['info']['_info_file_ctime'], $file->info['_info_file_ctime']);
      $projects[$project_name]['datestamp'] = max($projects[$project_name]['datestamp'], $file->info['datestamp']);
      if (!empty($sub_themes)) {
        $projects[$project_name]['sub_themes'] += $sub_themes;
      }
      if (!empty($base_themes)) {
        $projects[$project_name]['base_themes'] += $base_themes;
      }
    }
    elseif (empty($status)) {
      // If we have a project_name that matches, but the project_display_type
      // does not, it means we're processing a disabled module, theme, or layout
      // that belongs to a project that has some enabled code. In this case, we
      // add the disabled thing into a separate array for separate display.
      $projects[$project_name]['disabled'][$file->name] = $includes_name;
    }
  }
}

/**
 * Sets the database data source.
 */
function d2b_migrate_set_source($source, $type) {
  state_set('d2b_migrate_source', 
    array(
      'type' => $type,
      'source' => $source,
    )
  );
}

/**
 * Gets the database data source.
 */
function d2b_migrate_get_source() {
  return state_get('d2b_migrate_source', array());
}