From bd691e6b02e4a208676ed7abde0cefa4767643ab Mon Sep 17 00:00:00 2001 From: aude Date: Sat, 21 Jan 2017 16:50:25 -0500 Subject: [PATCH 1/4] Split code in importEntities execute method also improve error handling/formatting, and move QueryRunner obj construction to factory. --- WikibaseImport.php | 2 +- maintenance/importEntities.php | 119 +++++++++++------- src/EntityId/EntityIdListBuilderFactory.php | 38 +++--- src/LoggerFactory.php | 5 +- .../Maintenance/ImportEntitiesTest.php | 54 ++++++++ 5 files changed, 155 insertions(+), 63 deletions(-) create mode 100644 tests/integration/Maintenance/ImportEntitiesTest.php diff --git a/WikibaseImport.php b/WikibaseImport.php index 5837fee..977efe0 100644 --- a/WikibaseImport.php +++ b/WikibaseImport.php @@ -5,7 +5,7 @@ require_once __DIR__ . '/vendor/autoload.php'; } - wfLoadExtension( 'WikibaseImport' ); + wfLoadExtension( 'WikibaseImport', __DIR__ . '/extension.json' ); } else { die( 'WikibaseImport requires MediaWiki 1.25+' ); } diff --git a/maintenance/importEntities.php b/maintenance/importEntities.php index 0d1ff9c..342c5d4 100644 --- a/maintenance/importEntities.php +++ b/maintenance/importEntities.php @@ -2,15 +2,15 @@ namespace Wikibase\Import\Maintenance; -use Asparagus\QueryBuilder; -use Asparagus\QueryExecuter; use Exception; +use MediaWiki\MediaWikiServices; use Psr\Log\LoggerInterface; +use RuntimeException; use Wikibase\Import\Console\ImportOptions; +use Wikibase\Import\EntityId\EntityIdListBuilder; use Wikibase\Import\EntityId\EntityIdListBuilderFactory; use Wikibase\Import\EntityImporterFactory; use Wikibase\Import\LoggerFactory; -use Wikibase\Import\QueryRunner; use Wikibase\Import\PropertyIdLister; use Wikibase\Repo\WikibaseRepo; @@ -28,6 +28,11 @@ class ImportEntities extends \Maintenance { */ private $logger; + /** + * @var ImportOptions + */ + private $importOptions; + public function __construct() { parent::__construct(); @@ -44,45 +49,46 @@ private function addOptions() { public function execute() { $this->logger = LoggerFactory::newLogger( 'wikibase-import', $this->mQuiet ); + $this->importOptions = $this->extractOptions(); - $importOptions = $this->extractOptions(); + try { + $importMode = $this->getImportMode(); + $entityIdListBuilder = $this->newEntityIdListBuilder( $importMode ); - $entityIdListBuilderFactory = $this->newEntityIdListBuilderFactory(); + $input = $this->getInputForMode( $importMode ); + $ids = $entityIdListBuilder->getEntityIds( $input ); - foreach ( $this->getValidOptions() as $option ) { - if ( $importOptions->hasOption( $option ) ) { - $entityIdListBuilder = $entityIdListBuilderFactory->newEntityIdListBuilder( - $option - ); - - if ( $option === 'all-properties' ) { - $input = 'all-properties'; - } else { - $input = $importOptions->getOption( $option ); - } - - break; - } + $entityImporter = $this->newEntityImporter(); + $entityImporter->importEntities( $ids ); + } + catch ( Exception $ex ) { + $this->error( $ex->getMessage() ); } - if ( !isset( $entityIdListBuilder ) ) { - $this->logger->error( 'ERROR: No valid import option was provided' ); + $this->logger->info( 'Done' ); + } - return; - } else { - try { - $ids = $entityIdListBuilder->getEntityIds( $input ); + /** + * @inheritdoc + */ + protected function error( $err, $die = 0 ) { + $err = "\033[31mERROR:\033[0m $err"; - $entityImporter = $this->newEntityImporter(); - $entityImporter->importEntities( $ids ); - } catch ( Exception $ex ) { - $this->logger->error( $ex->getMessage() ); - } + $this->logger->error( $err ); + $this->maybeHelp( true ); + } - $this->logger->info( 'Done' ); - } + /** + * @return array + */ + private function getValidOptions() { + return [ 'entity', 'file', 'all-properties', 'query', 'range' ]; } + /** + * @return ImportOptions + * @throws RuntimeException + */ private function extractOptions() { $options = []; @@ -91,42 +97,65 @@ private function extractOptions() { } if ( empty( $options ) ) { - $this->maybeHelp( true ); + throw new RuntimeException( 'No valid import mode option provided' ); } return new ImportOptions( $options ); } - private function getValidOptions() { - return [ 'entity', 'file', 'all-properties', 'query', 'range' ]; + /** + * @return string + * @throws RuntimeException + */ + private function getImportMode() { + foreach ( $this->getValidOptions() as $option ) { + if ( $this->importOptions->hasOption( $option ) ) { + return $option; + } + } + + throw new RuntimeException( 'No valid import option was provided' ); } - private function newEntityIdListBuilderFactory() { - $queryRunner = new QueryRunner( - new QueryBuilder( $this->getConfig()->get( 'WBImportQueryPrefixes' ) ), - new QueryExecuter( $this->getConfig()->get( 'WBImportQueryUrl' ) ) - ); + /** + * @param string $mode + * @return mixed + */ + private function getInputForMode( $mode ) { + if ( $mode === 'all-properties' ) { + return 'all-properties'; + } else { + return $this->importOptions->getOption( $mode ); + } + } - return new EntityIdListBuilderFactory( + /** + * @param string $importMode + * @return EntityIdListBuilder + */ + private function newEntityIdListBuilder( $importMode ) { + $entityIdListBuilderFactory = new EntityIdListBuilderFactory( WikibaseRepo::getDefaultInstance()->getEntityIdParser(), new PropertyIdLister(), - $queryRunner, + $this->getConfig()->get( 'WBImportQueryPrefixes' ), + $this->getConfig()->get( 'WBImportQueryUrl' ), $this->getConfig()->get( 'WBImportSourceApi' ) ); + + return $entityIdListBuilderFactory->newEntityIdListBuilder( $importMode ); } private function newEntityImporter() { $entityImporterFactory = new EntityImporterFactory( WikibaseRepo::getDefaultInstance()->getStore()->getEntityStore(), - wfGetLB(), + MediaWikiServices::getInstance()->getDBLoadBalancer(), $this->logger, $this->getConfig()->get( 'WBImportSourceApi' ) ); return $entityImporterFactory->newEntityImporter(); } - } -$maintClass = "Wikibase\Import\Maintenance\ImportEntities"; +$maintClass = ImportEntities::class; require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/src/EntityId/EntityIdListBuilderFactory.php b/src/EntityId/EntityIdListBuilderFactory.php index 22055d0..2295baf 100644 --- a/src/EntityId/EntityIdListBuilderFactory.php +++ b/src/EntityId/EntityIdListBuilderFactory.php @@ -2,6 +2,8 @@ namespace Wikibase\Import\EntityId; +use Asparagus\QueryBuilder; +use Asparagus\QueryExecuter; use InvalidArgumentException; use Wikibase\DataModel\Entity\EntityIdParser; use Wikibase\Import\PropertyIdLister; @@ -20,9 +22,14 @@ class EntityIdListBuilderFactory { private $propertyIdLister; /** - * @var QueryRunner + * @var array */ - private $queryRunner; + private $queryPrefixes; + + /** + * @var string + */ + private $queryUrl; /** * @var string @@ -32,18 +39,18 @@ class EntityIdListBuilderFactory { /** * @param EntityIdParser $idParser * @param PropertyIdLister $propertyIdLister - * @param QueryRunner $queryRunner + * @param array $queryPrefixes + * @param string $queryUrl * @param string $apiUrl */ public function __construct( - EntityIdParser $idParser, - PropertyIdLister $propertyIdLister, - QueryRunner $queryRunner, - $apiUrl + EntityIdParser $idParser, PropertyIdLister $propertyIdLister, array $queryPrefixes, + $queryUrl, $apiUrl ) { $this->idParser = $idParser; $this->propertyIdLister = $propertyIdLister; - $this->queryRunner = $queryRunner; + $this->queryPrefixes = $queryPrefixes; + $this->queryUrl = $queryUrl; $this->apiUrl = $apiUrl; } @@ -71,10 +78,7 @@ public function newEntityIdListBuilder( $mode ) { } private function newAllPropertiesEntityIdListBuilder() { - return new AllPropertiesEntityIdListBuilder( - $this->propertyIdLister, - $this->apiUrl - ); + return new AllPropertiesEntityIdListBuilder( $this->propertyIdLister, $this->apiUrl ); } private function newFileEntityIdListBuilder() { @@ -86,10 +90,12 @@ private function newIndividualEntityIdListBuilder() { } private function newQueryEntityIdListBuilder() { - return new QueryEntityIdListBuilder( - $this->idParser, - $this->queryRunner - ); + return new QueryEntityIdListBuilder( $this->idParser, $this->newQueryRunner() ); + } + + private function newQueryRunner() { + return new QueryRunner( new QueryBuilder( $this->queryPrefixes ), + new QueryExecuter( $this->queryUrl ) ); } private function newRangeEntityIdListBuilder() { diff --git a/src/LoggerFactory.php b/src/LoggerFactory.php index 3d18009..4984f46 100644 --- a/src/LoggerFactory.php +++ b/src/LoggerFactory.php @@ -25,11 +25,14 @@ public static function newLogger( $loggerName, $quietMode ) { throw new InvalidArgumentException( '$quietMode must be boolean' ); } - $formatter = new LineFormatter( "[%datetime%]: %message%\n" ); + // unused + $dateTimeFormatter = new LineFormatter( "[%datetime%]: %message%\n" ); if ( $quietMode === true ) { $handler = new NullHandler(); } else { + $formatter = new LineFormatter( "%message%\n"); + $handler = new StreamHandler( 'php://stdout' ); $handler->setFormatter( $formatter ); } diff --git a/tests/integration/Maintenance/ImportEntitiesTest.php b/tests/integration/Maintenance/ImportEntitiesTest.php new file mode 100644 index 0000000..42914e9 --- /dev/null +++ b/tests/integration/Maintenance/ImportEntitiesTest.php @@ -0,0 +1,54 @@ +getApiEntityLookup(); + $item = $apiEntityLookup->getEntity( new ItemId( 'Q60' ) ); + + $this->assertInstanceOf( 'Wikibase\DataModel\Entity\Item', $item ); + $this->assertSame( + 'New York City', + $item->getFingerprint()->getLabel( 'en' )->getText(), + 'English label is New York City' + ); + + $statements = $item->getStatements()->getByPropertyId( new PropertyId( 'P17' ) ); + + foreach ( $statements as $statement ) { + $mainSnakValue = $statement->getMainSnak()->getDataValue(); + $this->assertEquals( 'Q30', $mainSnakValue->getEntityId()->getSerialization() ); + } + } + + private function getApiEntityLookup() { + $entityImporterFactory = new EntityImporterFactory( + WikibaseRepo::getDefaultInstance()->getStore()->getEntityStore(), + wfGetLB(), + $this->newLogger(), + 'https://www.wikidata.org/w/api.php' + ); + + return $entityImporterFactory->getApiEntityLookup(); + } + + private function newLogger() { + $logger = new Logger( 'wikibase-import' ); + $logger->pushHandler( new NullHandler() ); + + return $logger; + } + +} From b1563d6bd0a53c59c81aa9e5c67c1a893859a6da Mon Sep 17 00:00:00 2001 From: aude Date: Sun, 22 Jan 2017 14:43:12 -0500 Subject: [PATCH 2/4] Allow using multiple command options And clean up the command options code a bit --- maintenance/importEntities.php | 40 +++++++++++++--------------------- src/Console/ImportOptions.php | 12 ++++++++-- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/maintenance/importEntities.php b/maintenance/importEntities.php index 342c5d4..873da1d 100644 --- a/maintenance/importEntities.php +++ b/maintenance/importEntities.php @@ -52,14 +52,16 @@ public function execute() { $this->importOptions = $this->extractOptions(); try { - $importMode = $this->getImportMode(); - $entityIdListBuilder = $this->newEntityIdListBuilder( $importMode ); + foreach ( $this->importOptions as $importMode => $input ) { + $this->output( "Importing $importMode\n" ); - $input = $this->getInputForMode( $importMode ); - $ids = $entityIdListBuilder->getEntityIds( $input ); + $entityIdListBuilder = $this->newEntityIdListBuilder( $importMode ); - $entityImporter = $this->newEntityImporter(); - $entityImporter->importEntities( $ids ); + $ids = $entityIdListBuilder->getEntityIds( $input ); + + $entityImporter = $this->newEntityImporter(); + $entityImporter->importEntities( $ids ); + } } catch ( Exception $ex ) { $this->error( $ex->getMessage() ); @@ -93,7 +95,9 @@ private function extractOptions() { $options = []; foreach ( $this->getValidOptions() as $optionName ) { - $options[$optionName] = $this->getOption( $optionName ); + if ( $this->hasOption( $optionName ) ) { + $options[$optionName] = $this->getOptionValue( $optionName ); + } } if ( empty( $options ) ) { @@ -104,28 +108,14 @@ private function extractOptions() { } /** - * @return string - * @throws RuntimeException - */ - private function getImportMode() { - foreach ( $this->getValidOptions() as $option ) { - if ( $this->importOptions->hasOption( $option ) ) { - return $option; - } - } - - throw new RuntimeException( 'No valid import option was provided' ); - } - - /** - * @param string $mode + * @param string $optionName * @return mixed */ - private function getInputForMode( $mode ) { - if ( $mode === 'all-properties' ) { + private function getOptionValue( $optionName ) { + if ( $optionName === 'all-properties' ) { return 'all-properties'; } else { - return $this->importOptions->getOption( $mode ); + return $this->getOption( $optionName ); } } diff --git a/src/Console/ImportOptions.php b/src/Console/ImportOptions.php index 376ca87..fe7d5bf 100644 --- a/src/Console/ImportOptions.php +++ b/src/Console/ImportOptions.php @@ -2,9 +2,10 @@ namespace Wikibase\Import\Console; -use Maintenance; +use ArrayIterator; +use IteratorAggregate; -class ImportOptions { +class ImportOptions implements IteratorAggregate { /** * @var array @@ -44,4 +45,11 @@ public function getOption( $name ) { return $this->options[$name]; } + /** + * @return ArrayIterator + */ + public function getIterator() { + return new ArrayIterator( $this->options ); + } + } From 47418deda5805e03ef149e0879885de64cf6333c Mon Sep 17 00:00:00 2001 From: aude Date: Sun, 22 Jan 2017 19:26:33 -0500 Subject: [PATCH 3/4] Move ImportOptions to Maintenance namespace --- maintenance/importEntities.php | 2 +- .../ImportOptions.php | 2 +- .../Maintenance/ImportEntitiesTest.php | 54 ------------------- 3 files changed, 2 insertions(+), 56 deletions(-) rename src/{Console => Maintenance}/ImportOptions.php (95%) delete mode 100644 tests/integration/Maintenance/ImportEntitiesTest.php diff --git a/maintenance/importEntities.php b/maintenance/importEntities.php index 873da1d..d4a25dd 100644 --- a/maintenance/importEntities.php +++ b/maintenance/importEntities.php @@ -6,11 +6,11 @@ use MediaWiki\MediaWikiServices; use Psr\Log\LoggerInterface; use RuntimeException; -use Wikibase\Import\Console\ImportOptions; use Wikibase\Import\EntityId\EntityIdListBuilder; use Wikibase\Import\EntityId\EntityIdListBuilderFactory; use Wikibase\Import\EntityImporterFactory; use Wikibase\Import\LoggerFactory; +use Wikibase\Import\Maintenance\ImportOptions; use Wikibase\Import\PropertyIdLister; use Wikibase\Repo\WikibaseRepo; diff --git a/src/Console/ImportOptions.php b/src/Maintenance/ImportOptions.php similarity index 95% rename from src/Console/ImportOptions.php rename to src/Maintenance/ImportOptions.php index fe7d5bf..be4866f 100644 --- a/src/Console/ImportOptions.php +++ b/src/Maintenance/ImportOptions.php @@ -1,6 +1,6 @@ getApiEntityLookup(); - $item = $apiEntityLookup->getEntity( new ItemId( 'Q60' ) ); - - $this->assertInstanceOf( 'Wikibase\DataModel\Entity\Item', $item ); - $this->assertSame( - 'New York City', - $item->getFingerprint()->getLabel( 'en' )->getText(), - 'English label is New York City' - ); - - $statements = $item->getStatements()->getByPropertyId( new PropertyId( 'P17' ) ); - - foreach ( $statements as $statement ) { - $mainSnakValue = $statement->getMainSnak()->getDataValue(); - $this->assertEquals( 'Q30', $mainSnakValue->getEntityId()->getSerialization() ); - } - } - - private function getApiEntityLookup() { - $entityImporterFactory = new EntityImporterFactory( - WikibaseRepo::getDefaultInstance()->getStore()->getEntityStore(), - wfGetLB(), - $this->newLogger(), - 'https://www.wikidata.org/w/api.php' - ); - - return $entityImporterFactory->getApiEntityLookup(); - } - - private function newLogger() { - $logger = new Logger( 'wikibase-import' ); - $logger->pushHandler( new NullHandler() ); - - return $logger; - } - -} From bbc1b0f1a3caef5365ef5b11eea227c61a42a8d9 Mon Sep 17 00:00:00 2001 From: aude Date: Sun, 22 Jan 2017 20:48:54 -0500 Subject: [PATCH 4/4] Adding tests for ImportEntities [WIP] --- maintenance/importEntities.php | 141 +------------- src/EntityId/EntityIdListBuilderFactory.php | 7 +- src/EntityImporter.php | 3 +- src/Maintenance/ImportEntities.php | 176 ++++++++++++++++++ .../Maintenance/ImportEntitiesTest.php | 97 ++++++++++ 5 files changed, 282 insertions(+), 142 deletions(-) create mode 100644 src/Maintenance/ImportEntities.php create mode 100644 tests/integration/Maintenance/ImportEntitiesTest.php diff --git a/maintenance/importEntities.php b/maintenance/importEntities.php index d4a25dd..f66e15b 100644 --- a/maintenance/importEntities.php +++ b/maintenance/importEntities.php @@ -1,18 +1,6 @@ addOptions(); - } - - private function addOptions() { - $this->addOption( 'file', 'File with list of entity ids to import', false, true ); - $this->addOption( 'entity', 'ID of entity to import', false, true ); - $this->addOption( 'query', 'Import items with property and entity id value', false, true ); - $this->addOption( 'range', 'Range of ids to import', false, true ); - $this->addOption( 'all-properties', 'Import all properties', false, false ); - } - - public function execute() { - $this->logger = LoggerFactory::newLogger( 'wikibase-import', $this->mQuiet ); - $this->importOptions = $this->extractOptions(); - - try { - foreach ( $this->importOptions as $importMode => $input ) { - $this->output( "Importing $importMode\n" ); - - $entityIdListBuilder = $this->newEntityIdListBuilder( $importMode ); - - $ids = $entityIdListBuilder->getEntityIds( $input ); - - $entityImporter = $this->newEntityImporter(); - $entityImporter->importEntities( $ids ); - } - } - catch ( Exception $ex ) { - $this->error( $ex->getMessage() ); - } - - $this->logger->info( 'Done' ); - } - - /** - * @inheritdoc - */ - protected function error( $err, $die = 0 ) { - $err = "\033[31mERROR:\033[0m $err"; - - $this->logger->error( $err ); - $this->maybeHelp( true ); - } - - /** - * @return array - */ - private function getValidOptions() { - return [ 'entity', 'file', 'all-properties', 'query', 'range' ]; - } - - /** - * @return ImportOptions - * @throws RuntimeException - */ - private function extractOptions() { - $options = []; - - foreach ( $this->getValidOptions() as $optionName ) { - if ( $this->hasOption( $optionName ) ) { - $options[$optionName] = $this->getOptionValue( $optionName ); - } - } - - if ( empty( $options ) ) { - throw new RuntimeException( 'No valid import mode option provided' ); - } - - return new ImportOptions( $options ); - } - - /** - * @param string $optionName - * @return mixed - */ - private function getOptionValue( $optionName ) { - if ( $optionName === 'all-properties' ) { - return 'all-properties'; - } else { - return $this->getOption( $optionName ); - } - } - - /** - * @param string $importMode - * @return EntityIdListBuilder - */ - private function newEntityIdListBuilder( $importMode ) { - $entityIdListBuilderFactory = new EntityIdListBuilderFactory( - WikibaseRepo::getDefaultInstance()->getEntityIdParser(), - new PropertyIdLister(), - $this->getConfig()->get( 'WBImportQueryPrefixes' ), - $this->getConfig()->get( 'WBImportQueryUrl' ), - $this->getConfig()->get( 'WBImportSourceApi' ) - ); - - return $entityIdListBuilderFactory->newEntityIdListBuilder( $importMode ); - } - - private function newEntityImporter() { - $entityImporterFactory = new EntityImporterFactory( - WikibaseRepo::getDefaultInstance()->getStore()->getEntityStore(), - MediaWikiServices::getInstance()->getDBLoadBalancer(), - $this->logger, - $this->getConfig()->get( 'WBImportSourceApi' ) - ); - - return $entityImporterFactory->newEntityImporter(); - } -} +require_once __DIR__ . '/../src/Maintenance/ImportEntities.php'; $maintClass = ImportEntities::class; require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/src/EntityId/EntityIdListBuilderFactory.php b/src/EntityId/EntityIdListBuilderFactory.php index 2295baf..2ea1b34 100644 --- a/src/EntityId/EntityIdListBuilderFactory.php +++ b/src/EntityId/EntityIdListBuilderFactory.php @@ -44,8 +44,11 @@ class EntityIdListBuilderFactory { * @param string $apiUrl */ public function __construct( - EntityIdParser $idParser, PropertyIdLister $propertyIdLister, array $queryPrefixes, - $queryUrl, $apiUrl + EntityIdParser $idParser, + PropertyIdLister $propertyIdLister, + array $queryPrefixes, + $queryUrl, + $apiUrl ) { $this->idParser = $idParser; $this->propertyIdLister = $propertyIdLister; diff --git a/src/EntityImporter.php b/src/EntityImporter.php index 715086f..62f6b25 100644 --- a/src/EntityImporter.php +++ b/src/EntityImporter.php @@ -11,6 +11,7 @@ use Wikibase\DataModel\Snak\PropertyValueSnak; use Wikibase\DataModel\Statement\StatementList; use Wikibase\Import\Store\ImportedEntityMappingStore; +use Wikibase\Lib\Store\EntityStore; use Wikibase\Repo\Store\WikiPageEntityStore; class EntityImporter { @@ -39,7 +40,7 @@ public function __construct( StatementsImporter $statementsImporter, BadgeItemUpdater $badgeItemUpdater, ApiEntityLookup $apiEntityLookup, - WikiPageEntityStore $entityStore, + EntityStore $entityStore, ImportedEntityMappingStore $entityMappingStore, StatementsCountLookup $statementsCountLookup, LoggerInterface $logger diff --git a/src/Maintenance/ImportEntities.php b/src/Maintenance/ImportEntities.php new file mode 100644 index 0000000..276d38e --- /dev/null +++ b/src/Maintenance/ImportEntities.php @@ -0,0 +1,176 @@ +addOptions(); + } + + private function addOptions() { + $this->addOption( 'file', 'File with list of entity ids to import', false, true ); + $this->addOption( 'entity', 'ID of entity to import', false, true ); + $this->addOption( 'query', 'Import items with property and entity id value', false, true ); + $this->addOption( 'range', 'Range of ids to import', false, true ); + $this->addOption( 'all-properties', 'Import all properties', false, false ); + } + + public function setServices( + EntityIdListBuilderFactory $entityIdListBuilderFactory, + EntityImporterFactory $entityImporterFactory, + LoggerInterface $logger + ) { + $this->entityIdListBuilderFactory = $entityIdListBuilderFactory; + $this->entityImporterFactory = $entityImporterFactory; + $this->logger = $logger; + } + + public function initServices() { + // needed for EntityImporter + if ( !isset( $this->logger ) ) { + $this->logger = LoggerFactory::newLogger( 'wikibase-import', $this->mQuiet ); + } + + if ( !isset( $this->entityIdListBuilderFactory ) ) { + $this->entityIdListBuilderFactory = $this->newEntityIdListBuilderFactory(); + } + + if ( !isset( $this->entityImporterFactory ) ) { + $this->entityImporterFactory = $this->newEntityImporterFactory( $this->logger ); + } + } + + public function execute() { + $this->initServices(); + + try { + $this->doImport( $this->extractOptions() ); + } catch ( Exception $ex ) { + $this->error( $ex->getMessage() ); + } + + $this->logger->info( 'Done' ); + } + + /** + * @param ImportOptions $importOptions + */ + private function doImport( ImportOptions $importOptions ) { + foreach ( $importOptions as $importMode => $input ) { + $this->logger->info( "Importing $importMode\n" ); + + $entityIdListBuilder = $this->entityIdListBuilderFactory->newEntityIdListBuilder( + $importMode + ); + + $ids = $entityIdListBuilder->getEntityIds( $input ); + $this->entityImporterFactory->newEntityImporter()->importEntities( $ids ); + } + } + + /** + * @inheritdoc + */ + protected function error( $err, $die = 0 ) { + $err = "\033[31mERROR:\033[0m $err"; + + $this->logger->error( $err ); + $this->maybeHelp( true ); + } + + /** + * @return array + */ + private function getValidOptions() { + return [ 'entity', 'file', 'all-properties', 'query', 'range' ]; + } + + /** + * @return ImportOptions + * @throws RuntimeException + */ + private function extractOptions() { + $options = []; + + foreach ( $this->getValidOptions() as $optionName ) { + if ( $this->hasOption( $optionName ) ) { + $options[$optionName] = $this->getOptionValue( $optionName ); + } + } + + if ( empty( $options ) ) { + throw new RuntimeException( 'No valid import mode option provided' ); + } + + return new ImportOptions( $options ); + } + + /** + * @param string $optionName + * @return mixed + */ + private function getOptionValue( $optionName ) { + if ( $optionName === 'all-properties' ) { + return 'all-properties'; + } else { + return $this->getOption( $optionName ); + } + } + + /** + * @return EntityIdListBuilderFactory + */ + private function newEntityIdListBuilderFactory() { + $entityIdListBuilderFactory = new EntityIdListBuilderFactory( + WikibaseRepo::getDefaultInstance()->getEntityIdParser(), + new PropertyIdLister(), + $this->getConfig()->get( 'WBImportQueryPrefixes' ), + $this->getConfig()->get( 'WBImportQueryUrl' ), + $this->getConfig()->get( 'WBImportSourceApi' ) + ); + + return $entityIdListBuilderFactory; + } + + /** + * @return EntityImporterFactory + */ + private function newEntityImporterFactory( LoggerInterface $logger ) { + $entityImporterFactory = new EntityImporterFactory( + WikibaseRepo::getDefaultInstance()->getStore()->getEntityStore(), + MediaWikiServices::getInstance()->getDBLoadBalancer(), + $logger, + $this->getConfig()->get( 'WBImportSourceApi' ) + ); + + return $entityImporterFactory; + } +} \ No newline at end of file diff --git a/tests/integration/Maintenance/ImportEntitiesTest.php b/tests/integration/Maintenance/ImportEntitiesTest.php new file mode 100644 index 0000000..88eeef4 --- /dev/null +++ b/tests/integration/Maintenance/ImportEntitiesTest.php @@ -0,0 +1,97 @@ +initServices(); + + // sanity check + $this->assertTrue( true ); + } + + public function testExecute() { + $name = 'extensions/Wikidata/extensions/Import/maintenance/importEntities.php'; + + $options = [ + 'entity' => 'Q14085872', + 'memory-limit' => 'max' + ]; + + $args = []; + + $importEntities = new ImportEntities(); + $importEntities->setServices( + $this->getEntityIdListBuilderFactory(), + $this->getEntityImporterFactory(), + $this->getLogger() + ); + + $importEntities->loadParamsAndArgs( $name, $options, $args ); + $importEntities->execute(); + + $this->assertTrue( true ); + } + + private function getEntityIdListBuilderFactory() { + $entityIdListBuilder = $this->getMockBuilder( EntityIdListBuilder::class ) + ->getMock(); + + $entityIdListBuilder->expects( $this->any() ) + ->method( 'getEntityIds' ) + ->will( $this->returnValue( [ 'Q147' ] ) ); + + // TODO: EntityIdListBuilderFactory really doesn't necessarily need to be mocked + $entityIdListBuilderFactory = $this->getMockBuilder( EntityIdListBuilderFactory::class ) + ->disableOriginalConstructor() + ->getMock(); + + $entityIdListBuilderFactory->expects( $this->once() ) + ->method( 'newEntityIdListBuilder' ) + ->will( $this->returnValue( $entityIdListBuilder ) ); + + return $entityIdListBuilderFactory; + } + + private function getEntityImporterFactory() { + $entityImporter = $this->getMockBuilder( EntityImporter::class ) + ->disableOriginalConstructor() + ->getMock(); + + $entityImporter->expects( $this->any() ) + ->method( 'importEntities' ) + ->will( $this->returnValue( null ) ); + + $entityImporterFactory = $this->getMockBuilder( EntityImporterFactory::class ) + ->disableOriginalConstructor() + ->getMock(); + + $entityImporterFactory->expects( $this->once() ) + ->method( 'newEntityImporter' ) + ->will( $this->returnValue( $entityImporter ) ); + + return $entityImporterFactory; + } + + private function getLogger() { + $logger = new Logger( 'wikibase-import' ); + $logger->pushHandler( new NullHandler() ); + + return $logger; + } + +}