diff --git a/WikibaseImport.php b/WikibaseImport.php index 5837fee..977efe0 100644 --- a/WikibaseImport.php +++ b/WikibaseImport.php @@ -5,7 +5,7 @@ require_once __DIR__ . '/vendor/autoload.php'; } - wfLoadExtension( 'WikibaseImport' ); + wfLoadExtension( 'WikibaseImport', __DIR__ . '/extension.json' ); } else { die( 'WikibaseImport requires MediaWiki 1.25+' ); } diff --git a/maintenance/importEntities.php b/maintenance/importEntities.php index 0d1ff9c..f66e15b 100644 --- a/maintenance/importEntities.php +++ b/maintenance/importEntities.php @@ -1,18 +1,6 @@ addOptions(); - } - - private function addOptions() { - $this->addOption( 'file', 'File with list of entity ids to import', false, true ); - $this->addOption( 'entity', 'ID of entity to import', false, true ); - $this->addOption( 'query', 'Import items with property and entity id value', false, true ); - $this->addOption( 'range', 'Range of ids to import', false, true ); - $this->addOption( 'all-properties', 'Import all properties', false, false ); - } - - public function execute() { - $this->logger = LoggerFactory::newLogger( 'wikibase-import', $this->mQuiet ); - - $importOptions = $this->extractOptions(); - - $entityIdListBuilderFactory = $this->newEntityIdListBuilderFactory(); - - foreach ( $this->getValidOptions() as $option ) { - if ( $importOptions->hasOption( $option ) ) { - $entityIdListBuilder = $entityIdListBuilderFactory->newEntityIdListBuilder( - $option - ); - - if ( $option === 'all-properties' ) { - $input = 'all-properties'; - } else { - $input = $importOptions->getOption( $option ); - } - - break; - } - } - - if ( !isset( $entityIdListBuilder ) ) { - $this->logger->error( 'ERROR: No valid import option was provided' ); - - return; - } else { - try { - $ids = $entityIdListBuilder->getEntityIds( $input ); - - $entityImporter = $this->newEntityImporter(); - $entityImporter->importEntities( $ids ); - } catch ( Exception $ex ) { - $this->logger->error( $ex->getMessage() ); - } - - $this->logger->info( 'Done' ); - } - } - - private function extractOptions() { - $options = []; - - foreach ( $this->getValidOptions() as $optionName ) { - $options[$optionName] = $this->getOption( $optionName ); - } - - if ( empty( $options ) ) { - $this->maybeHelp( true ); - } - - return new ImportOptions( $options ); - } - - private function getValidOptions() { - return [ 'entity', 'file', 'all-properties', 'query', 'range' ]; - } - - private function newEntityIdListBuilderFactory() { - $queryRunner = new QueryRunner( - new QueryBuilder( $this->getConfig()->get( 'WBImportQueryPrefixes' ) ), - new QueryExecuter( $this->getConfig()->get( 'WBImportQueryUrl' ) ) - ); - - return new EntityIdListBuilderFactory( - WikibaseRepo::getDefaultInstance()->getEntityIdParser(), - new PropertyIdLister(), - $queryRunner, - $this->getConfig()->get( 'WBImportSourceApi' ) - ); - } - - private function newEntityImporter() { - $entityImporterFactory = new EntityImporterFactory( - WikibaseRepo::getDefaultInstance()->getStore()->getEntityStore(), - wfGetLB(), - $this->logger, - $this->getConfig()->get( 'WBImportSourceApi' ) - ); - - return $entityImporterFactory->newEntityImporter(); - } - -} - -$maintClass = "Wikibase\Import\Maintenance\ImportEntities"; +$maintClass = ImportEntities::class; require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/src/EntityId/EntityIdListBuilderFactory.php b/src/EntityId/EntityIdListBuilderFactory.php index 22055d0..2ea1b34 100644 --- a/src/EntityId/EntityIdListBuilderFactory.php +++ b/src/EntityId/EntityIdListBuilderFactory.php @@ -2,6 +2,8 @@ namespace Wikibase\Import\EntityId; +use Asparagus\QueryBuilder; +use Asparagus\QueryExecuter; use InvalidArgumentException; use Wikibase\DataModel\Entity\EntityIdParser; use Wikibase\Import\PropertyIdLister; @@ -20,9 +22,14 @@ class EntityIdListBuilderFactory { private $propertyIdLister; /** - * @var QueryRunner + * @var array */ - private $queryRunner; + private $queryPrefixes; + + /** + * @var string + */ + private $queryUrl; /** * @var string @@ -32,18 +39,21 @@ class EntityIdListBuilderFactory { /** * @param EntityIdParser $idParser * @param PropertyIdLister $propertyIdLister - * @param QueryRunner $queryRunner + * @param array $queryPrefixes + * @param string $queryUrl * @param string $apiUrl */ public function __construct( EntityIdParser $idParser, PropertyIdLister $propertyIdLister, - QueryRunner $queryRunner, + array $queryPrefixes, + $queryUrl, $apiUrl ) { $this->idParser = $idParser; $this->propertyIdLister = $propertyIdLister; - $this->queryRunner = $queryRunner; + $this->queryPrefixes = $queryPrefixes; + $this->queryUrl = $queryUrl; $this->apiUrl = $apiUrl; } @@ -71,10 +81,7 @@ public function newEntityIdListBuilder( $mode ) { } private function newAllPropertiesEntityIdListBuilder() { - return new AllPropertiesEntityIdListBuilder( - $this->propertyIdLister, - $this->apiUrl - ); + return new AllPropertiesEntityIdListBuilder( $this->propertyIdLister, $this->apiUrl ); } private function newFileEntityIdListBuilder() { @@ -86,10 +93,12 @@ private function newIndividualEntityIdListBuilder() { } private function newQueryEntityIdListBuilder() { - return new QueryEntityIdListBuilder( - $this->idParser, - $this->queryRunner - ); + return new QueryEntityIdListBuilder( $this->idParser, $this->newQueryRunner() ); + } + + private function newQueryRunner() { + return new QueryRunner( new QueryBuilder( $this->queryPrefixes ), + new QueryExecuter( $this->queryUrl ) ); } private function newRangeEntityIdListBuilder() { diff --git a/src/EntityImporter.php b/src/EntityImporter.php index 715086f..62f6b25 100644 --- a/src/EntityImporter.php +++ b/src/EntityImporter.php @@ -11,6 +11,7 @@ use Wikibase\DataModel\Snak\PropertyValueSnak; use Wikibase\DataModel\Statement\StatementList; use Wikibase\Import\Store\ImportedEntityMappingStore; +use Wikibase\Lib\Store\EntityStore; use Wikibase\Repo\Store\WikiPageEntityStore; class EntityImporter { @@ -39,7 +40,7 @@ public function __construct( StatementsImporter $statementsImporter, BadgeItemUpdater $badgeItemUpdater, ApiEntityLookup $apiEntityLookup, - WikiPageEntityStore $entityStore, + EntityStore $entityStore, ImportedEntityMappingStore $entityMappingStore, StatementsCountLookup $statementsCountLookup, LoggerInterface $logger diff --git a/src/LoggerFactory.php b/src/LoggerFactory.php index 3d18009..4984f46 100644 --- a/src/LoggerFactory.php +++ b/src/LoggerFactory.php @@ -25,11 +25,14 @@ public static function newLogger( $loggerName, $quietMode ) { throw new InvalidArgumentException( '$quietMode must be boolean' ); } - $formatter = new LineFormatter( "[%datetime%]: %message%\n" ); + // unused + $dateTimeFormatter = new LineFormatter( "[%datetime%]: %message%\n" ); if ( $quietMode === true ) { $handler = new NullHandler(); } else { + $formatter = new LineFormatter( "%message%\n"); + $handler = new StreamHandler( 'php://stdout' ); $handler->setFormatter( $formatter ); } diff --git a/src/Maintenance/ImportEntities.php b/src/Maintenance/ImportEntities.php new file mode 100644 index 0000000..276d38e --- /dev/null +++ b/src/Maintenance/ImportEntities.php @@ -0,0 +1,176 @@ +addOptions(); + } + + private function addOptions() { + $this->addOption( 'file', 'File with list of entity ids to import', false, true ); + $this->addOption( 'entity', 'ID of entity to import', false, true ); + $this->addOption( 'query', 'Import items with property and entity id value', false, true ); + $this->addOption( 'range', 'Range of ids to import', false, true ); + $this->addOption( 'all-properties', 'Import all properties', false, false ); + } + + public function setServices( + EntityIdListBuilderFactory $entityIdListBuilderFactory, + EntityImporterFactory $entityImporterFactory, + LoggerInterface $logger + ) { + $this->entityIdListBuilderFactory = $entityIdListBuilderFactory; + $this->entityImporterFactory = $entityImporterFactory; + $this->logger = $logger; + } + + public function initServices() { + // needed for EntityImporter + if ( !isset( $this->logger ) ) { + $this->logger = LoggerFactory::newLogger( 'wikibase-import', $this->mQuiet ); + } + + if ( !isset( $this->entityIdListBuilderFactory ) ) { + $this->entityIdListBuilderFactory = $this->newEntityIdListBuilderFactory(); + } + + if ( !isset( $this->entityImporterFactory ) ) { + $this->entityImporterFactory = $this->newEntityImporterFactory( $this->logger ); + } + } + + public function execute() { + $this->initServices(); + + try { + $this->doImport( $this->extractOptions() ); + } catch ( Exception $ex ) { + $this->error( $ex->getMessage() ); + } + + $this->logger->info( 'Done' ); + } + + /** + * @param ImportOptions $importOptions + */ + private function doImport( ImportOptions $importOptions ) { + foreach ( $importOptions as $importMode => $input ) { + $this->logger->info( "Importing $importMode\n" ); + + $entityIdListBuilder = $this->entityIdListBuilderFactory->newEntityIdListBuilder( + $importMode + ); + + $ids = $entityIdListBuilder->getEntityIds( $input ); + $this->entityImporterFactory->newEntityImporter()->importEntities( $ids ); + } + } + + /** + * @inheritdoc + */ + protected function error( $err, $die = 0 ) { + $err = "\033[31mERROR:\033[0m $err"; + + $this->logger->error( $err ); + $this->maybeHelp( true ); + } + + /** + * @return array + */ + private function getValidOptions() { + return [ 'entity', 'file', 'all-properties', 'query', 'range' ]; + } + + /** + * @return ImportOptions + * @throws RuntimeException + */ + private function extractOptions() { + $options = []; + + foreach ( $this->getValidOptions() as $optionName ) { + if ( $this->hasOption( $optionName ) ) { + $options[$optionName] = $this->getOptionValue( $optionName ); + } + } + + if ( empty( $options ) ) { + throw new RuntimeException( 'No valid import mode option provided' ); + } + + return new ImportOptions( $options ); + } + + /** + * @param string $optionName + * @return mixed + */ + private function getOptionValue( $optionName ) { + if ( $optionName === 'all-properties' ) { + return 'all-properties'; + } else { + return $this->getOption( $optionName ); + } + } + + /** + * @return EntityIdListBuilderFactory + */ + private function newEntityIdListBuilderFactory() { + $entityIdListBuilderFactory = new EntityIdListBuilderFactory( + WikibaseRepo::getDefaultInstance()->getEntityIdParser(), + new PropertyIdLister(), + $this->getConfig()->get( 'WBImportQueryPrefixes' ), + $this->getConfig()->get( 'WBImportQueryUrl' ), + $this->getConfig()->get( 'WBImportSourceApi' ) + ); + + return $entityIdListBuilderFactory; + } + + /** + * @return EntityImporterFactory + */ + private function newEntityImporterFactory( LoggerInterface $logger ) { + $entityImporterFactory = new EntityImporterFactory( + WikibaseRepo::getDefaultInstance()->getStore()->getEntityStore(), + MediaWikiServices::getInstance()->getDBLoadBalancer(), + $logger, + $this->getConfig()->get( 'WBImportSourceApi' ) + ); + + return $entityImporterFactory; + } +} \ No newline at end of file diff --git a/src/Console/ImportOptions.php b/src/Maintenance/ImportOptions.php similarity index 71% rename from src/Console/ImportOptions.php rename to src/Maintenance/ImportOptions.php index 376ca87..be4866f 100644 --- a/src/Console/ImportOptions.php +++ b/src/Maintenance/ImportOptions.php @@ -1,10 +1,11 @@ options[$name]; } + /** + * @return ArrayIterator + */ + public function getIterator() { + return new ArrayIterator( $this->options ); + } + } diff --git a/tests/integration/Maintenance/ImportEntitiesTest.php b/tests/integration/Maintenance/ImportEntitiesTest.php new file mode 100644 index 0000000..88eeef4 --- /dev/null +++ b/tests/integration/Maintenance/ImportEntitiesTest.php @@ -0,0 +1,97 @@ +initServices(); + + // sanity check + $this->assertTrue( true ); + } + + public function testExecute() { + $name = 'extensions/Wikidata/extensions/Import/maintenance/importEntities.php'; + + $options = [ + 'entity' => 'Q14085872', + 'memory-limit' => 'max' + ]; + + $args = []; + + $importEntities = new ImportEntities(); + $importEntities->setServices( + $this->getEntityIdListBuilderFactory(), + $this->getEntityImporterFactory(), + $this->getLogger() + ); + + $importEntities->loadParamsAndArgs( $name, $options, $args ); + $importEntities->execute(); + + $this->assertTrue( true ); + } + + private function getEntityIdListBuilderFactory() { + $entityIdListBuilder = $this->getMockBuilder( EntityIdListBuilder::class ) + ->getMock(); + + $entityIdListBuilder->expects( $this->any() ) + ->method( 'getEntityIds' ) + ->will( $this->returnValue( [ 'Q147' ] ) ); + + // TODO: EntityIdListBuilderFactory really doesn't necessarily need to be mocked + $entityIdListBuilderFactory = $this->getMockBuilder( EntityIdListBuilderFactory::class ) + ->disableOriginalConstructor() + ->getMock(); + + $entityIdListBuilderFactory->expects( $this->once() ) + ->method( 'newEntityIdListBuilder' ) + ->will( $this->returnValue( $entityIdListBuilder ) ); + + return $entityIdListBuilderFactory; + } + + private function getEntityImporterFactory() { + $entityImporter = $this->getMockBuilder( EntityImporter::class ) + ->disableOriginalConstructor() + ->getMock(); + + $entityImporter->expects( $this->any() ) + ->method( 'importEntities' ) + ->will( $this->returnValue( null ) ); + + $entityImporterFactory = $this->getMockBuilder( EntityImporterFactory::class ) + ->disableOriginalConstructor() + ->getMock(); + + $entityImporterFactory->expects( $this->once() ) + ->method( 'newEntityImporter' ) + ->will( $this->returnValue( $entityImporter ) ); + + return $entityImporterFactory; + } + + private function getLogger() { + $logger = new Logger( 'wikibase-import' ); + $logger->pushHandler( new NullHandler() ); + + return $logger; + } + +}