load-sentences.php

#!/usr/bin/env php
<?php
declare(strict_types=1);
require_once __DIR__.'/vendor/autoload.php';
ini_set('memory_limit', '24G');

$db_file = $argv[1] ?? 'sentences.sqlite';
$db = new \TDC\PDO\SQLite($db_file);

$db->exec("PRAGMA journal_mode = delete");
$db->exec("PRAGMA page_size = 65536");
$db->exec("VACUUM");

$db->exec("PRAGMA auto_vacuum = INCREMENTAL");
$db->exec("PRAGMA case_sensitive_like = ON");
$db->exec("PRAGMA foreign_keys = OFF");
$db->exec("PRAGMA ignore_check_constraints = ON");
$db->exec("PRAGMA journal_mode = MEMORY");
$db->exec("PRAGMA locking_mode = EXCLUSIVE");
$db->exec("PRAGMA synchronous = OFF");
$db->exec("PRAGMA threads = 4");
$db->exec("PRAGMA trusted_schema = OFF");

$db->exec("CREATE TABLE sents (
	s_id INTEGER NOT NULL,
	s_tokens INTEGER NOT NULL,
	s_text TEXT NOT NULL,
	PRIMARY KEY (s_id)
) WITHOUT ROWID");

$db->beginTransaction();
$ins = $db->prepare("INSERT INTO sents (s_id, s_tokens, s_text) VALUES (?, ?, ?)");

$uniq = [];

$in_par = false;
$tokens = 0;
$sent = '';
$cohort = '';
$i = 0;

function handle_cohort() {
	global $tokens, $sent, $cohort;

	if (!empty($cohort)) {
		if (!preg_match('~^"<(.+?)>"~u', $cohort, $m)) {
			echo "BAD COHORT: $cohort\n";
			$cohort = '';
			return;
		}
		$word = $m[1];
		if (strpos($cohort, ' Prop') === false && strpos($cohort, ' ?') === false) {
			$word = mb_strtolower($word);
		}
		$sent .= $word;
		$sent .= ' ';
		++$tokens;
	}

	$cohort = '';
}

function save_sent() {
	global $db, $ins, $uniq, $tokens, $sent, $i;

	handle_cohort();

	$sent = trim($sent);
	if (empty($sent)) {
		return;
	}

	$hash = sha1($sent);
	if (!array_key_exists($hash, $uniq)) {
		$uniq[$hash] = ++$i;
		$ins->execute([$i, $tokens, $sent]);
		if ($i % 10000 === 0) {
			echo "$i\r";
			$db->commit();
			$db->beginTransaction();
		}
	}

	$sent = '';
	$tokens = 0;
}

while ($line = fgets(STDIN)) {
	if (preg_match('~^<s(\d+)>~', $line, $m)) {
		$in_par = intval($m[1]);
	}
	else if (preg_match('~^</s(\d+)>~', $line, $m)) {
		if (intval($m[1]) !== $in_par) {
			echo "MISMATCH: {$m[1]} != {$in_par}\n";
			$in_par = false;
			$tokens = 0;
			$sent = '';
			continue;
		}
		save_sent();
		$in_par = false;
		$tokens = 0;
		$sent = '';
	}
	else if ($in_par) {
		if (preg_match('~^"<~u', $line)) {
			handle_cohort();
			++$tokens;
			$cohort = $line;
		}
		else if (preg_match('~^\s+"~u', $line)) {
			$cohort .= $line;
		}
		else {
			// Empty line means sentence break
			save_sent();
		}
	}
}
save_sent();
echo "$i\n";

$db->commit();
$db->exec("PRAGMA ignore_check_constraints = OFF");
$db->exec("PRAGMA locking_mode = NORMAL");