impl stable sentence

Author: rerender2021

.gitignore

@@ -7,4 +7,5 @@ dist
 
 asr-server
 nlp-server
-/config.json
+/config.json
+subtitle

package-lock.json

@@ -34,6 +34,7 @@
   "dependencies": {
     "ave-react": "^0.1.4",
     "axios": "^1.3.2",
-    "debounce": "^1.2.1"
+    "debounce": "^1.2.1",
+    "sentence-splitter": "^4.2.0"
   }
 }

package.json

@@ -1,11 +1,11 @@
-import React, { useCallback, useEffect, useMemo } from "react";
-import { AveRenderer, Grid, Window, getAppContext, IIconResource, IWindowComponentProps, Button, CheckBox, ICheckBoxComponentProps } from "ave-react";
+import React, { useCallback, useEffect, useMemo, useState } from "react";
+import { AveRenderer, Grid, Window, getAppContext, IIconResource, IWindowComponentProps, Button, CheckBox, ICheckBoxComponentProps, ScrollBar, Label, IScrollBarComponentProps } from "ave-react";
 import { App, ThemePredefined_Dark, CheckValue } from "ave-ui";
 import { VoskAsrEngine } from "./asr";
 import { HelsinkiNlpEngine } from "./nlp";
 import { containerLayout, controlLayout } from "./layout";
 import { iconResource } from "./resource";
-import { onMeasure, onTranslate, shadowRelated } from "./shadow";
+import { logger, onMeasure, onTranslate, shadowRelated } from "./shadow";
 import { getAsrConfig, getNlpConfig } from "./config";
 
 function onInit(app: App) {
@@ -20,6 +20,14 @@ function initTheme() {
 	themeDark.SetStyle(themeImage, 0);
 }
 
+enum ButtonText {
+	Measure = "设置字幕区",
+	Recognize = "语音识别",
+	SetTopMost = "字幕置顶",
+	SubtitleEn = "英文字幕",
+	SubtitleZh = "中文字幕",
+}
+
 export function Echo() {
 	const asrEngine = useMemo(
 		() =>
@@ -57,29 +65,39 @@ export function Echo() {
 	}, []);
 
 	const onSetRecognize = useCallback<ICheckBoxComponentProps["onCheck"]>((sender) => {
+		shadowRelated.subtitleQueue = [];
+
 		let shouldRecognize = false;
 
 		const checkValue = sender.GetValue();
 		if (checkValue === CheckValue.Unchecked) {
 			shouldRecognize = false;
+			logger.end();
 		} else if (checkValue === CheckValue.Checked) {
 			shouldRecognize = true;
+			logger.start();
 		}
 
 		shadowRelated.shouldRecognize = shouldRecognize;
 	}, []);
 
-	const onSetPunct = useCallback<ICheckBoxComponentProps["onCheck"]>((sender) => {
-		let shouldResotrePunct = false;
-
+	const onSetDisplaySubtitle = useCallback<ICheckBoxComponentProps["onCheck"]>((sender) => {
 		const checkValue = sender.GetValue();
-		if (checkValue === CheckValue.Unchecked) {
-			shouldResotrePunct = false;
-		} else if (checkValue === CheckValue.Checked) {
-			shouldResotrePunct = true;
+		const text = sender.GetText();
+		const isChecked = checkValue === CheckValue.Checked;
+		if (text === ButtonText.SubtitleEn) {
+			shadowRelated.subtitleConfig.en = isChecked;
+		} else if (text === ButtonText.SubtitleZh) {
+			shadowRelated.subtitleConfig.zh = isChecked;
 		}
+		shadowRelated.onUpdateTranslationConfig();
+	}, []);
 
-		shadowRelated.shouldResotrePunct = shouldResotrePunct;
+	const [fontSize, setFontSize] = useState(24);
+	const onSetFontSize = useCallback<IScrollBarComponentProps["onScrolling"]>((sender) => {
+		const fontSize = sender.GetValue();
+		shadowRelated.onUpdateFontSize(fontSize);
+		setFontSize(fontSize);
 	}, []);
 
 	useEffect(() => {
@@ -94,16 +112,28 @@ export function Echo() {
 			<Grid style={{ layout: containerLayout }}>
 				<Grid style={{ area: containerLayout.areas.control, layout: controlLayout }}>
 					<Grid style={{ area: controlLayout.areas.measure }}>
-						<Button text="设置字幕区" iconInfo={{ name: "measure", size: 16 }} onClick={onMeasure}></Button>
+						<Button text={ButtonText.Measure} iconInfo={{ name: "measure", size: 16 }} onClick={onMeasure}></Button>
 					</Grid>
 					<Grid style={{ area: controlLayout.areas.recognize }}>
-						<CheckBox text="语音识别" value={CheckValue.Unchecked} onCheck={onSetRecognize}></CheckBox>
-					</Grid>
-					<Grid style={{ area: controlLayout.areas.punct }}>
-						<CheckBox text="标点恢复" value={CheckValue.Unchecked} onCheck={onSetPunct}></CheckBox>
+						<CheckBox text={ButtonText.Recognize} value={CheckValue.Unchecked} onCheck={onSetRecognize}></CheckBox>
 					</Grid>
 					<Grid style={{ area: controlLayout.areas.topmost }}>
-						<CheckBox text="字幕置顶" value={CheckValue.Checked} onCheck={onSetTopMost}></CheckBox>
+						<CheckBox text={ButtonText.SetTopMost} value={CheckValue.Checked} onCheck={onSetTopMost}></CheckBox>
+					</Grid>
+					<Grid style={{ area: controlLayout.areas.en }}>
+						<CheckBox text={ButtonText.SubtitleEn} value={CheckValue.Checked} onCheck={onSetDisplaySubtitle}></CheckBox>
+					</Grid>
+					<Grid style={{ area: controlLayout.areas.zh }}>
+						<CheckBox text={ButtonText.SubtitleZh} value={CheckValue.Checked} onCheck={onSetDisplaySubtitle}></CheckBox>
+					</Grid>
+					<Grid style={{ area: controlLayout.areas.fontSizeLabel }}>
+						<Label text="字体大小"></Label>
+					</Grid>
+					<Grid style={{ area: controlLayout.areas.fontSize, margin: "10dpx 0 10dpx 0" }}>
+						<ScrollBar min={10} max={50} value={24} /** default value */ onScrolling={onSetFontSize}></ScrollBar>
+					</Grid>
+					<Grid style={{ area: controlLayout.areas.fontSizeValue }}>
+						<Label text={`${fontSize}`}></Label>
 					</Grid>
 				</Grid>
 			</Grid>

src/app.tsx

@@ -2,15 +2,18 @@ import axios from "axios";
 import path from "path";
 import fs from "fs";
 import childProcess from "child_process";
-import { IAsrEngine, IAsrEngineOptions, IAsrResult } from "./base";
-import { shadowRelated } from "../shadow";
+import { IAsrEngine, IAsrEngineOptions, ISentence } from "./base";
+import { TranslationSession } from "./postasr";
+import { emptySentence } from "../shadow";
 
 export class VoskAsrEngine implements IAsrEngine {
 	private options: IAsrEngineOptions;
 	private asr: childProcess.ChildProcessWithoutNullStreams;
+	private sessionList: Array<TranslationSession>;
 
 	constructor(options: IAsrEngineOptions) {
 		this.options = options;
+		this.sessionList = [];
 	}
 
 	async init() {
@@ -53,29 +56,55 @@ export class VoskAsrEngine implements IAsrEngine {
 		}
 	}
 
-	async recognize(): Promise<IAsrResult> {
-		// const base64 = buffer.toString("base64");
-		let text = "";
+	getCurrentSession() {
+		return this.sessionList[this.sessionList.length - 1] || null;
+	}
+
+	async addAsrTextToSession(asrText: string, time: number) {
+		if (!asrText) {
+			const session = this.getCurrentSession();
+			if (session) {
+				// handle only one speech case
+				await session.flushSpeech();
+
+				// create new session if the last is not empty
+				if (!session.isEmpty()) {
+					const newSession = new TranslationSession();
+					this.sessionList.push(newSession);
+				}
+			} else {
+				// it's the first session
+				this.sessionList.push(new TranslationSession());
+			}
+			return;
+		}
+
+		const session = this.getCurrentSession();
+		if (session) {
+			await session.addAsrText(asrText, time);
+		}
+	}
+
+	async recognize(): Promise<ISentence> {
+		let sentence: ISentence = emptySentence;
 		try {
 			const timeout = this.options?.timeout || 3000;
 			const response = await axios.post("http://localhost:8200/asr", {}, { timeout });
-			const data = JSON.parse(response.data.result);
-			console.log(data);
+			console.log(response.data);
 
-			text = data.partial || data.text || "";
+			const data = JSON.parse(response?.data?.result || "{}");
 
-			if (text && shadowRelated.shouldResotrePunct) {
-				const withPunctResponse = await axios.post("http://localhost:8200/punct", { text }, { timeout });
-				if (withPunctResponse.data.text) {
-					text = withPunctResponse.data.text;
-					console.log({ text });
-				}
-			}
+			const asrText = data.partial || "";
+			await this.addAsrTextToSession(asrText, Date.now());
+
+			const session = this.getCurrentSession();
+			sentence = session.getCurrentSentence();
+			sentence.asr = asrText;
 		} catch (error) {
 			console.log(`asr failed: ${error.message}`);
 			this.options?.onError(error.message);
 		} finally {
-			return { text };
+			return sentence;
 		}
 	}
 }

src/asr/asr.ts

@@ -1,5 +1,14 @@
-export interface IAsrResult {
+export interface ISentence {
 	text: string;
+	time: number;
+	sessionId: number;
+	speechId: number;
+	asr?: string
+}
+
+export interface IAsrText {
+	text: string;
+	time: number;
 }
 
 export interface IAsrEngineOptions {
@@ -13,7 +22,7 @@ export interface IAsrEngineConstructor {
 }
 
 export interface IAsrEngine {
-	recognize(): Promise<IAsrResult>;
+	recognize(): Promise<ISentence>;
 	init(): void;
 	destroy(): void;
 }

src/asr/base.ts

@@ -1 +1,2 @@
-export * from "./asr";
+export * from "./asr";
+export * from "./base";