Skip to content

feat: add inline timestamps STT output format #364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
brittain9 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add inline timestamps STT output format #364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion desktop/qml/Notepad.qml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,17 +95,21 @@ ColumnLayout {
currentIndex: {
if (_settings.stt_tts_text_format === Settings.TextFormatRaw) return 0
if (_settings.stt_tts_text_format === Settings.TextFormatSubRip) return 1
if (_settings.stt_tts_text_format === Settings.TextFormatInlineTimestamp) return 2
return 0
}
model: [
qsTr("Plain text"),
qsTr("SRT Subtitles")
qsTr("SRT Subtitles"),
qsTr("Inline timestamps")
]
onActivated: {
if (index === 0)
_settings.stt_tts_text_format = Settings.TextFormatRaw
else if (index === 1)
_settings.stt_tts_text_format = Settings.TextFormatSubRip
else if (index === 2)
_settings.stt_tts_text_format = Settings.TextFormatInlineTimestamp
}
}

Expand Down
120 changes: 120 additions & 0 deletions desktop/qml/SettingsSttPage.qml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,126 @@ ColumnLayout {
}
}

ColumnLayout {
Layout.fillWidth: true
spacing: 0
visible: _settings.stt_tts_text_format === Settings.TextFormatInlineTimestamp

SectionLabel {
text: qsTranslate("SettingsPage", "Inline timestamp settings")
}

TextFieldForm {
label.text: qsTranslate("SettingsPage", "Template")
compact: true
textField {
text: _settings.inline_timestamp_template
readOnly: true
color: palette.text
background: Rectangle {
color: palette.base
border.color: palette.mid
border.width: 1
radius: 2
}
}
button {
text: qsTranslate("SettingsPage", "Edit")
onClicked: inlineTimestampDialog.open()
}
}

SpinBoxForm {
label.text: qsTranslate("SettingsPage", "Timestamp interval")
toolTip: qsTranslate("SettingsPage", "Minimum seconds between timestamps.")
spinBox {
from: 5
to: 3600
stepSize: 5
value: _settings.inline_timestamp_min_interval
editable: true
textFromValue: function(value) { return value + " s" }
valueFromText: function(text) { return parseInt(text) }
onValueModified: {
_settings.inline_timestamp_min_interval = value
}
}
}
}

Dialog {
id: inlineTimestampDialog

modal: true
title: qsTranslate("SettingsPage", "Edit Timestamp Template")
parent: Overlay.overlay
anchors.centerIn: parent
width: Math.min(450, parent.width - 40)
closePolicy: Popup.CloseOnEscape | Popup.CloseOnPressOutside
standardButtons: Dialog.Ok | Dialog.Cancel

onAccepted: {
var newText = templateEditField.text.trim()
if (newText === "") newText = "[{mm}:{ss}] {text}"
_settings.inline_timestamp_template = newText
}

onOpened: {
templateEditField.text = _settings.inline_timestamp_template
templateEditField.forceActiveFocus()
templateEditField.selectAll()
}

ColumnLayout {
width: parent.width
spacing: 12

TextField {
id: templateEditField
Layout.fillWidth: true
placeholderText: "[{hh}:{mm}:{ss}] {text}"
selectByMouse: true
}

Label {
Layout.fillWidth: true
wrapMode: Text.Wrap
font.pixelSize: appWin.textFontSize * 0.9
color: palette.text
textFormat: Text.RichText
text: qsTranslate("SettingsPage", "Available tokens:<br><b>{hh}</b> (hours), <b>{mm}</b> (minutes), <b>{ss}</b> (seconds), <b>{text}</b> (transcribed text)")
}

Rectangle {
Layout.fillWidth: true
height: 1
color: palette.mid
opacity: 0.5
}

RowLayout {
Layout.fillWidth: true
spacing: 8

Label {
text: qsTranslate("SettingsPage", "Presets:")
color: palette.text
opacity: 0.8
}

Button {
text: qsTranslate("SettingsPage", "Standard")
onClicked: templateEditField.text = "[{hh}:{mm}:{ss}] {text}"
}

Button {
text: qsTranslate("SettingsPage", "Short")
onClicked: templateEditField.text = "[{mm}:{ss}] {text}"
}
}
}
}

BusyIndicator {
visible: app.busy && !sttEnginesBar.visible
running: visible
Expand Down
27 changes: 19 additions & 8 deletions src/april_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,8 @@ void april_engine::decode_speech(april_buf_t& buf, bool eof) {
#endif

bool prev_segment_finished =
m_config.text_format == text_format_t::subrip &&
(m_config.text_format == text_format_t::subrip ||
m_config.text_format == text_format_t::inline_timestamp) &&
((m_prev_segment_end_time && m_prev_segment_start_time) || eof);

if (prev_segment_finished) {
Expand Down Expand Up @@ -290,7 +291,8 @@ void april_engine::decode_speech(april_buf_t& buf, bool eof) {
m_prev_segment_end_time.reset();
}

if (eof && m_config.text_format == text_format_t::subrip) {
if (eof && (m_config.text_format == text_format_t::subrip ||
m_config.text_format == text_format_t::inline_timestamp)) {
ltrim(m_result_prev_segment);
rtrim(m_result_prev_segment);

Expand All @@ -304,12 +306,21 @@ void april_engine::decode_speech(april_buf_t& buf, bool eof) {
text_tools::restore_punctuation_in_segments(m_result_prev_segment,
m_segments);

text_tools::break_segments_to_multiline(
m_config.sub_config.min_line_length,
m_config.sub_config.max_line_length, m_segments);

set_intermediate_text(text_tools::segments_to_subrip_text(m_segments),
m_config.lang);
if (m_config.text_format == text_format_t::subrip) {
text_tools::break_segments_to_multiline(
m_config.sub_config.min_line_length,
m_config.sub_config.max_line_length, m_segments);

set_intermediate_text(text_tools::segments_to_subrip_text(m_segments),
m_config.lang);
} else if (m_config.text_format == text_format_t::inline_timestamp) {
set_intermediate_text(
text_tools::format_segments_inline(
m_segments, m_config.inline_timestamp_template,
m_config.inline_timestamp_min_interval,
m_last_inline_timestamp_t0),
m_config.lang);
}

m_result_prev_segment.clear();
m_result_size_consumed = 0;
Expand Down
26 changes: 18 additions & 8 deletions src/ds_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,8 @@ void ds_engine::decode_speech(const ds_buf_t& buf, bool eof) {

m_ds_api.STT_FeedAudioContent(m_ds_stream, buf.data(), buf.size());

if (eof && m_config.text_format == text_format_t::subrip) {
if (eof && (m_config.text_format == text_format_t::subrip ||
m_config.text_format == text_format_t::inline_timestamp)) {
auto* meta = m_ds_api.STT_FinishStreamWithMetadata(m_ds_stream, 1);

LOGD("speech decoded");
Expand All @@ -360,13 +361,22 @@ void ds_engine::decode_speech(const ds_buf_t& buf, bool eof) {
segments.second);
}

text_tools::break_segments_to_multiline(
m_config.sub_config.min_line_length,
m_config.sub_config.max_line_length, segments.second);

set_intermediate_text(
text_tools::segments_to_subrip_text(segments.second),
m_config.lang);
if (m_config.text_format == text_format_t::subrip) {
text_tools::break_segments_to_multiline(
m_config.sub_config.min_line_length,
m_config.sub_config.max_line_length, segments.second);

set_intermediate_text(
text_tools::segments_to_subrip_text(segments.second),
m_config.lang);
} else if (m_config.text_format == text_format_t::inline_timestamp) {
set_intermediate_text(
text_tools::format_segments_inline(
segments.second, m_config.inline_timestamp_template,
m_config.inline_timestamp_min_interval,
m_last_inline_timestamp_t0),
m_config.lang);
}
} else {
auto* cstr = eof ? m_ds_api.STT_FinishStream(m_ds_stream)
: m_ds_api.STT_IntermediateDecode(m_ds_stream);
Expand Down
2 changes: 1 addition & 1 deletion src/dsnote_app.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3968,7 +3968,7 @@ void dsnote_app::handle_translator_settings_changed() {
void dsnote_app::handle_note_changed() {
emit note_changed();

if (!settings::instance()->subtitles_support() || note().isEmpty()) {
if (!settings::instance()->subtitles_support()) {
settings::instance()->set_stt_tts_text_format(
settings::text_format_t::TextFormatRaw);
settings::instance()->set_mnt_text_format(
Expand Down
30 changes: 22 additions & 8 deletions src/fasterwhisper_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,9 @@ void fasterwhisper_engine::decode_speech(const whisper_buf_t& buf) {
std::ostringstream os;

bool subrip = m_config.text_format == text_format_t::subrip;
bool inline_ts = m_config.text_format == text_format_t::inline_timestamp;

std::vector<text_tools::segment_t> inline_segments;

auto i = 0;
for (auto& segment : segments) {
Expand All @@ -377,7 +380,7 @@ void fasterwhisper_engine::decode_speech(const whisper_buf_t& buf) {
LOGD("segment: " << text);
#endif

if (subrip) {
if (subrip || inline_ts) {
auto t0 = static_cast<size_t>(std::max(
0.0, segment.attr("start").cast<double>())) *
1000;
Expand All @@ -388,13 +391,17 @@ void fasterwhisper_engine::decode_speech(const whisper_buf_t& buf) {
t0 += m_segment_time_offset;
t1 += m_segment_time_offset;

text_tools::segment_t segment{i + 1 + m_segment_offset, t0,
t1, text};
text_tools::break_segment_to_multiline(
m_config.sub_config.min_line_length,
m_config.sub_config.max_line_length, segment);

text_tools::segment_to_subrip_text(segment, os);
text_tools::segment_t seg{i + 1 + m_segment_offset, t0,
t1, text};

if (subrip) {
text_tools::break_segment_to_multiline(
m_config.sub_config.min_line_length,
m_config.sub_config.max_line_length, seg);
text_tools::segment_to_subrip_text(seg, os);
} else {
inline_segments.push_back(std::move(seg));
}
} else {
if (i != 0) os << ' ';
os << std::move(text);
Expand All @@ -403,6 +410,13 @@ void fasterwhisper_engine::decode_speech(const whisper_buf_t& buf) {
++i;
}

if (inline_ts && !inline_segments.empty()) {
os << text_tools::format_segments_inline(
inline_segments, m_config.inline_timestamp_template,
m_config.inline_timestamp_min_interval,
m_last_inline_timestamp_t0);
}

m_segment_offset += i;

return std::pair<std::string, std::string>(os.str(),
Expand Down
24 changes: 24 additions & 0 deletions src/settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,30 @@ void settings::set_stt_tts_text_format(text_format_t value) {
}
}

QString settings::inline_timestamp_template() const {
return value(QStringLiteral("inline_timestamp_template"),
QStringLiteral("[{hh}:{mm}:{ss}] {text}"))
.toString();
}

void settings::set_inline_timestamp_template(const QString& value) {
if (inline_timestamp_template() != value) {
setValue(QStringLiteral("inline_timestamp_template"), value);
emit inline_timestamp_template_changed();
}
}

int settings::inline_timestamp_min_interval() const {
return value(QStringLiteral("inline_timestamp_min_interval"), 30).toInt();
}

void settings::set_inline_timestamp_min_interval(int value) {
if (inline_timestamp_min_interval() != value) {
setValue(QStringLiteral("inline_timestamp_min_interval"), value);
emit inline_timestamp_min_interval_changed();
}
}

unsigned int settings::hint_done_flags() const {
return value(QStringLiteral("hint_done_flags"), 0).toUInt();
}
Expand Down
15 changes: 14 additions & 1 deletion src/settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,12 @@ class settings : public QSettings, public singleton<settings> {
set_mnt_text_format NOTIFY mnt_text_format_changed)
Q_PROPERTY(text_format_t stt_tts_text_format READ stt_tts_text_format WRITE
set_stt_tts_text_format NOTIFY stt_tts_text_format_changed)
Q_PROPERTY(QString inline_timestamp_template READ inline_timestamp_template
WRITE set_inline_timestamp_template NOTIFY
inline_timestamp_template_changed)
Q_PROPERTY(int inline_timestamp_min_interval READ inline_timestamp_min_interval
WRITE set_inline_timestamp_min_interval NOTIFY
inline_timestamp_min_interval_changed)
Q_PROPERTY(int qt_style_idx READ qt_style_idx WRITE set_qt_style_idx NOTIFY
qt_style_changed)
Q_PROPERTY(QString qt_style_name READ qt_style_name WRITE set_qt_style_name
Expand Down Expand Up @@ -509,7 +515,8 @@ class settings : public QSettings, public singleton<settings> {
TextFormatRaw = 0,
TextFormatHtml = 1,
TextFormatMarkdown = 2,
TextFormatSubRip = 3
TextFormatSubRip = 3,
TextFormatInlineTimestamp = 4
};
Q_ENUM(text_format_t)

Expand Down Expand Up @@ -746,6 +753,10 @@ class settings : public QSettings, public singleton<settings> {
text_format_t mnt_text_format() const;
void set_stt_tts_text_format(text_format_t value);
text_format_t stt_tts_text_format() const;
QString inline_timestamp_template() const;
void set_inline_timestamp_template(const QString &value);
int inline_timestamp_min_interval() const;
void set_inline_timestamp_min_interval(int value);
QString default_tts_model_for_mnt_lang(const QString &lang);
void set_default_tts_model_for_mnt_lang(const QString &lang,
const QString &value);
Expand Down Expand Up @@ -1051,6 +1062,8 @@ class settings : public QSettings, public singleton<settings> {
void active_tts_for_out_mnt_ref_voice_changed();
void mnt_text_format_changed();
void stt_tts_text_format_changed();
void inline_timestamp_template_changed();
void inline_timestamp_min_interval_changed();
void addon_flags_changed();
void system_flags_changed();
void hint_done_flags_changed();
Expand Down
Loading