feat: add expressive offline Russian book TTS

This commit is contained in:
Курнат Андрей
2026-07-21 21:28:48 +03:00
parent 28312e3fbc
commit cf82ab61eb
47 changed files with 127843 additions and 124 deletions
+2
View File
@@ -61,3 +61,5 @@
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain
app/src/main/assets/tts/**/*.onnx filter=lfs diff=lfs merge=lfs -text
app/src/main/assets/tts/**/dict.tsv filter=lfs diff=lfs merge=lfs -text
+8
View File
@@ -18,6 +18,9 @@ android {
vectorDrawables {
useSupportLibrary = true
}
ndk {
abiFilters += "arm64-v8a"
}
}
buildTypes {
@@ -53,6 +56,10 @@ android {
excludes += "/META-INF/{AL2.0,LGPL2.1}"
}
}
androidResources {
noCompress += "onnx"
}
}
dependencies {
@@ -67,6 +74,7 @@ dependencies {
implementation("androidx.constraintlayout:constraintlayout:2.2.1")
implementation("androidx.swiperefreshlayout:swiperefreshlayout:1.1.0")
implementation("androidx.webkit:webkit:1.12.1")
implementation("com.microsoft.onnxruntime:onnxruntime-android:1.27.0")
testImplementation("junit:junit:4.13.2")
}
+4
View File
@@ -5,3 +5,7 @@
# Keep the manifest-declared Argus package installer callback constructable by Android.
-keep class xyz.kusoft.argusupdater.ArgusPackageInstallerStatusReceiver { public <init>(); }
# ONNX Runtime's native JNI layer resolves these Java classes and members by
# their original names when converting native OrtValues to Java OnnxValues.
-keep class ai.onnxruntime.** { *; }
+85 -3
View File
@@ -301,6 +301,68 @@
}
}
function visibleTextSlice(element, width, height) {
const doc = element && element.ownerDocument;
if (!doc || typeof doc.createTreeWalker !== 'function' || typeof doc.createRange !== 'function') return '';
const nodeFilter = (doc.defaultView && doc.defaultView.NodeFilter) || global.NodeFilter;
if (!nodeFilter) return '';
const walker = doc.createTreeWalker(element, nodeFilter.SHOW_TEXT);
const range = doc.createRange();
let raw = '';
let visibleStart = Number.POSITIVE_INFINITY;
let visibleEnd = -1;
let node;
while ((node = walker.nextNode())) {
const value = String(node.nodeValue || '');
const base = raw.length;
raw += value;
const words = /\S+/g;
let match;
while ((match = words.exec(value))) {
range.setStart(node, match.index);
range.setEnd(node, match.index + match[0].length);
const visible = Array.from(range.getClientRects()).some(function (rect) {
return rect.width > 0 && rect.height > 0 &&
rect.right > 0 && rect.bottom > 0 && rect.left < width && rect.top < height;
});
if (!visible) continue;
visibleStart = Math.min(visibleStart, base + match.index);
visibleEnd = Math.max(visibleEnd, base + match.index + match[0].length);
}
}
range.detach();
if (!Number.isFinite(visibleStart) || visibleEnd <= visibleStart) return '';
let start = visibleStart;
const before = raw.slice(0, start).replace(/\s+$/g, '');
if (before && !/[.!?…][\"'»”)]?$/.test(before)) {
const remainder = raw.slice(start, visibleEnd);
const nextSentence = /[.!?…]+[\"'»”)]?\s+/.exec(remainder);
if (nextSentence) {
const candidate = start + nextSentence.index + nextSentence[0].length;
if (candidate < visibleEnd) start = candidate;
}
}
return raw.slice(start, visibleEnd).replace(/\s+/g, ' ').trim();
}
function rangeSpeechText(range) {
if (!range || typeof range.cloneContents !== 'function') return '';
const doc = range.startContainer && range.startContainer.ownerDocument;
if (!doc) return String(range.toString() || '').trim();
const container = doc.createElement('div');
container.appendChild(range.cloneContents());
const blockSelector = 'p,h1,h2,h3,h4,h5,h6,pre,blockquote,li,td,th';
Array.from(container.querySelectorAll(blockSelector)).forEach(function (element) {
if (element.querySelector(blockSelector)) return;
element.appendChild(doc.createTextNode('\n'));
});
return String(container.textContent || '')
.replace(/[\t\f\v ]+/g, ' ')
.replace(/ *\n+ */g, '\n')
.trim();
}
function visibleSpeechText(documents, maxCharacters) {
const limit = clamp(Number(maxCharacters) || 3500, 200, 3500);
const blocks = [];
@@ -318,13 +380,32 @@
rect.right > 0 && rect.bottom > 0 && rect.left < width && rect.top < height;
});
if (!visible) return;
const text = String(element.innerText || element.textContent || '')
.replace(/\s+/g, ' ')
.trim();
const text = visibleTextSlice(element, width, height);
if (!text || seen.has(text)) return;
seen.add(text);
blocks.push(text);
});
if (blocks.length || typeof doc.elementFromPoint !== 'function') return;
// EPUB pagination can place the current column in a clipped iframe where
// getClientRects() does not intersect the document's nominal viewport.
// Sampling the actual hit-tested viewport reliably finds the paragraphs
// the reader is displaying without reading hidden chapters or controls.
const blockSelector = 'p,h1,h2,h3,h4,h5,h6,pre,blockquote,li,td,th';
const xSamples = [0.12, 0.5, 0.88];
const ySamples = [0.08, 0.22, 0.38, 0.54, 0.70, 0.86, 0.96];
ySamples.forEach(function (yRatio) {
xSamples.forEach(function (xRatio) {
const hit = doc.elementFromPoint(width * xRatio, height * yRatio);
const element = hit && hit.closest ? hit.closest(blockSelector) : null;
if (!element) return;
const text = visibleTextSlice(element, width, height) ||
String(element.innerText || element.textContent || '').replace(/\s+/g, ' ').trim();
if (!text || seen.has(text)) return;
seen.add(text);
blocks.push(text);
});
});
});
let text = blocks.join('\n').trim();
if (text.length <= limit) return text;
@@ -363,5 +444,6 @@
Internal.installReaderFontFaces = installReaderFontFaces;
Internal.waitForPreferredFont = waitForPreferredFont;
Internal.validExternalUrl = validExternalUrl;
Internal.rangeSpeechText = rangeSpeechText;
Internal.visibleSpeechText = visibleSpeechText;
})(window);
@@ -566,10 +566,95 @@
const contents = this.rendition && this.rendition.getContents
? this.rendition.getContents()
: [];
const location = this.rendition && typeof this.rendition.currentLocation === 'function'
? this.rendition.currentLocation()
: this.currentLocation;
const startCfi = location && location.start && location.start.cfi;
const endCfi = location && location.end && location.end.cfi;
const limit = Internal.clamp(Number(maxCharacters) || 3500, 200, 3500);
if (startCfi && endCfi) {
for (let index = 0; index < contents.length; index += 1) {
const item = contents[index];
if (!item || typeof item.range !== 'function' || !item.document) continue;
try {
const start = item.range(startCfi);
const end = item.range(endCfi);
if (!start || !end) continue;
const pageRange = item.document.createRange();
pageRange.setStart(start.startContainer, start.startOffset);
pageRange.setEnd(end.endContainer, end.endOffset);
const pageText = Internal.rangeSpeechText(pageRange);
let text = pageText;
let extendedCharacters = 0;
if (text && !/[.!?…][\"'»”)]?\s*$/.test(text)) {
const continuationRange = item.document.createRange();
continuationRange.setStart(end.endContainer, end.endOffset);
continuationRange.selectNodeContents(item.document.body);
continuationRange.setStart(end.endContainer, end.endOffset);
const continuation = Internal.rangeSpeechText(continuationRange).trimStart();
const sentenceEnd = /^[\s\S]*?[.!?…]+[\"'»”)]?/.exec(continuation);
if (sentenceEnd) {
const extension = sentenceEnd[0].trimEnd();
text = (text + ' ' + extension).trim();
extendedCharacters = extension.length;
}
}
const startNode = start.startContainer;
const startElement = startNode && (startNode.nodeType === 3 ? startNode.parentElement : startNode);
const startBlock = startElement && startElement.closest
? startElement.closest('p,h1,h2,h3,h4,h5,h6,pre,blockquote,li,td,th')
: null;
let textBeforeStart = '';
if (startBlock) {
const beforeRange = item.document.createRange();
beforeRange.selectNodeContents(startBlock);
beforeRange.setEnd(start.startContainer, start.startOffset);
textBeforeStart = String(beforeRange.toString() || '').trimEnd();
} else if (startNode && startNode.nodeType === 3 && start.startOffset > 0) {
textBeforeStart = String(startNode.nodeValue || '').slice(0, start.startOffset).trimEnd();
}
const startsInsideSentence = Boolean(textBeforeStart) &&
!/[.!?…][\"'»”)]?\s*$/.test(textBeforeStart);
if (startsInsideSentence) {
const boundary = /[.!?…]+[\"'»”)]?\s+/.exec(text);
if (boundary && boundary.index + boundary[0].length < text.length) {
text = text.slice(boundary.index + boundary[0].length).trim();
}
}
if (text.length > limit) {
const candidate = text.slice(0, limit + 1);
const sentenceEnd = Math.max(
candidate.lastIndexOf('. '),
candidate.lastIndexOf('! '),
candidate.lastIndexOf('? '),
candidate.lastIndexOf('… ')
);
const wordEnd = candidate.lastIndexOf(' ', limit);
const cut = sentenceEnd >= Math.floor(limit * .55)
? sentenceEnd + 1
: (wordEnd > 0 ? wordEnd : limit);
text = candidate.slice(0, cut).trim();
}
if (text) {
return {
text: text,
locator: this.currentLocator(),
canAdvance: this.currentProgress < .9999,
source: 'epub-cfi-page',
pageCharacters: pageText.length,
extendedCharacters: extendedCharacters
};
}
} catch (_) {}
}
}
return {
text: Internal.visibleSpeechText(contents.map(function (item) { return item.document; }), maxCharacters),
locator: this.currentLocator(),
canAdvance: this.currentProgress < .9999
canAdvance: this.currentProgress < .9999,
source: 'visible-fallback',
pageCharacters: 0,
extendedCharacters: 0
};
}
+4 -3
View File
@@ -222,9 +222,10 @@
},
getSpeechPageJson: function (maxCharacters) {
try {
return JSON.stringify(callEngine('speechPage', [maxCharacters], {
text: '', locator: null, canAdvance: false
}));
const page = state.engine && typeof state.engine.speechPage === 'function'
? state.engine.speechPage(maxCharacters)
: { text: '', locator: null, canAdvance: false };
return JSON.stringify(page);
} catch (error) {
Internal.reportError(error, 'api.getSpeechPageJson', true);
return JSON.stringify({ text: '', locator: null, canAdvance: false });
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Binary file not shown.
Binary file not shown.
1 version https://git-lfs.github.com/spec/v1
2 oid sha256:db5eaadd86c5fde2b68b371bafbf07d2102de3bef861fed3468456a839190378
3 size 96070019
File diff suppressed because it is too large Load Diff
Binary file not shown.
File diff suppressed because it is too large Load Diff
@@ -520,7 +520,7 @@ class ReaderActivity : AppCompatActivity() {
binding.readerVoiceButton.contentDescription = if (active) {
"Приостановить озвучивание"
} else {
"Слушать книгу. Удерживайте для настройки скорости и интонации"
"Слушать книгу. Удерживайте для настройки голоса, темпа и пауз"
}
}
@@ -401,7 +401,10 @@ class ReaderWebController(
data class ReaderSpeechPage(
val text: String,
val locator: String?,
val canAdvance: Boolean
val canAdvance: Boolean,
val source: String,
val pageCharacters: Int,
val extendedCharacters: Int
) {
companion object {
internal fun fromJson(json: JSONObject): ReaderSpeechPage? {
@@ -415,7 +418,10 @@ data class ReaderSpeechPage(
return ReaderSpeechPage(
text = text,
locator = locator,
canAdvance = json.optBoolean("canAdvance", true)
canAdvance = json.optBoolean("canAdvance", true),
source = json.optString("source", "unknown"),
pageCharacters = json.optInt("pageCharacters", text.length),
extendedCharacters = json.optInt("extendedCharacters", 0)
)
}
}
@@ -1,25 +1,15 @@
package com.aletheia.app.ui.reader.tts
import android.content.Context
import android.media.AudioAttributes
import android.speech.tts.TextToSpeech
import android.speech.tts.UtteranceProgressListener
import android.util.Log
import com.aletheia.app.ui.reader.ReaderSpeechPage
import com.aletheia.app.ui.reader.ReaderWebController
import java.util.Locale
import java.util.UUID
import java.util.concurrent.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
/**
* Reads book pages through an installed Russian voice that explicitly declares
* that it does not require a network connection.
*
* The controller deliberately refuses network-only voices. A project-owned
* neural model can replace this engine behind the same reader-page contract.
*/
class ReaderSpeechController(
context: Context,
private val scope: CoroutineScope,
@@ -31,10 +21,11 @@ class ReaderSpeechController(
private val appContext = context.applicationContext
private val preferences = appContext.getSharedPreferences(PREFERENCES_NAME, Context.MODE_PRIVATE)
private var tts: TextToSpeech? = null
private val engine = VoskSpeechEngine(appContext)
private var state = State.IDLE
private var initializationComplete = false
private var resumeAfterInitialization = false
private var initializationJob: Job? = null
private var playbackJob: Job? = null
private var currentPage: ReaderSpeechPage? = null
private var lastSpokenSignature: String? = null
@@ -45,6 +36,16 @@ class ReaderSpeechController(
val pitch: Float
get() = preferences.getFloat(KEY_PITCH, DEFAULT_PITCH).coerceIn(MIN_PITCH, MAX_PITCH)
val pauseScale: Float
get() = preferences.getFloat(KEY_PAUSE_SCALE, DEFAULT_PAUSE_SCALE).coerceIn(MIN_PAUSE_SCALE, MAX_PAUSE_SCALE)
val articulationScale: Float
get() = preferences.getFloat(KEY_ARTICULATION_SCALE, DEFAULT_ARTICULATION_SCALE)
.coerceIn(MIN_ARTICULATION_SCALE, MAX_ARTICULATION_SCALE)
val speakerId: Int
get() = preferences.getInt(KEY_SPEAKER_ID, DEFAULT_SPEAKER_ID).coerceIn(0, VOICES.lastIndex)
fun toggle() {
when (state) {
State.PLAYING, State.INITIALIZING -> pause()
@@ -58,16 +59,17 @@ class ReaderSpeechController(
initialize()
return
}
val resumePausedPage = state == State.PAUSED
currentPage = null
lastSpokenSignature = null
updateState(State.PLAYING)
speakCurrentPage(reusePausedPage = resumePausedPage)
speakCurrentPage(reusePausedPage = false)
}
fun pause() {
resumeAfterInitialization = false
playbackJob?.cancel()
playbackJob = null
tts?.stop()
engine.stop()
updateState(State.PAUSED)
}
@@ -75,94 +77,61 @@ class ReaderSpeechController(
resumeAfterInitialization = false
playbackJob?.cancel()
playbackJob = null
tts?.stop()
engine.stop()
currentPage = null
lastSpokenSignature = null
updateState(State.IDLE)
}
fun updateSettings(rate: Float, pitch: Float) {
fun updateSettings(
rate: Float,
pitch: Float,
pauseScale: Float,
articulationScale: Float,
speakerId: Int
) {
preferences.edit()
.putFloat(KEY_RATE, rate.coerceIn(MIN_RATE, MAX_RATE))
.putFloat(KEY_PITCH, pitch.coerceIn(MIN_PITCH, MAX_PITCH))
.putFloat(KEY_PAUSE_SCALE, pauseScale.coerceIn(MIN_PAUSE_SCALE, MAX_PAUSE_SCALE))
.putFloat(
KEY_ARTICULATION_SCALE,
articulationScale.coerceIn(MIN_ARTICULATION_SCALE, MAX_ARTICULATION_SCALE)
)
.putInt(KEY_SPEAKER_ID, speakerId.coerceIn(0, VOICES.lastIndex))
.apply()
applySpeechSettings()
if (state == State.PLAYING) {
tts?.stop()
engine.stop()
speakCurrentPage(reusePausedPage = true)
}
}
fun destroy() {
initializationJob?.cancel()
playbackJob?.cancel()
initializationJob = null
playbackJob = null
tts?.stop()
tts?.shutdown()
tts = null
engine.destroy()
initializationComplete = false
}
private fun initialize() {
if (tts != null) {
updateState(State.INITIALIZING)
return
}
if (initializationJob?.isActive == true) return
updateState(State.INITIALIZING)
tts = TextToSpeech(appContext) { status ->
if (status != TextToSpeech.SUCCESS) {
fail("Не удалось запустить синтез речи на устройстве")
return@TextToSpeech
}
val engine = tts ?: return@TextToSpeech
val russianLocale = Locale.Builder().setLanguage("ru").setRegion("RU").build()
if (engine.setLanguage(russianLocale) < TextToSpeech.LANG_AVAILABLE) {
fail("Русский язык не поддерживается установленным синтезатором речи")
return@TextToSpeech
}
val offlineRussianVoice = engine.voices
?.asSequence()
?.filter { it.locale.language.equals("ru", ignoreCase = true) }
?.filterNot { it.isNetworkConnectionRequired }
?.sortedWith(compareByDescending<android.speech.tts.Voice> { it.quality }.thenBy { it.name })
?.firstOrNull()
if (offlineRussianVoice == null) {
fail("На телефоне не установлен русский офлайн-голос")
return@TextToSpeech
}
if (engine.setVoice(offlineRussianVoice) == TextToSpeech.ERROR) {
fail("Не удалось выбрать русский офлайн-голос")
return@TextToSpeech
}
engine.setAudioAttributes(
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
)
engine.setOnUtteranceProgressListener(object : UtteranceProgressListener() {
override fun onStart(utteranceId: String?) = Unit
override fun onDone(utteranceId: String?) {
scope.launch { advanceAfterPage() }
initializationJob = scope.launch {
try {
engine.initialize()
initializationComplete = true
if (resumeAfterInitialization) {
resumeAfterInitialization = false
updateState(State.PLAYING)
speakCurrentPage(reusePausedPage = false)
} else {
updateState(State.IDLE)
}
@Deprecated("Deprecated in Java")
override fun onError(utteranceId: String?) {
scope.launch { fail("Ошибка синтеза речи") }
}
override fun onError(utteranceId: String?, errorCode: Int) {
scope.launch { fail("Ошибка синтеза речи: $errorCode") }
}
})
initializationComplete = true
applySpeechSettings()
if (resumeAfterInitialization) {
resumeAfterInitialization = false
updateState(State.PLAYING)
speakCurrentPage(reusePausedPage = false)
} else {
updateState(State.IDLE)
} catch (error: Throwable) {
Log.e(TAG, "Offline voice initialization failed", error)
fail("Не удалось запустить встроенный офлайн-голос Vosk: ${error.message ?: error.javaClass.simpleName}")
}
}
}
@@ -170,29 +139,37 @@ class ReaderSpeechController(
private fun speakCurrentPage(reusePausedPage: Boolean) {
playbackJob?.cancel()
playbackJob = scope.launch {
val page = if (reusePausedPage) currentPage else null
?: reader.getSpeechPage(TextToSpeech.getMaxSpeechInputLength().coerceAtMost(MAX_PAGE_CHARS))
if (state != State.PLAYING) return@launch
if (page == null || page.text.isBlank()) {
fail("На текущей странице нет текста для озвучивания")
return@launch
try {
val page = if (reusePausedPage) currentPage else null
?: reader.getSpeechPage(MAX_PAGE_CHARS)
if (state != State.PLAYING) return@launch
if (page == null || page.text.isBlank()) {
fail("На текущей странице нет текста для озвучивания")
return@launch
}
Log.d(
TAG,
"Speech page extracted: chars=${page.text.length}, pageChars=${page.pageCharacters}, " +
"extended=${page.extendedCharacters}, canAdvance=${page.canAdvance}, source=${page.source}"
)
currentPage = page
val signature = page.signature()
if (!reusePausedPage && signature == lastSpokenSignature) {
stop()
onMessage("Достигнут конец книги")
return@launch
}
lastSpokenSignature = signature
engine.speak(page.text, rate, pitch, pauseScale, articulationScale)
advanceAfterPage()
} catch (_: CancellationException) {
// Pause and stop cancel the active synthesis/playback job.
} catch (error: Throwable) {
if (state == State.PLAYING) {
Log.e(TAG, "Offline speech playback failed", error)
fail("Ошибка встроенного синтеза речи: ${error.message ?: error.javaClass.simpleName}")
}
}
currentPage = page
val signature = page.signature()
if (!reusePausedPage && signature == lastSpokenSignature) {
stop()
onMessage("Достигнут конец книги")
return@launch
}
lastSpokenSignature = signature
applySpeechSettings()
val result = tts?.speak(
page.text,
TextToSpeech.QUEUE_FLUSH,
null,
UUID.randomUUID().toString()
) ?: TextToSpeech.ERROR
if (result == TextToSpeech.ERROR) fail("Не удалось начать озвучивание")
}
}
@@ -210,20 +187,16 @@ class ReaderSpeechController(
if (state == State.PLAYING) speakCurrentPage(reusePausedPage = false)
}
private fun applySpeechSettings() {
tts?.setSpeechRate(rate)
tts?.setPitch(pitch)
}
private fun fail(message: String) {
playbackJob?.cancel()
playbackJob = null
tts?.stop()
engine.stop()
updateState(State.ERROR)
onMessage(message)
}
private fun updateState(next: State) {
Log.d(TAG, "State: $state -> $next")
state = next
onStateChanged(next)
}
@@ -231,15 +204,27 @@ class ReaderSpeechController(
private fun ReaderSpeechPage.signature(): String = "$locator|${text.take(160)}"
companion object {
private const val TAG = "AletheiaVoskTts"
const val MIN_RATE = 0.55f
const val MAX_RATE = 1.80f
const val MIN_PITCH = 0.75f
const val MAX_PITCH = 1.30f
const val MIN_PAUSE_SCALE = 0.50f
const val MAX_PAUSE_SCALE = 2.00f
const val MIN_ARTICULATION_SCALE = 0.90f
const val MAX_ARTICULATION_SCALE = 1.20f
val VOICES = listOf("Мужской 1")
private const val DEFAULT_RATE = 1.0f
private const val DEFAULT_PITCH = 1.0f
private const val DEFAULT_PAUSE_SCALE = 1.0f
private const val DEFAULT_ARTICULATION_SCALE = 1.06f
private const val DEFAULT_SPEAKER_ID = 2
private const val PREFERENCES_NAME = "reader_speech"
private const val KEY_RATE = "rate"
private const val KEY_PITCH = "pitch"
private const val KEY_PAUSE_SCALE = "pause_scale"
private const val KEY_ARTICULATION_SCALE = "articulation_scale"
private const val KEY_SPEAKER_ID = "speaker_id"
private const val MAX_PAGE_CHARS = 3_500
private const val PAGE_TURN_DELAY_MS = 450L
}
@@ -2,8 +2,10 @@ package com.aletheia.app.ui.reader.tts
import android.content.Context
import android.view.ViewGroup
import android.widget.ArrayAdapter
import android.widget.LinearLayout
import android.widget.SeekBar
import android.widget.Spinner
import android.widget.TextView
import com.google.android.material.dialog.MaterialAlertDialogBuilder
import java.util.Locale
@@ -18,10 +20,25 @@ object ReaderSpeechSettingsDialog {
val vertical = (8 * density).roundToInt()
setPadding(horizontal, vertical, horizontal, vertical)
}
val voiceTitle = TextView(context).apply {
text = "Голос Vosk TTS 0.9"
textSize = 15f
setPadding(0, 16, 0, 4)
}
val voice = Spinner(context).apply {
adapter = ArrayAdapter(
context,
android.R.layout.simple_spinner_dropdown_item,
ReaderSpeechController.VOICES
)
setSelection(controller.speakerId)
}
content.addView(voiceTitle)
content.addView(voice)
val rateValue = TextView(context)
val rate = slider(
context = context,
label = "Скорость",
label = "Темп чтения",
value = controller.rate,
min = ReaderSpeechController.MIN_RATE,
max = ReaderSpeechController.MAX_RATE,
@@ -31,23 +48,54 @@ object ReaderSpeechSettingsDialog {
val pitchValue = TextView(context)
val pitch = slider(
context = context,
label = "Интонация",
label = "Высота голоса",
value = controller.pitch,
min = ReaderSpeechController.MIN_PITCH,
max = ReaderSpeechController.MAX_PITCH,
valueView = pitchValue,
parent = content
)
val pauseValue = TextView(context)
val pause = slider(
context = context,
label = "Паузы на знаках препинания",
value = controller.pauseScale,
min = ReaderSpeechController.MIN_PAUSE_SCALE,
max = ReaderSpeechController.MAX_PAUSE_SCALE,
valueView = pauseValue,
parent = content
)
val articulationValue = TextView(context)
val articulation = slider(
context = context,
label = "Разборчивость слов и букв",
value = controller.articulationScale,
min = ReaderSpeechController.MIN_ARTICULATION_SCALE,
max = ReaderSpeechController.MAX_ARTICULATION_SCALE,
valueView = articulationValue,
parent = content
)
MaterialAlertDialogBuilder(context)
.setTitle("Озвучивание книги")
.setMessage("Голос работает офлайн. Интонация регулирует высоту голоса.")
.setMessage("Все параметры применяются к встроенному офлайн-голосу. Настройки меняются плавно.")
.setView(content)
.setNegativeButton("Отмена", null)
.setPositiveButton("Сохранить") { _, _ ->
controller.updateSettings(
progressToValue(rate.progress, ReaderSpeechController.MIN_RATE, ReaderSpeechController.MAX_RATE),
progressToValue(pitch.progress, ReaderSpeechController.MIN_PITCH, ReaderSpeechController.MAX_PITCH)
progressToValue(pitch.progress, ReaderSpeechController.MIN_PITCH, ReaderSpeechController.MAX_PITCH),
progressToValue(
pause.progress,
ReaderSpeechController.MIN_PAUSE_SCALE,
ReaderSpeechController.MAX_PAUSE_SCALE
),
progressToValue(
articulation.progress,
ReaderSpeechController.MIN_ARTICULATION_SCALE,
ReaderSpeechController.MAX_ARTICULATION_SCALE
),
voice.selectedItemPosition
)
}
.show()
@@ -97,5 +145,5 @@ object ReaderSpeechSettingsDialog {
private fun progressToValue(progress: Int, min: Float, max: Float): Float =
min + (max - min) * progress.coerceIn(0, SLIDER_STEPS) / SLIDER_STEPS
private const val SLIDER_STEPS = 100
private const val SLIDER_STEPS = 300
}
@@ -0,0 +1,141 @@
package com.aletheia.app.ui.reader.tts
internal object RussianNumberNormalizer {
private val yearInPrepositionalPattern = Regex(
"(?<![\\p{L}\\p{N}])(\\d{4})\\s+(году|г\\.)",
RegexOption.IGNORE_CASE
)
private val numberPattern = Regex(
"(?<![\\p{L}\\p{N}])([+-]?)(\\d{1,18})(?:[,.](\\d+))?(?![\\p{L}\\p{N}])"
)
fun normalize(text: String): String {
val prepared = text
.replace(Regex("(?<=\\d)[\\u00A0\\u202F](?=\\d{3}(?:\\D|$))"), "")
.replace(Regex("\\s*(?=\\d)"), "номер ")
val yearsNormalized = yearInPrepositionalPattern.replace(prepared) { match ->
val year = match.groupValues[1].toInt()
"${yearInPrepositionalWords(year)} ${match.groupValues[2]}"
}
return numberPattern.replace(yearsNormalized) { match ->
val sign = when (match.groupValues[1]) {
"-" -> "минус "
"+" -> "плюс "
else -> ""
}
val integerDigits = match.groupValues[2]
val integer = if (integerDigits.length > 1 && integerDigits.startsWith('0')) {
integerDigits.toDigitWords()
} else {
integerDigits.toLongOrNull()?.let(::integerToWords) ?: integerDigits.toDigitWords()
}
val fractionDigits = match.groupValues[3]
if (fractionDigits.isBlank()) "$sign$integer" else "$sign$integer запятая ${fractionDigits.toDigitWords()}"
}
}
private fun yearInPrepositionalWords(value: Int): String {
require(value in 1000..9999)
if (value % 1000 == 0) return EXACT_THOUSANDTH_PREPOSITIONAL[value / 1000]
val thousands = value / 1000
val thousandsCardinal = underThousandToWords(thousands, feminine = true)
val thousandsWords = if (thousandsCardinal == "одна") {
SCALES[1]!!.formFor(thousands)
} else {
"$thousandsCardinal ${SCALES[1]!!.formFor(thousands)}"
}
return "$thousandsWords ${ordinalUnderThousandPrepositional(value % 1000)}"
}
private fun ordinalUnderThousandPrepositional(value: Int): String {
require(value in 1..999)
val lastTwo = value % 100
val units = value % 10
return when {
units != 0 && lastTwo !in 11..19 ->
"${integerToWords((value - units).toLong())} ${ORDINAL_UNITS_PREPOSITIONAL[units]}".trim()
lastTwo in 11..19 ->
"${integerToWords((value - lastTwo).toLong())} ${ORDINAL_TEENS_PREPOSITIONAL[lastTwo - 10]}".trim()
lastTwo >= 20 ->
"${integerToWords((value - lastTwo).toLong())} ${ORDINAL_TENS_PREPOSITIONAL[lastTwo / 10]}".trim()
else -> ORDINAL_HUNDREDS_PREPOSITIONAL[value / 100]
}
}
private fun integerToWords(value: Long): String {
if (value == 0L) return DIGITS[0]
if (value < 0L) return "минус ${integerToWords(-value)}"
val groups = mutableListOf<Int>()
var remainder = value
while (remainder > 0) {
groups += (remainder % 1_000).toInt()
remainder /= 1_000
}
return groups.indices.reversed().mapNotNull { groupIndex ->
val group = groups[groupIndex]
if (group == 0) return@mapNotNull null
val feminine = groupIndex == 1
val words = underThousandToWords(group, feminine)
val scale = SCALES.getOrNull(groupIndex)
if (scale == null) words else "$words ${scale.formFor(group)}"
}.joinToString(" ")
}
private fun underThousandToWords(value: Int, feminine: Boolean): String {
val words = mutableListOf<String>()
val hundreds = value / 100
if (hundreds > 0) words += HUNDREDS[hundreds]
val tail = value % 100
when {
tail in 10..19 -> words += TEENS[tail - 10]
else -> {
val tens = tail / 10
if (tens > 0) words += TENS[tens]
val units = tail % 10
if (units > 0) {
words += when {
feminine && units == 1 -> "одна"
feminine && units == 2 -> "две"
else -> DIGITS[units]
}
}
}
}
return words.joinToString(" ")
}
private fun String.toDigitWords(): String = map { DIGITS[it.digitToInt()] }.joinToString(" ")
private data class Scale(val one: String, val few: String, val many: String) {
fun formFor(value: Int): String {
val lastTwo = value % 100
if (lastTwo in 11..14) return many
return when (value % 10) {
1 -> one
2, 3, 4 -> few
else -> many
}
}
}
private val DIGITS = listOf("ноль", "один", "два", "три", "четыре", "пять", "шесть", "семь", "восемь", "девять")
private val TEENS = listOf(
"десять", "одиннадцать", "двенадцать", "тринадцать", "четырнадцать",
"пятнадцать", "шестнадцать", "семнадцать", "восемнадцать", "девятнадцать"
)
private val TENS = listOf("", "десять", "двадцать", "тридцать", "сорок", "пятьдесят", "шестьдесят", "семьдесят", "восемьдесят", "девяносто")
private val HUNDREDS = listOf("", "сто", "двести", "триста", "четыреста", "пятьсот", "шестьсот", "семьсот", "восемьсот", "девятьсот")
private val ORDINAL_UNITS_PREPOSITIONAL = listOf("", "первом", "втором", "третьем", "четвёртом", "пятом", "шестом", "седьмом", "восьмом", "девятом")
private val ORDINAL_TEENS_PREPOSITIONAL = listOf("десятом", "одиннадцатом", "двенадцатом", "тринадцатом", "четырнадцатом", "пятнадцатом", "шестнадцатом", "семнадцатом", "восемнадцатом", "девятнадцатом")
private val ORDINAL_TENS_PREPOSITIONAL = listOf("", "десятом", "двадцатом", "тридцатом", "сороковом", "пятидесятом", "шестидесятом", "семидесятом", "восьмидесятом", "девяностом")
private val ORDINAL_HUNDREDS_PREPOSITIONAL = listOf("", "сотом", "двухсотом", "трёхсотом", "четырёхсотом", "пятисотом", "шестисотом", "семисотом", "восьмисотом", "девятисотом")
private val EXACT_THOUSANDTH_PREPOSITIONAL = listOf("", "тысячном", "двухтысячном", "трёхтысячном", "четырёхтысячном", "пятитысячном", "шеститысячном", "семитысячном", "восьмитысячном", "девятитысячном")
private val SCALES = listOf(
null,
Scale("тысяча", "тысячи", "тысяч"),
Scale("миллион", "миллиона", "миллионов"),
Scale("миллиард", "миллиарда", "миллиардов"),
Scale("триллион", "триллиона", "триллионов"),
Scale("квадриллион", "квадриллиона", "квадриллионов")
)
}
@@ -0,0 +1,62 @@
package com.aletheia.app.ui.reader.tts
import java.util.Locale
internal object RussianSpeechTextSplitter {
fun splitSentences(text: String): List<String> {
val result = mutableListOf<String>()
var start = 0
var index = 0
while (index < text.length) {
val char = text[index]
val isEllipsis = char == '…' || text.startsWith("...", index)
val isTerminal = when {
char == '!' || char == '?' || isEllipsis -> true
char == '.' -> isSentencePeriod(text, index)
else -> false
}
if (!isTerminal) {
index += 1
continue
}
if (text.startsWith("...", index)) index += 2
var end = index + 1
while (end < text.length && text[end] in SENTENCE_CLOSERS) end += 1
val hasSentenceBoundary = end == text.length || text[end].isWhitespace()
if (hasSentenceBoundary && !isFollowedByDialogueDash(text, end)) {
val sentence = text.substring(start, end).trim()
if (sentence.isNotBlank()) result += sentence
start = end
while (start < text.length && text[start].isWhitespace()) start += 1
index = start
} else {
index = end
}
}
val remainder = text.substring(start).trim()
if (remainder.isNotBlank()) result += remainder
return result
}
private fun isFollowedByDialogueDash(text: String, end: Int): Boolean {
var index = end
while (index < text.length && text[index].isWhitespace()) index += 1
return text.getOrNull(index) in DIALOGUE_DASHES
}
private fun isSentencePeriod(text: String, index: Int): Boolean {
if (text.getOrNull(index - 1)?.isDigit() == true && text.getOrNull(index + 1)?.isDigit() == true) return false
val prefix = text.substring(0, index)
val token = prefix.takeLastWhile { it.isLetter() }.lowercase(Locale.forLanguageTag("ru"))
if (token in NON_TERMINAL_ABBREVIATIONS) return false
if (token.length == 1 && prefix.lastOrNull()?.isUpperCase() == true) return false
return true
}
private val SENTENCE_CLOSERS = setOf('"', '\'', '»', '”', ')')
private val DIALOGUE_DASHES = setOf('—', '', '-')
private val NON_TERMINAL_ABBREVIATIONS = setOf(
"г", "гг", "ул", "стр", "рис", "им", "т", "д", "п", "н", "э", "е", "к",
"др", "см", "руб", "коп", "тыс", "млн", "млрд"
)
}
@@ -0,0 +1,708 @@
package com.aletheia.app.ui.reader.tts
import ai.onnxruntime.OnnxTensor
import ai.onnxruntime.OrtEnvironment
import ai.onnxruntime.OrtSession
import android.content.Context
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioTrack
import android.media.PlaybackParams
import android.util.Log
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.FileOutputStream
import java.io.RandomAccessFile
import java.nio.FloatBuffer
import java.nio.LongBuffer
import java.nio.charset.StandardCharsets
import java.util.LinkedHashMap
import java.util.Locale
import java.util.concurrent.atomic.AtomicInteger
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.withContext
import kotlin.coroutines.coroutineContext
class VoskSpeechEngine(context: Context) {
private val appContext = context.applicationContext
private val playbackGeneration = AtomicInteger()
private val trackLock = Any()
private val environment = OrtEnvironment.getEnvironment()
private var synthesisSession: OrtSession? = null
private var bertSession: OrtSession? = null
private var dictionary: PronunciationDictionary? = null
private var tokenizer: WordPieceTokenizer? = null
private var activeTrack: AudioTrack? = null
suspend fun initialize() = withContext(Dispatchers.IO) {
if (synthesisSession != null) return@withContext
val modelFile = copyAssetToInternalStorage(MODEL_ASSET, MODEL_FILE)
val bertFile = copyAssetToInternalStorage(BERT_ASSET, BERT_FILE)
val dictionaryFile = copyAssetToInternalStorage(DICTIONARY_ASSET, DICTIONARY_FILE)
val dictionaryIndex = appContext.assets.open(DICTIONARY_INDEX_ASSET).bufferedReader().useLines { lines ->
lines.mapNotNull { line ->
val separator = line.lastIndexOf('\t')
if (separator <= 0) null else DictionaryIndexEntry(
line.substring(0, separator),
line.substring(separator + 1).toLong()
)
}.toList()
}
val vocabulary = appContext.assets.open(VOCABULARY_ASSET).bufferedReader().useLines { lines ->
lines.mapIndexed { index, token -> token to index.toLong() }.toMap()
}
val options = OrtSession.SessionOptions().apply {
setOptimizationLevel(OrtSession.SessionOptions.OptLevel.NO_OPT)
setIntraOpNumThreads(INFERENCE_THREADS)
setInterOpNumThreads(1)
}
synthesisSession = environment.createSession(modelFile.absolutePath, options)
bertSession = environment.createSession(bertFile.absolutePath, options)
dictionary = PronunciationDictionary(dictionaryFile, dictionaryIndex)
tokenizer = WordPieceTokenizer(vocabulary)
Log.i(
TAG,
"Vosk TTS 0.9 loaded: model=${modelFile.length()} bert=${bertFile.length()} " +
"dictionary=${dictionaryFile.length()}"
)
}
suspend fun speak(
text: String,
rate: Float,
pitch: Float,
pauseScale: Float,
articulationScale: Float
) = withContext(Dispatchers.Default) {
val localSynthesis = checkNotNull(synthesisSession) { "Vosk TTS is not initialized" }
val localBert = checkNotNull(bertSession) { "Vosk BERT is not initialized" }
val localDictionary = checkNotNull(dictionary) { "Vosk dictionary is not initialized" }
val localTokenizer = checkNotNull(tokenizer) { "Vosk tokenizer is not initialized" }
val generation = playbackGeneration.incrementAndGet()
for (chunk in splitText(text)) {
coroutineContext.ensureActive()
check(generation == playbackGeneration.get()) { "Playback stopped" }
val prepared = prepareInput(chunk.text, localDictionary, localTokenizer, localBert)
if (prepared.timeSteps <= 2) continue
Log.d(TAG, "Synthesizing ${chunk.text.length} chars as ${prepared.timeSteps} phonemes")
val output = synthesize(localSynthesis, prepared, articulationScale)
check(output.isNotEmpty()) { "Vosk returned empty audio" }
val playbackSpeed = (rate * BOOK_SPEED_SCALE).coerceIn(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED)
val scaledPauseMs = if (chunk.trailingPauseMs == NO_TRAILING_PAUSE_MS) {
NO_TRAILING_PAUSE_MS
} else {
(chunk.trailingPauseMs * pauseScale / rate.coerceAtLeast(MIN_RATE_FOR_PAUSE_SCALING))
.toInt()
.coerceAtLeast(MIN_TRAILING_PAUSE_MS)
.coerceAtMost(chunk.maximumPauseMs)
}
Log.d(
TAG,
"Generated ${output.size} samples at $SAMPLE_RATE Hz for ${chunk.text.length} chars; " +
"speaker=$MALE_1_SPEAKER_ID speed=$playbackSpeed pitch=$pitch " +
"articulation=$articulationScale pause=${scaledPauseMs}ms"
)
play(output, generation, playbackSpeed, pitch, scaledPauseMs)
}
}
fun stop() {
playbackGeneration.incrementAndGet()
synchronized(trackLock) {
activeTrack?.runCatching { pause() }
activeTrack?.runCatching { flush() }
activeTrack?.runCatching { stop() }
activeTrack?.release()
activeTrack = null
}
}
fun destroy() {
stop()
dictionary?.close()
dictionary = null
tokenizer = null
bertSession?.close()
bertSession = null
synthesisSession?.close()
synthesisSession = null
}
private fun prepareInput(
text: String,
dictionary: PronunciationDictionary,
tokenizer: WordPieceTokenizer,
bert: OrtSession
): PreparedInput {
val normalized = RussianNumberNormalizer.normalize(text)
.replace(Regex("([А-Яа-яЁё])\\u0301"), "+$1")
.lowercase(Locale.forLanguageTag("ru"))
.replace('—', '-')
.replace('', '-')
.replace("", "...")
.replace(Regex("[«»“”]"), "\"")
.replace(Regex("\\s+"), " ")
.trim()
val encoding = tokenizer.encode(normalized)
val tokenEmbeddings = runBert(bert, encoding.ids)
val selectedEmbeddings = encoding.embeddingPositions.map { position -> tokenEmbeddings[position] }
val phones = buildMultistreamPhones(normalized, dictionary)
val timeSteps = phones.size
val features = LongArray(FEATURE_CHANNELS * timeSteps)
val embeddings = FloatArray(BERT_DIMENSIONS * timeSteps)
phones.forEachIndexed { time, phone ->
phone.features.forEachIndexed { channel, value -> features[channel * timeSteps + time] = value }
val embedding = selectedEmbeddings[phone.bertWordIndex.coerceIn(selectedEmbeddings.indices)]
for (channel in 0 until BERT_DIMENSIONS) {
embeddings[channel * timeSteps + time] = embedding[channel]
}
}
return PreparedInput(features, embeddings, timeSteps)
}
private fun runBert(session: OrtSession, ids: LongArray): Array<FloatArray> {
val shape = longArrayOf(1, ids.size.toLong())
val inputIds = OnnxTensor.createTensor(environment, LongBuffer.wrap(ids), shape)
val attentionMask = OnnxTensor.createTensor(
environment,
LongBuffer.wrap(LongArray(ids.size) { 1L }),
shape
)
val tokenTypes = OnnxTensor.createTensor(environment, LongBuffer.wrap(LongArray(ids.size)), shape)
try {
session.run(
mapOf(
"input_ids" to inputIds,
"attention_mask" to attentionMask,
"token_type_ids" to tokenTypes
)
).use { result ->
@Suppress("UNCHECKED_CAST")
return result[0].value as Array<FloatArray>
}
} finally {
inputIds.close()
attentionMask.close()
tokenTypes.close()
}
}
private fun synthesize(session: OrtSession, prepared: PreparedInput, articulationScale: Float): FloatArray {
val input = OnnxTensor.createTensor(
environment,
LongBuffer.wrap(prepared.features),
longArrayOf(1, FEATURE_CHANNELS.toLong(), prepared.timeSteps.toLong())
)
val inputLengths = OnnxTensor.createTensor(
environment,
LongBuffer.wrap(longArrayOf(prepared.timeSteps.toLong())),
longArrayOf(1)
)
val scales = OnnxTensor.createTensor(
environment,
FloatBuffer.wrap(
floatArrayOf(
NOISE_SCALE,
articulationScale.coerceIn(MIN_ARTICULATION_SCALE, MAX_ARTICULATION_SCALE),
DURATION_NOISE_SCALE
)
),
longArrayOf(3)
)
val speaker = OnnxTensor.createTensor(
environment,
LongBuffer.wrap(longArrayOf(MALE_1_SPEAKER_ID.toLong())),
longArrayOf(1)
)
val bert = OnnxTensor.createTensor(
environment,
FloatBuffer.wrap(prepared.bertEmbeddings),
longArrayOf(1, BERT_DIMENSIONS.toLong(), prepared.timeSteps.toLong())
)
try {
session.run(
mapOf(
"input" to input,
"input_lengths" to inputLengths,
"scales" to scales,
"sid" to speaker,
"bert" to bert
)
).use { result ->
return flattenFloatOutput(result[0].value)
}
} finally {
input.close()
inputLengths.close()
scales.close()
speaker.close()
bert.close()
}
}
private fun buildMultistreamPhones(
text: String,
dictionary: PronunciationDictionary
): List<PhoneFeatures> {
val rawPhones = mutableListOf(RawPhone("^", emptyList(), 0, 0))
val word = StringBuilder()
val pendingPunctuation = mutableListOf<String>()
var inQuote = 0
var bertWordIndex = 1
fun flushWord() {
if (word.isEmpty()) return
val value = word.toString()
val pronunciation = dictionary.find(value) ?: RussianG2p.convert(value)
pronunciation.split(' ').filter(String::isNotBlank).forEach { phone ->
rawPhones += RawPhone(phone, emptyList(), inQuote, bertWordIndex)
}
word.clear()
bertWordIndex += 1
}
fun appendSpace() {
rawPhones += RawPhone(" ", pendingPunctuation.toList(), inQuote, bertWordIndex)
pendingPunctuation.clear()
}
var index = 0
while (index < text.length) {
if (text.startsWith("...", index)) {
flushWord()
pendingPunctuation += "..."
index += 3
continue
}
val char = text[index]
when {
char == '"' || char == '«' || char == '»' || char == '“' || char == '”' -> {
flushWord()
inQuote = 1 - inQuote
}
char.isWhitespace() -> {
flushWord()
appendSpace()
}
char == '-' && text.getOrNull(index - 1)?.isLetterOrDigit() == true &&
text.getOrNull(index + 1)?.isLetterOrDigit() == true -> {
// A lexical hyphen separates WordPiece words but must not create an acoustic pause.
flushWord()
}
char in MULTISTREAM_PUNCTUATION -> {
flushWord()
pendingPunctuation += char.toString()
}
char.isLetterOrDigit() || char == '+' -> word.append(char)
}
index += 1
}
flushWord()
appendSpace()
rawPhones += RawPhone("$", emptyList(), 0, bertWordIndex)
var lastPunctuation = " "
var lastSentencePunctuation = " "
val reversed = mutableListOf<PhoneFeatures>()
for (phone in rawPhones.asReversed()) {
lastSentencePunctuation = when {
"..." in phone.punctuation -> "..."
"." in phone.punctuation -> "."
"!" in phone.punctuation -> "!"
"?" in phone.punctuation -> "?"
"-" in phone.punctuation -> "-"
else -> lastSentencePunctuation
}
if (phone.punctuation.isNotEmpty()) lastPunctuation = phone.punctuation.first()
val currentPunctuation = phone.punctuation.firstOrNull() ?: "_"
val values = longArrayOf(
PHONEME_IDS.getValue(phone.phone),
PHONEME_IDS.getValue(currentPunctuation),
phone.inQuote.toLong(),
PHONEME_IDS.getValue(lastPunctuation),
PHONEME_IDS.getValue(lastSentencePunctuation)
)
reversed += PhoneFeatures(values, phone.bertWordIndex)
}
return reversed.asReversed()
}
private fun flattenFloatOutput(value: Any?): FloatArray = when (value) {
is FloatArray -> value
is Array<*> -> value.asSequence().flatMap { flattenFloatOutput(it).asSequence() }.toList().toFloatArray()
else -> error("Unexpected Vosk output type: ${value?.javaClass?.name}")
}
private fun play(
samples: FloatArray,
generation: Int,
playbackSpeed: Float,
pitch: Float,
trailingPauseMs: Int
) {
val minBytes = AudioTrack.getMinBufferSize(
SAMPLE_RATE,
AudioFormat.CHANNEL_OUT_MONO,
AudioFormat.ENCODING_PCM_FLOAT
).coerceAtLeast(4096)
val track = AudioTrack.Builder()
.setAudioAttributes(
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_FLOAT)
.setSampleRate(SAMPLE_RATE)
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
.build()
)
.setBufferSizeInBytes(minBytes)
.setTransferMode(AudioTrack.MODE_STREAM)
.build()
track.playbackParams = PlaybackParams()
.allowDefaults()
.setSpeed(playbackSpeed)
.setPitch(pitch.coerceIn(MIN_PLAYBACK_PITCH, MAX_PLAYBACK_PITCH))
synchronized(trackLock) { activeTrack = track }
try {
track.play()
var offset = 0
while (offset < samples.size && generation == playbackGeneration.get()) {
val written = track.write(samples, offset, samples.size - offset, AudioTrack.WRITE_BLOCKING)
if (written <= 0) error("AudioTrack write failed: $written")
offset += written
}
var silenceSamples = trailingPauseMs * SAMPLE_RATE / 1_000
while (silenceSamples > 0 && generation == playbackGeneration.get()) {
val count = minOf(silenceSamples, SILENCE_BUFFER.size)
val written = track.write(SILENCE_BUFFER, 0, count, AudioTrack.WRITE_BLOCKING)
if (written <= 0) error("AudioTrack silence write failed: $written")
silenceSamples -= written
}
check(generation == playbackGeneration.get()) { "Playback stopped" }
} finally {
synchronized(trackLock) { if (activeTrack === track) activeTrack = null }
track.runCatching { stop() }
track.release()
}
}
private fun splitText(text: String): List<SpeechChunk> {
val normalized = text
.replace(Regex("\\[\\d+(?:[,-]\\d+)*\\]"), " ")
.replace("\r\n", "\n")
.replace('\r', '\n')
.replace(Regex("[\\t\\x0B\\f ]+"), " ")
.replace(Regex(" *\\n+ *"), "\n")
.trim()
if (normalized.isBlank()) return emptyList()
val chunks = mutableListOf<SpeechChunk>()
for (paragraph in normalized.split('\n').filter(String::isNotBlank)) {
for (sentence in RussianSpeechTextSplitter.splitSentences(paragraph)) {
for (unit in splitControlledPauses(sentence)) {
val parts = splitAtWordBoundaries(unit.text)
parts.forEachIndexed { index, part ->
val isLastPart = index == parts.lastIndex
chunks += SpeechChunk(
text = part.trim(),
trailingPauseMs = if (isLastPart) unit.trailingPauseMs else NO_TRAILING_PAUSE_MS,
maximumPauseMs = if (isLastPart) unit.maximumPauseMs else NO_TRAILING_PAUSE_MS
)
}
}
}
}
return chunks.filter { it.text.isNotBlank() }
}
private fun splitControlledPauses(text: String): List<ProsodicUnit> {
val units = mutableListOf<ProsodicUnit>()
val current = StringBuilder()
fun flushControlledPause() {
val value = current.toString().trim()
if (value.isNotBlank()) {
units += ProsodicUnit(value, CONTROLLED_PUNCTUATION_PAUSE_MS, CONTROLLED_PUNCTUATION_MAX_PAUSE_MS)
}
current.clear()
}
text.forEachIndexed { index, char ->
val isWordInternalHyphen = char == '-' &&
text.getOrNull(index - 1)?.isLetterOrDigit() == true &&
text.getOrNull(index + 1)?.isLetterOrDigit() == true
when {
char in CONTROLLED_PAUSE_PUNCTUATION && !isWordInternalHyphen -> flushControlledPause()
else -> current.append(char)
}
}
val remainder = current.toString().trim()
if (remainder.isNotBlank()) units += ProsodicUnit(remainder, pauseAfter(remainder), Int.MAX_VALUE)
return units
}
private fun splitAtWordBoundaries(text: String): List<String> {
if (text.length <= MAX_CHUNK_CHARS) return listOf(text)
val parts = mutableListOf<String>()
var remaining = text.trim()
while (remaining.length > MAX_CHUNK_CHARS) {
val boundary = remaining.lastIndexOf(' ', startIndex = MAX_CHUNK_CHARS)
.takeIf { it > MIN_CHUNK_CHARS }
?: remaining.indexOf(' ', startIndex = MAX_CHUNK_CHARS).takeIf { it >= 0 }
?: remaining.length
parts += remaining.substring(0, boundary).trim()
remaining = remaining.substring(boundary).trimStart()
}
if (remaining.isNotBlank()) parts += remaining
return parts
}
private fun pauseAfter(text: String): Int = when (text.trimEnd().lastOrNull()) {
'…' -> ELLIPSIS_PAUSE_MS
'?' -> QUESTION_PAUSE_MS
'!' -> EXCLAMATION_PAUSE_MS
'.' -> SENTENCE_PAUSE_MS
':', ';' -> CLAUSE_PAUSE_MS
else -> NO_TRAILING_PAUSE_MS
}
private fun copyAssetToInternalStorage(assetPath: String, fileName: String): File {
val destination = File(appContext.filesDir, "tts/vosk_0_9/$fileName")
val expectedSize = appContext.assets.open(assetPath).use { input ->
var total = 0L
val buffer = ByteArray(COPY_BUFFER_SIZE)
while (true) {
val read = input.read(buffer)
if (read < 0) break
total += read
}
total
}
if (destination.isFile && destination.length() == expectedSize) return destination
destination.parentFile?.mkdirs()
val temporary = File(destination.parentFile, "$fileName.tmp")
appContext.assets.open(assetPath).use { input ->
FileOutputStream(temporary).use { output -> input.copyTo(output, COPY_BUFFER_SIZE) }
}
check(temporary.length() == expectedSize) { "Vosk asset copy is incomplete: $fileName" }
if (destination.exists()) check(destination.delete()) { "Cannot replace old Vosk asset: $fileName" }
check(temporary.renameTo(destination)) { "Cannot install Vosk asset: $fileName" }
return destination
}
private class WordPieceTokenizer(private val vocabulary: Map<String, Long>) {
fun encode(text: String): BertEncoding {
val ids = mutableListOf(vocabulary.getValue("[CLS]"))
val embeddingPositions = mutableListOf(0)
for (token in basicTokens(text)) {
val punctuation = token in BERT_EXCLUDED_PUNCTUATION
val pieces = if (punctuation) {
listOf(vocabulary[token] ?: vocabulary.getValue("[UNK]"))
} else {
wordPieces(token)
}
if (!punctuation) embeddingPositions += ids.size
ids += pieces
}
ids += vocabulary.getValue("[SEP]")
embeddingPositions += ids.lastIndex
return BertEncoding(ids.toLongArray(), embeddingPositions.toIntArray())
}
private fun basicTokens(text: String): List<String> {
val result = mutableListOf<String>()
val word = StringBuilder()
fun flush() {
if (word.isNotEmpty()) result += word.toString().also { word.clear() }
}
text.forEach { char ->
when {
char.isWhitespace() -> flush()
char.isLetterOrDigit() -> word.append(char.lowercaseChar())
char == '+' -> Unit // Explicit stress marker belongs to G2P, not to BERT tokenization.
else -> {
flush()
result += char.toString()
}
}
}
flush()
return result
}
private fun wordPieces(token: String): List<Long> {
if (token.length > MAX_WORD_PIECE_CHARS) return listOf(vocabulary.getValue("[UNK]"))
val result = mutableListOf<Long>()
var start = 0
while (start < token.length) {
var end = token.length
var found: Long? = null
while (start < end) {
val piece = token.substring(start, end).let { if (start == 0) it else "##$it" }
found = vocabulary[piece]
if (found != null) break
end -= 1
}
if (found == null) return listOf(vocabulary.getValue("[UNK]"))
result += found
start = end
}
return result
}
}
private data class DictionaryIndexEntry(val word: String, val offset: Long)
private class PronunciationDictionary(
file: File,
private val index: List<DictionaryIndexEntry>
) : AutoCloseable {
private val source = RandomAccessFile(file, "r")
private val cache = object : LinkedHashMap<String, String>(CACHE_SIZE, 0.75f, true) {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, String>?): Boolean = size > CACHE_SIZE
}
@Synchronized
fun find(word: String): String? {
cache[word]?.let { return it }
var low = 0
var high = index.lastIndex
while (low <= high) {
val midpoint = (low + high) ushr 1
if (index[midpoint].word <= word) low = midpoint + 1 else high = midpoint - 1
}
val rangeIndex = high.coerceAtLeast(0)
val rangeEnd = index.getOrNull(rangeIndex + 1)?.offset ?: source.length()
source.seek(index[rangeIndex].offset)
while (source.filePointer < rangeEnd) {
val line = readUtf8Line(source) ?: break
val entryWord = line.substringBefore('\t')
when {
entryWord < word -> continue
entryWord > word -> return null
else -> {
val pronunciation = line.substringAfter('\t', "").trim()
if (pronunciation.isNotEmpty()) cache[word] = pronunciation
return pronunciation.takeIf(String::isNotEmpty)
}
}
}
return null
}
override fun close() = source.close()
private fun readUtf8Line(file: RandomAccessFile): String? {
val bytes = ByteArrayOutputStream(64)
while (true) {
val value = file.read()
if (value < 0) return if (bytes.size() == 0) null else bytes.toString(StandardCharsets.UTF_8.name())
if (value == '\n'.code) break
if (value != '\r'.code) bytes.write(value)
}
return bytes.toString(StandardCharsets.UTF_8.name())
}
}
private object RussianG2p {
private val softLetters = setOf('я', 'ё', 'ю', 'и', 'ь', 'е')
private val syllableStarts = setOf('#', 'ъ', 'ь', 'а', 'я', 'о', 'ё', 'у', 'ю', 'э', 'е', 'и', 'ы', '-')
private val softHardConsonants = mapOf(
'б' to "b", 'в' to "v", 'г' to "g", 'д' to "d", 'з' to "z", 'к' to "k",
'л' to "l", 'м' to "m", 'н' to "n", 'п' to "p", 'р' to "r", 'с' to "s",
'т' to "t", 'ф' to "f", 'х' to "h"
)
private val otherConsonants = mapOf('ж' to "zh", 'ц' to "c", 'ч' to "ch", 'ш' to "sh", 'щ' to "sch", 'й' to "j")
private val vowels = mapOf('а' to "a", 'я' to "a", 'у' to "u", 'ю' to "u", 'о' to "o", 'ё' to "o", 'э' to "e", 'е' to "e", 'и' to "i", 'ы' to "y")
fun convert(word: String): String {
val marked = mutableListOf<Pair<Char, Int>>()
var stress = 0
for (char in "#$word#") {
if (char == '+') stress = 1 else {
marked += char to stress
stress = 0
}
}
val phones = mutableListOf<String>()
for (index in marked.indices) {
val (char, accent) = marked[index]
val next = marked.getOrNull(index + 1)?.first
when {
char in softHardConsonants -> phones += softHardConsonants.getValue(char) + if (next in softLetters) "j" else ""
char in otherConsonants -> phones += otherConsonants.getValue(char)
char in vowels -> {
val previous = marked.getOrNull(index - 1)?.first ?: '#'
if (previous in syllableStarts && char in setOf('я', 'ю', 'е', 'ё')) phones += "j"
phones += vowels.getValue(char) + accent
}
}
}
return phones.joinToString(" ")
}
}
private companion object {
data class PreparedInput(
val features: LongArray,
val bertEmbeddings: FloatArray,
val timeSteps: Int
)
data class BertEncoding(val ids: LongArray, val embeddingPositions: IntArray)
data class RawPhone(val phone: String, val punctuation: List<String>, val inQuote: Int, val bertWordIndex: Int)
data class PhoneFeatures(val features: LongArray, val bertWordIndex: Int)
data class SpeechChunk(val text: String, val trailingPauseMs: Int, val maximumPauseMs: Int = Int.MAX_VALUE)
data class ProsodicUnit(val text: String, val trailingPauseMs: Int, val maximumPauseMs: Int)
const val TAG = "AletheiaVoskTts"
const val MODEL_FILE = "model.onnx"
const val BERT_FILE = "bert.int8.onnx"
const val DICTIONARY_FILE = "dict.tsv"
const val MODEL_ASSET = "tts/vosk_0_9/$MODEL_FILE"
const val BERT_ASSET = "tts/vosk_0_9/$BERT_FILE"
const val DICTIONARY_ASSET = "tts/vosk_0_9/$DICTIONARY_FILE"
const val DICTIONARY_INDEX_ASSET = "tts/vosk_0_9/dictionary.index"
const val VOCABULARY_ASSET = "tts/vosk_0_9/vocab.txt"
const val SAMPLE_RATE = 22_050
const val MALE_1_SPEAKER_ID = 4
const val FEATURE_CHANNELS = 5
const val BERT_DIMENSIONS = 768
const val INFERENCE_THREADS = 4
const val COPY_BUFFER_SIZE = 1024 * 1024
const val CACHE_SIZE = 4_096
const val MAX_WORD_PIECE_CHARS = 100
const val MAX_CHUNK_CHARS = 120
const val MIN_CHUNK_CHARS = 80
const val BOOK_SPEED_SCALE = 0.82f
const val MIN_RATE_FOR_PAUSE_SCALING = 0.10f
const val MIN_PLAYBACK_SPEED = 0.50f
const val MAX_PLAYBACK_SPEED = 1.50f
const val MIN_PLAYBACK_PITCH = 0.75f
const val MAX_PLAYBACK_PITCH = 1.30f
const val MIN_ARTICULATION_SCALE = 0.90f
const val MAX_ARTICULATION_SCALE = 1.20f
const val NOISE_SCALE = 0.8f
const val DURATION_NOISE_SCALE = 0.8f
const val NO_TRAILING_PAUSE_MS = 0
const val MIN_TRAILING_PAUSE_MS = 80
const val CLAUSE_PAUSE_MS = 320
const val CONTROLLED_PUNCTUATION_PAUSE_MS = 300
const val CONTROLLED_PUNCTUATION_MAX_PAUSE_MS = 300
const val SENTENCE_PAUSE_MS = 600
const val EXCLAMATION_PAUSE_MS = 600
const val QUESTION_PAUSE_MS = 600
const val ELLIPSIS_PAUSE_MS = 1_200
val SILENCE_BUFFER = FloatArray(SAMPLE_RATE / 10)
val CONTROLLED_PAUSE_PUNCTUATION = setOf('—', '', '-', '(', ')', ';', ':')
val MULTISTREAM_PUNCTUATION = setOf('!', '(', ')', ',', '-', '.', ':', ';', '?')
val BERT_EXCLUDED_PUNCTUATION = setOf("-", ",", ".", "?", "!", ";", ":", "\"")
val PHONEME_IDS = listOf(
"_", "^", "$", " ", "!", "'", "(", ")", ",", "-", ".", "...", ":", ";", "?",
"a0", "a1", "b", "bj", "c", "ch", "d", "dj", "e0", "e1", "f", "fj", "g", "gj",
"h", "hj", "i0", "i1", "j", "k", "kj", "l", "lj", "m", "mj", "n", "nj", "o0",
"o1", "p", "pj", "r", "rj", "s", "sch", "sh", "sj", "t", "tj", "u0", "u1", "v",
"vj", "y0", "y1", "z", "zh", "zj"
).mapIndexed { index, phoneme -> phoneme to index.toLong() }.toMap()
}
}
@@ -0,0 +1,40 @@
package com.aletheia.app.ui.reader.tts
import org.junit.Assert.assertEquals
import org.junit.Test
class RussianNumberNormalizerTest {
@Test
fun `normalizes cardinal integers`() {
assertEquals("ноль один двадцать один", RussianNumberNormalizer.normalize("0 1 21"))
assertEquals("одна тысяча один", RussianNumberNormalizer.normalize("1001"))
assertEquals("две тысячи двадцать шесть", RussianNumberNormalizer.normalize("2026"))
assertEquals("двадцать две тысячи", RussianNumberNormalizer.normalize("22000"))
}
@Test
fun `preserves leading zeroes and reads decimal digits`() {
assertEquals("ноль ноль семь", RussianNumberNormalizer.normalize("007"))
assertEquals("минус двенадцать запятая ноль пять", RussianNumberNormalizer.normalize("-12,05"))
}
@Test
fun `normalizes number sign and non breaking groups`() {
assertEquals("номер пять", RussianNumberNormalizer.normalize("№ 5"))
assertEquals("двенадцать тысяч триста сорок пять", RussianNumberNormalizer.normalize("12\u00A0345"))
}
@Test
fun `does not rewrite digits inside words`() {
assertEquals("глава2 и abc123", RussianNumberNormalizer.normalize("глава2 и abc123"))
}
@Test
fun `inflects four digit years before году`() {
assertEquals(
"В две тысячи двадцать шестом году и в тысяча восемьсот девяносто седьмом году",
RussianNumberNormalizer.normalize("В 2026 году и в 1897 году")
)
assertEquals("В двухтысячном году", RussianNumberNormalizer.normalize("В 2000 году"))
}
}
@@ -0,0 +1,27 @@
package com.aletheia.app.ui.reader.tts
import org.junit.Assert.assertEquals
import org.junit.Test
class RussianSpeechTextSplitterTest {
@Test
fun keepsTerminalPunctuationAndDialogueDashInOneSpeechSentence() {
listOf(
"Стой. — сказал он.",
"Стой! — сказал он.",
"Стой? — спросил он.",
"«Стой!» — сказал он.",
"Стой? - спросил он."
).forEach { text ->
assertEquals(listOf(text), RussianSpeechTextSplitter.splitSentences(text))
}
}
@Test
fun stillSeparatesOrdinarySentences() {
assertEquals(
listOf("Первое предложение.", "Второе предложение!", "Третье?"),
RussianSpeechTextSplitter.splitSentences("Первое предложение. Второе предложение! Третье?")
)
}
}
+4
View File
@@ -38,6 +38,10 @@ Before accepting the model, synthesize a held-out Russian validation list at mul
`noise_scale`, and `noise_scale_w` settings, measure real-time factor on the target phone, and listen for skipped
words, unstable stress, clicks, and repeated phonemes. A successful export alone is not a quality gate.
The epoch-500 book-reading profile selected by listening comparison is stored in
`aletheia_book_profile.json`. Keep its duration, noise, and sentence-pause values together when evaluating the ONNX
model or integrating it into the reader.
On native Windows, first run the one-batch CUDA check:
```powershell
+50
View File
@@ -33,3 +33,53 @@ before producing WAV files.
`train_piper_windows.ps1` provides the equivalent native Windows path and a `-SmokeTest` mode that executes one
training and validation batch on CUDA before a long run. See `GPU_TRAINING.md` for host prerequisites and the
acceptance gate.
## Vosk BookTTS compression experiment
`vosk_booktts_experiment.py` reproduces the Android Vosk frontend, prepares real acoustic-model feeds from
`booktts_calibration_ru.txt`, performs calibrated QDQ int8 quantization of convolution layers, and renders
deterministic baseline/candidate WAV pairs. Keep generated `.npz`, candidate ONNX, and WAV files outside tracked
source directories, for example under `.codex-temp`. Do not replace the application asset until the candidate
loads with ONNX Runtime, runs on the target phone, and passes listening comparison.
Use `prepare --bert-model` plus `compare-feeds` to isolate a BERT replacement while keeping the acoustic model
identical in both WAV branches. `compare-feeds --resume` preserves existing WAVs and atomically updates its
report after every pair; `run_booktts_ab_windows.ps1` is the persistent Windows entry point.
`booktts_student.py` builds token-level teacher targets from the current int8 BERT and trains a four-layer,
256-dimensional Transformer student with a 768-dimensional drop-in output. Its small calibration corpus is only
for validating the training/export pipeline; a candidate for the application requires a much larger licensed
literary corpus, held-out evaluation, ONNX quantization, and listening tests. The optional
`--max-training-batches` and `--max-validation-batches` limits are for CUDA memory smoke tests only.
Each completed epoch atomically replaces `checkpoint.latest.pt`; pass `--resume` with the same output directory
to continue an interrupted run.
`run_booktts_student_windows.ps1` is the native Windows CUDA entry point for a full or resumed student run.
The exported student includes the dataset's original-to-compact token map so its `input_ids` remain compatible
with the existing Vosk vocabulary. `patch_booktts_student_token_map.py` adds the same lookup to an older export
without retraining.
`compare_booktts_bert.py` measures teacher/student and FP32/INT8 embedding error, inference time, and unseen-token
coverage on a held-out text file.
`build_booktts_prosody_corpus.py` creates a deterministic balance of questions, exclamations, quotations,
ellipsis, and neutral prose. Build its teacher dataset with `--reuse-token-map`, then fine-tune from the prior
`student.pt` with `--initial-model`; the held-out listening phrases must be passed through `--exclude`.
`run_booktts_prosody_dataset_windows.ps1` builds this reused-vocabulary teacher dataset persistently on Windows.
After auditing that dataset, `run_booktts_prosody_finetune_windows.ps1` starts the low-learning-rate CUDA run.
`extract_wikisource_corpus.py` streams an official Russian Wikisource XML/BZip2 dump and writes a deduplicated,
filtered sentence corpus plus provenance metadata. The source URL and dump size must be retained. Wikisource is
not a blanket rights clearance for every included work, so review the resulting corpus and applicable source
licenses before distributing a trained model.
`run_booktts_corpus_windows.ps1` completes the resumable BITS download on the CUDA host, verifies the official
dump size and SHA-1, extracts the configured number of sentences, and builds the streamed float16 teacher
dataset. It intentionally stops before training so vocabulary coverage and a held-out evaluation plan can be
reviewed first.
`audit_booktts_text_corpus.py` records the corpus SHA-256, sentence-length and word-count distributions,
residual wiki-markup counters, and deterministic review samples. Run it before training; its report is a
technical quality check and does not replace a rights review of the source works.
`audit_booktts_dataset.py` opens every generated teacher shard and verifies sample counts, tensor shapes,
token ranges, binary masks, and finite teacher values. It also prints a reproducible SHA-256 over shard bytes.
+12
View File
@@ -0,0 +1,12 @@
{
"id": "aletheia-book-moderate-v1",
"model_milestone": 500,
"length_scale": 1.5,
"noise_scale": 0.75,
"noise_w_scale": 0.95,
"sentence_silence_seconds": 0.25,
"selection": {
"method": "human_listening_test",
"result": "preferred_over_pause_only_and_stronger_variation"
}
}
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Validate every shard of a BookTTS teacher dataset."""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from pathlib import Path
import numpy as np
def audit(args: argparse.Namespace) -> None:
metadata_path = args.dataset / "dataset.json"
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
token_map = np.load(args.dataset / "token_id_map.npy")
samples = 0
tokens = 0
bytes_total = metadata_path.stat().st_size + (args.dataset / "token_id_map.npy").stat().st_size
digest = hashlib.sha256()
for record in metadata.get("shards", []):
path = args.dataset / record["file"]
digest.update(path.read_bytes())
bytes_total += path.stat().st_size
with np.load(path) as shard:
ids = shard["original_input_ids"]
mask = shard["attention_mask"]
teacher = shard["teacher"]
if ids.shape != mask.shape or teacher.shape[:2] != ids.shape or teacher.shape[2:] != (768,):
raise ValueError(f"Incompatible shapes in {path.name}: {ids.shape}, {mask.shape}, {teacher.shape}")
if ids.shape[0] != record["samples"]:
raise ValueError(f"Sample count mismatch in {path.name}")
if ids.size and (ids.min() < 0 or ids.max() >= token_map.shape[0]):
raise ValueError(f"Token id outside token map in {path.name}")
if not np.isfinite(teacher).all():
raise ValueError(f"Non-finite teacher value in {path.name}")
if not np.all((mask == 0) | (mask == 1)):
raise ValueError(f"Non-binary attention mask in {path.name}")
samples += ids.shape[0]
tokens += int(mask.sum())
if samples != metadata["samples"]:
raise ValueError(f"Dataset sample count is {samples}, metadata says {metadata['samples']}")
report = {
"dataset": str(args.dataset),
"samples": samples,
"shards": len(metadata.get("shards", [])),
"tokens": tokens,
"vocab_size": metadata["vocab_size"],
"token_map_entries": int(token_map.shape[0]),
"bytes": bytes_total,
"shards_sha256": digest.hexdigest().upper(),
"teacher_dtype": metadata.get("teacher_dtype"),
}
print(json.dumps(report, ensure_ascii=False, indent=2))
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--dataset", type=Path, required=True)
return parser.parse_args()
if __name__ == "__main__":
sys.exit(audit(parse_args()) or 0)
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
"""Audit a BookTTS text corpus and emit reproducible statistics and samples."""
from __future__ import annotations
import argparse
import hashlib
import json
import random
import re
import sys
from pathlib import Path
MARKUP_PATTERNS = ("{{", "}}", "[[", "]]", "<ref", "</", "{|", "|}", "http://", "https://")
RUSSIAN = re.compile(r"[А-Яа-яЁё]")
LETTERS = re.compile(r"[^\W\d_]", re.UNICODE)
def percentile(sorted_values: list[int], fraction: float) -> int:
if not sorted_values:
return 0
return sorted_values[round((len(sorted_values) - 1) * fraction)]
def audit(args: argparse.Namespace) -> None:
rng = random.Random(args.seed)
lengths: list[int] = []
words: list[int] = []
samples: list[str] = []
suspicious = {pattern: 0 for pattern in MARKUP_PATTERNS}
low_russian_ratio = 0
with_digits = 0
with_dialogue_dash = 0
digest = hashlib.sha256()
with args.corpus.open("rb") as binary:
for raw in binary:
digest.update(raw)
line = raw.decode("utf-8").rstrip("\r\n")
index = len(lengths)
lengths.append(len(line)); words.append(len(line.split()))
if any(char.isdigit() for char in line):
with_digits += 1
if line.startswith(("", "", "-")):
with_dialogue_dash += 1
letters = LETTERS.findall(line)
russian = RUSSIAN.findall(line)
if letters and len(russian) / len(letters) < 0.75:
low_russian_ratio += 1
for pattern in MARKUP_PATTERNS:
if pattern in line:
suspicious[pattern] += 1
if len(samples) < args.samples:
samples.append(line)
else:
replacement = rng.randint(0, index)
if replacement < args.samples:
samples[replacement] = line
ordered_lengths = sorted(lengths); ordered_words = sorted(words)
report = {
"corpus": str(args.corpus), "sha256": digest.hexdigest().upper(), "lines": len(lengths),
"bytes": args.corpus.stat().st_size,
"characters": {"min": min(lengths, default=0), "mean": sum(lengths) / len(lengths) if lengths else 0, "p50": percentile(ordered_lengths, 0.50), "p95": percentile(ordered_lengths, 0.95), "max": max(lengths, default=0)},
"words": {"min": min(words, default=0), "mean": sum(words) / len(words) if words else 0, "p50": percentile(ordered_words, 0.50), "p95": percentile(ordered_words, 0.95), "max": max(words, default=0)},
"with_digits": with_digits, "with_dialogue_dash": with_dialogue_dash,
"below_75_percent_russian_letters": low_russian_ratio, "suspicious_markup": suspicious,
"sample_seed": args.seed, "samples": samples,
}
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.output:
if args.output.exists():
raise FileExistsError(f"Refusing to overwrite {args.output}")
args.output.write_text(rendered, encoding="utf-8")
print(rendered, end="")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--corpus", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--samples", type=int, default=20)
parser.add_argument("--seed", type=int, default=20260721)
return parser.parse_args()
if __name__ == "__main__":
sys.exit(audit(parse_args()) or 0)
+100
View File
@@ -0,0 +1,100 @@
— Что-то случилось? — тихо спросил он, остановившись у двери.
Кое-кто торопится, кое-что забывает, а кто-то всё-таки возвращается.
«Послушай, — сказала Ксения, — нам необходимо поговорить серьёзно».
Он открыл письмо и прочитал: решение принято; отступать уже поздно.
В комнате было тихо... Слишком тихо, чтобы не заметить чужого дыхания.
Почему вы молчите? Неужели ответ оказался настолько неожиданным?
Стой! Не делай ни шага, пока я не объясню происходящее.
На столе лежали старые часы (они давно остановились), ключ и записка.
Северо-западный ветер постепенно стих, но море оставалось беспокойным.
Едва заметная улыбка появилась на её лице и сразу исчезла.
Во-первых, нужно проверить дорогу; во-вторых, дождаться остальных.
Он говорил медленно, отчётливо и спокойно, не проглатывая окончания слов.
Слова «замок», «мука» и «атлас» меняют ударение вместе со значением.
У ворот стоял старый замок, а вдали виднелся заброшенный замок.
В 2026 году экспедиция прошла 12 345 километров за 21 день.
Температура опустилась до -12,05 градуса, а давление выросло на +7 единиц.
Поезд номер 5 отправляется в 07:30; посадка закончится через 10 минут.
Глава вторая. Ночная встреча у реки.
Господин Петров, ул. Лесная, дом 8 — так было написано на конверте.
Когда предложение переходит на следующую страницу, оно должно звучать непрерывно.
Рассказчик сделал короткую паузу — ровно настолько, чтобы сохранить напряжение.
После двоеточия пауза короче: мысль продолжается без завершения фразы.
После точки, вопросительного и восклицательного знаков нужна ясная граница.
Чёткая дикция важнее скорости, особенно в длинных книжных предложениях.
За окном, где ещё недавно шумел дождь, медленно светлело утреннее небо.
— Подождите меня здесь, — произнёс проводник, — и никуда не уходите.
Она хотела возразить, однако передумала: спор всё равно ничего бы не изменил.
Если дверь заперта, постучи трижды; если никто не ответит — возвращайся.
Где-то далеко прокричала ночная птица, и лес снова погрузился в тишину.
Как странно: знакомый дом казался теперь меньше, темнее и гораздо старше.
«Я вернусь до рассвета!» — крикнул он и исчез за поворотом.
Неужели всё это — лишь совпадение, которому мы напрасно придали значение?
Ветер перебирал сухие листья — шорох за шорохом, вздох за вздохом.
Она подняла глаза (в них блестели слёзы), но голос её остался твёрдым.
Старик помолчал; затем, словно решившись, заговорил совсем другим тоном.
Ни света, ни звука, ни единого следа — только холодная пустая дорога.
Прежде чем ответить, он медленно сложил письмо и убрал его во внутренний карман.
— Вы уверены? — Да. — Тогда начинайте, времени у нас почти не осталось.
По ту сторону реки поднимался древний лес, окутанный сизым предрассветным туманом.
Всё-таки, по-моему, кто-нибудь должен был предупредить нас заранее.
Из-за полуоткрытой двери донёсся чей-то приглушённый, едва различимый голос.
В письме стояло одно слово: «Жди»; ни подписи, ни даты не было.
Она замерла на полуслове, будто внезапно услышала то, чего не слышали остальные.
Гром прогремел совсем близко — окна задрожали, а свеча внезапно погасла.
Мальчик нёс в руках старинный атлас, а на плечах у него лежал атласный плащ.
Мука́ закончилась к полудню, но му́ка ожидания продолжалась до самого вечера.
На берегу белели старые и́рисы, а в письме лежал неоплаченный ири́с.
Острый клино́к сверкнул в темноте, и сухой кли́нок дерева хрустнул под ногой.
Мы обошли весь квартал, но нужный дом так и не нашли.
Он поставил подпись под договором, хотя каждый пункт вызывал у него сомнения.
В первой главе герой уезжает; во второй — возвращается под чужим именем.
Запомни главное: нельзя перебивать фразу только потому, что закончилась страница.
Последние слова предложения должны прозвучать на следующем экране без повторения начала.
Она читала неторопливо, оставляя между словами едва заметное пространство.
Слишком быстрый темп съедает согласные, а чрезмерно медленный разрушает смысл фразы.
На отметке 3,5 километра дорога раздваивается: налево — к озеру, направо — к селу.
В архиве значились тома № 7, 12 и 18; восьмого тома в описи не было.
Экспедиция началась 14 июля 1897 года и завершилась лишь через восемь месяцев.
В 6 часов 45 минут колокол ударил дважды, хотя должен был ударить шесть раз.
Расстояние составляло около 1 250 метров, то есть чуть больше одной версты.
Цена книги — 19 рублей 90 копеек; на обороте карандашом написано: «Не продавать».
Глава двенадцатая. Письмо, которого никто не ждал.
Часть III. Возвращение домой после двадцати лет странствий.
— Кто там? — Это я. — Кто «я»? — Откройте, и вы всё поймёте.
«Нет, нет и ещё раз нет», — отчётливо повторила она.
Он прошептал: «Тише… нас могут услышать», — и указал на окно.
Сначала послышались шаги; потом скрипнула лестница; наконец открылась дверь.
Дорога была трудна: снег слепил глаза, ветер сбивал с ног, силы иссякали.
Он не ответил — не потому, что не знал ответа, а потому, что боялся его произнести.
Кто бы мог подумать, что маленькая находка изменит судьбу целого города!
Когда часы пробили полночь, незнакомец снял шляпу и назвал своё настоящее имя.
Утром всё выглядело обыкновенно; только следы у калитки напоминали о ночном госте.
Я хотел было уйти, но она сказала: «Останьтесь ещё на одну минуту».
Голос звучал спокойно, почти равнодушно, и от этого становилось ещё тревожнее.
Вдали показалась станция — низкая платформа, жёлтый фонарь и одинокий смотритель.
Он читал старую рукопись строка за строкой, боясь пропустить хотя бы одну букву.
Некоторые слова были стёрты; другие — исправлены чужой рукой много лет спустя.
В примечании значилось: см. главу 4, стр. 217, абзац второй.
На табличке было написано: «Вход воспрещён с 22:00 до 06:00».
Температура воды равнялась 18,4 °C, скорость течения — 2,1 метра в секунду.
Вероятность ошибки оценили в 0,03 процента, но случай всё-таки произошёл.
Координаты точки: 55°45′ северной широты, 37°37′ восточной долготы.
В комнате находились А. П. Чехов, Л. Н. Толстой и ещё двое гостей.
Иван Сергеевич, будьте добры, прочтите последний абзац ещё раз.
Мать-и-мачеха росла у дороги, а где-то рядом кричала птица-пересмешник.
Поезд шёл с северо-востока на юго-запад, постепенно набирая скорость.
Что-либо менять было поздно; кое-как собрав вещи, они отправились в путь.
Кто-нибудь видел серо-зелёную папку, лежавшую здесь полчаса назад?
Из-под земли доносился глухой рокот, то усиливаясь, то почти исчезая.
Он всё повторял одно и то же, словно заученное заклинание: «Нельзя опаздывать».
Свет погас на мгновение — и именно в это мгновение картина исчезла со стены.
Слово за словом, страница за страницей перед читателем раскрывалась чужая жизнь.
Её ответ был прост, но окончателен: она не вернётся ни завтра, ни через год.
Вопрос состоял не в том, кто виноват, а в том, можно ли ещё что-нибудь исправить.
Не открывая глаз, он прислушался: рядом потрескивал огонь, за стеной шумела вода.
«Вы опоздали на семь минут», — заметил человек в сером пальто.
На мгновение ей показалось, будто портрет улыбнулся; разумеется, этого не могло быть.
Туман рассеивался медленно, открывая то крышу, то колокольню, то дальний берег.
Никто не произнёс ни слова — прощание и без того затянулось.
И всё же где-то в глубине души оставалась слабая, почти невозможная надежда.
+4
View File
@@ -0,0 +1,4 @@
Почему вы молчите? Неужели ответ оказался настолько неожиданным?
В 2026 году экспедиция прошла 12 345 километров за 21 день.
Мука́ закончилась к полудню, но му́ка ожидания продолжалась до самого вечера.
Мать-и-мачеха росла у дороги, а где-то рядом кричала птица-пересмешник.
+1
View File
@@ -0,0 +1 @@
Мука́ закончилась к полудню, но му́ка ожидания продолжалась до самого вечера.
+5
View File
@@ -0,0 +1,5 @@
Почему вы молчите? Неужели ответ оказался настолько неожиданным?
— Что-то случилось? — тихо спросил он, остановившись у двери.
Чёткая дикция важнее скорости, особенно в длинных книжных предложениях.
Неужели всё это — лишь совпадение, которому мы напрасно придали значение?
И всё же где-то в глубине души оставалась слабая, почти невозможная надежда.
@@ -0,0 +1,3 @@
— Что-то случилось?
Почему вы молчите?
Неужели ответ оказался настолько неожиданным?
+435
View File
@@ -0,0 +1,435 @@
#!/usr/bin/env python3
"""Build and train a compact drop-in BERT student for Aletheia BookTTS."""
from __future__ import annotations
import argparse
import json
import random
import shutil
import sys
import time
from collections import Counter
from pathlib import Path
import numpy as np
from vosk_booktts_experiment import WordPieceTokenizer, load_corpus, normalize_text
def build_dataset(args: argparse.Namespace) -> None:
import onnxruntime as ort
args.output.mkdir(parents=True, exist_ok=False)
tokenizer = WordPieceTokenizer(args.assets / "vocab.txt")
options = ort.SessionOptions()
options.log_severity_level = 3
teacher = ort.InferenceSession(
str(args.assets / "bert.int8.onnx"),
sess_options=options,
providers=["CPUExecutionProvider"],
)
token_counts: Counter[int] = Counter()
pending: list[tuple[str, np.ndarray]] = []
shard_samples: list[tuple[np.ndarray, np.ndarray]] = []
shard_records = []
sample_index = 0
shard_index = 0
teacher_dtype = np.float16 if args.teacher_dtype == "float16" else np.float32
def flush_shard() -> None:
nonlocal shard_index
if not shard_samples:
return
longest = max(ids.size for ids, _ in shard_samples)
shard_ids = np.zeros((len(shard_samples), longest), dtype=np.int64)
shard_mask = np.zeros_like(shard_ids)
shard_teacher = np.zeros((len(shard_samples), longest, 768), dtype=teacher_dtype)
for row, (ids, embeddings) in enumerate(shard_samples):
shard_ids[row, : ids.size] = ids
shard_mask[row, : ids.size] = 1
shard_teacher[row, : ids.size] = embeddings
filename = f"shard-{shard_index:05d}.npz"
np.savez(
args.output / filename,
original_input_ids=shard_ids,
attention_mask=shard_mask,
teacher=shard_teacher,
)
shard_records.append({"file": filename, "samples": len(shard_samples), "max_tokens": longest})
shard_samples.clear()
shard_index += 1
def flush_batch(manifest) -> None:
nonlocal sample_index
if not pending:
return
longest = max(ids.size for _, ids in pending)
batch_ids = np.zeros((len(pending), longest), dtype=np.int64)
batch_mask = np.zeros_like(batch_ids)
for row, (_, ids) in enumerate(pending):
batch_ids[row, : ids.size] = ids
batch_mask[row, : ids.size] = 1
outputs = teacher.run(None, {
"input_ids": batch_ids,
"attention_mask": batch_mask,
"token_type_ids": np.zeros_like(batch_ids),
})[0]
if outputs.ndim == 2:
outputs = outputs[np.newaxis, :, :]
for row, (text, ids) in enumerate(pending):
embeddings = outputs[row, : ids.size].astype(teacher_dtype, copy=False)
if embeddings.shape != (ids.size, 768):
raise ValueError(f"Unexpected teacher shape {embeddings.shape}")
shard_samples.append((ids.copy(), embeddings.copy()))
manifest.write(json.dumps({"sample": sample_index, "text": text, "tokens": int(ids.size)}, ensure_ascii=False) + "\n")
sample_index += 1
if len(shard_samples) >= args.shard_size:
flush_shard()
pending.clear()
with (args.output / "manifest.jsonl").open("w", encoding="utf-8", newline="\n") as manifest:
for text in load_corpus(args.corpus):
ids, _ = tokenizer.encode(normalize_text(text))
token_counts.update(int(value) for value in ids)
pending.append((text, ids))
if len(pending) >= args.teacher_batch_size:
flush_batch(manifest)
if sample_index % 1_000 == 0:
print(f"teacher_samples={sample_index}", flush=True)
flush_batch(manifest)
flush_shard()
observed = set(token_counts)
if args.reuse_token_map:
source_metadata = json.loads((args.reuse_token_map / "dataset.json").read_text(encoding="utf-8"))
source_map = np.load(args.reuse_token_map / "token_id_map.npy")
if source_map.size != len(tokenizer.tokens):
raise ValueError("Reused token map and teacher vocabulary have different sizes")
shutil.copy2(args.reuse_token_map / "token_id_map.npy", args.output / "token_id_map.npy")
shutil.copy2(args.reuse_token_map / "vocab.txt", args.output / "vocab.txt")
vocab_size = int(source_metadata["vocab_size"])
unknown_new = int(source_map[tokenizer.vocabulary["[UNK]"]])
replaced_occurrences = sum(
count for token, count in token_counts.items()
if token != tokenizer.vocabulary["[UNK]"] and int(source_map[token]) == unknown_new
)
else:
special_ids = [tokenizer.vocabulary[name] for name in ("[PAD]", "[UNK]", "[CLS]", "[SEP]")]
if args.max_vocab and len(observed) > args.max_vocab:
retained = set(special_ids)
retained.update(token for token, _ in token_counts.most_common(args.max_vocab - len(retained)))
else:
retained = observed | set(special_ids)
ordered_old_ids = special_ids + sorted(retained - set(special_ids))
old_to_new = {old: new for new, old in enumerate(ordered_old_ids)}
unknown_new = old_to_new[tokenizer.vocabulary["[UNK]"]]
reduced_tokens = [tokenizer.tokens[index] for index in ordered_old_ids]
(args.output / "vocab.txt").write_text("\n".join(reduced_tokens) + "\n", encoding="utf-8")
token_id_map = np.full(len(tokenizer.tokens), unknown_new, dtype=np.int64)
for old, new in old_to_new.items():
token_id_map[old] = new
np.save(args.output / "token_id_map.npy", token_id_map)
vocab_size = len(reduced_tokens)
replaced_occurrences = sum(count for token, count in token_counts.items() if token not in retained)
metadata = {
"samples": sample_index,
"vocab_size": vocab_size,
"observed_original_tokens": len(observed),
"replaced_token_occurrences": replaced_occurrences,
"reused_token_map": str(args.reuse_token_map) if args.reuse_token_map else None,
"teacher_dtype": args.teacher_dtype,
"teacher_batch_size": args.teacher_batch_size,
"shard_size": args.shard_size,
"shards": shard_records,
}
(args.output / "dataset.json").write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(metadata, indent=2))
def train_student(args: argparse.Namespace) -> None:
import torch
from torch import nn
from torch.utils.data import DataLoader, Dataset, IterableDataset, Subset
class DistillationDataset(Dataset):
def __init__(self, root: Path):
self.files = sorted(root.glob("sample-*.npz"))
if not self.files:
raise ValueError(f"No samples in {root}")
mapping = root / "token_id_map.npy"
self.token_id_map = np.load(mapping) if mapping.exists() else None
def __len__(self) -> int:
return len(self.files)
def __getitem__(self, index: int) -> tuple[torch.Tensor, torch.Tensor]:
with np.load(self.files[index]) as data:
if "original_input_ids" in data:
original = data["original_input_ids"]
if self.token_id_map is None:
raise ValueError("token_id_map.npy is required for original_input_ids")
input_ids = self.token_id_map[original]
else:
input_ids = data["input_ids"]
teacher = data["teacher"].astype(np.float32)
return torch.from_numpy(input_ids.copy()), torch.from_numpy(teacher.copy())
class ShardedDataset(IterableDataset):
def __init__(self, root: Path, records: list[dict], shuffle: bool, seed: int):
self.root = root
self.records = list(records)
self.shuffle = shuffle
self.seed = seed
self.epoch = 0
self.token_id_map = np.load(root / "token_id_map.npy")
def __len__(self) -> int:
return sum(record["samples"] for record in self.records)
def set_epoch(self, epoch: int) -> None:
self.epoch = epoch
def __iter__(self):
rng = random.Random(self.seed + self.epoch)
records = list(self.records)
if self.shuffle:
rng.shuffle(records)
for record in records:
with np.load(self.root / record["file"]) as data:
original = data["original_input_ids"]
masks = data["attention_mask"]
teachers = data["teacher"]
rows = list(range(original.shape[0]))
if self.shuffle:
rng.shuffle(rows)
for row in rows:
length = int(masks[row].sum())
ids = self.token_id_map[original[row, :length]]
teacher = teachers[row, :length].astype(np.float32)
yield torch.from_numpy(ids.copy()), torch.from_numpy(teacher.copy())
def collate(batch):
longest = max(ids.size(0) for ids, _ in batch)
ids = torch.zeros((len(batch), longest), dtype=torch.long)
mask = torch.zeros((len(batch), longest), dtype=torch.long)
targets = torch.zeros((len(batch), longest, 768), dtype=torch.float32)
for row, (sample_ids, teacher) in enumerate(batch):
length = sample_ids.size(0)
ids[row, :length] = sample_ids
mask[row, :length] = 1
targets[row, :length] = teacher
return ids, mask, targets
class Student(nn.Module):
def __init__(self, vocab_size: int):
super().__init__()
self.token_embedding = nn.Embedding(vocab_size, args.hidden, padding_idx=0)
self.position_embedding = nn.Embedding(args.max_length, args.hidden)
self.type_embedding = nn.Embedding(2, args.hidden)
layer = nn.TransformerEncoderLayer(
d_model=args.hidden, nhead=args.heads, dim_feedforward=args.feed_forward,
dropout=args.dropout, activation="gelu", batch_first=True, norm_first=True,
)
self.encoder = nn.TransformerEncoder(layer, num_layers=args.layers, enable_nested_tensor=False)
self.norm = nn.LayerNorm(args.hidden)
self.projection = nn.Linear(args.hidden, 768)
def forward(self, input_ids, attention_mask, token_type_ids=None):
if token_type_ids is None:
token_type_ids = torch.zeros_like(input_ids)
positions = torch.arange(input_ids.size(1), device=input_ids.device).unsqueeze(0)
values = (
self.token_embedding(input_ids)
+ self.position_embedding(positions)
+ self.type_embedding(token_type_ids)
)
values = self.encoder(values, src_key_padding_mask=attention_mask == 0)
return self.projection(self.norm(values))
class ExportWrapper(nn.Module):
def __init__(self, model, token_id_map):
super().__init__(); self.model = model
self.register_buffer("token_id_map", token_id_map)
def forward(self, input_ids, attention_mask, token_type_ids):
mapped_input_ids = self.token_id_map[input_ids]
return self.model(mapped_input_ids, attention_mask, token_type_ids)[0]
random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
metadata = json.loads((args.dataset / "dataset.json").read_text(encoding="utf-8"))
model = Student(metadata["vocab_size"]).to(device)
if args.initial_model:
payload = torch.load(args.initial_model, map_location=device, weights_only=False)
state_dict = payload.get("state_dict") or payload.get("best_state") or payload.get("model")
if state_dict is None:
raise ValueError(f"No model state in {args.initial_model}")
model.load_state_dict(state_dict)
if metadata.get("shards"):
records = list(metadata["shards"])
rng = random.Random(args.seed); rng.shuffle(records)
validation_shards = max(1, round(len(records) * args.validation_fraction)) if len(records) > 1 else 0
validation_records = records[:validation_shards]
training_records = records[validation_shards:] or records
training_data = ShardedDataset(args.dataset, training_records, True, args.seed)
validation_data = ShardedDataset(args.dataset, validation_records, False, args.seed) if validation_records else None
loader = DataLoader(training_data, batch_size=args.batch_size, collate_fn=collate)
validation_loader = DataLoader(validation_data, batch_size=args.batch_size, collate_fn=collate) if validation_data else None
else:
all_data = DistillationDataset(args.dataset)
validation_size = max(1, round(len(all_data) * args.validation_fraction)) if len(all_data) > 1 else 0
indices = list(range(len(all_data))); random.Random(args.seed).shuffle(indices)
validation_data = Subset(all_data, indices[:validation_size]) if validation_size else None
training_data = Subset(all_data, indices[validation_size:] or indices)
loader = DataLoader(training_data, batch_size=args.batch_size, shuffle=True, collate_fn=collate)
validation_loader = DataLoader(validation_data, batch_size=args.batch_size, collate_fn=collate) if validation_data else None
optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=0.01)
scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda")
checkpoint_path = args.output / "checkpoint.latest.pt"
if args.resume:
if not checkpoint_path.exists():
raise FileNotFoundError(f"Cannot resume without {checkpoint_path}")
else:
args.output.mkdir(parents=True, exist_ok=False)
started = time.time(); history = []; best_loss = float("inf"); best_state = None; best_epoch = 0; stale_epochs = 0
start_epoch = 1
if args.resume:
checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=False)
model.load_state_dict(checkpoint["model"])
optimizer.load_state_dict(checkpoint["optimizer"])
scaler.load_state_dict(checkpoint["scaler"])
history = checkpoint["history"]
best_loss = checkpoint["best_loss"]
best_state = checkpoint["best_state"]
best_epoch = checkpoint["best_epoch"]
stale_epochs = checkpoint["stale_epochs"]
start_epoch = checkpoint["epoch"] + 1
def calculate_loss(prediction, target, mask):
active = mask.bool().unsqueeze(-1).expand_as(prediction)
mse = torch.mean((prediction[active] - target[active]) ** 2)
cosine = 1.0 - torch.nn.functional.cosine_similarity(prediction[mask.bool()], target[mask.bool()], dim=-1).mean()
return mse + args.cosine_weight * cosine
for epoch in range(start_epoch, args.epochs + 1):
if isinstance(training_data, ShardedDataset):
training_data.set_epoch(epoch)
model.train(); total = 0.0; batches = 0
for ids, mask, target in loader:
ids, mask, target = ids.to(device), mask.to(device), target.to(device)
optimizer.zero_grad(set_to_none=True)
with torch.amp.autocast("cuda", dtype=torch.float16, enabled=device.type == "cuda"):
prediction = model(ids, mask)
loss = calculate_loss(prediction, target, mask)
scaler.scale(loss).backward(); scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
scaler.step(optimizer); scaler.update()
total += float(loss.detach()); batches += 1
if args.max_training_batches and batches >= args.max_training_batches:
break
training_loss = total / batches
validation_loss = training_loss
if validation_loader is not None:
model.eval(); validation_total = 0.0; validation_batches = 0
with torch.no_grad():
for ids, mask, target in validation_loader:
ids, mask, target = ids.to(device), mask.to(device), target.to(device)
with torch.amp.autocast("cuda", dtype=torch.float16, enabled=device.type == "cuda"):
validation_total += float(calculate_loss(model(ids, mask), target, mask))
validation_batches += 1
if args.max_validation_batches and validation_batches >= args.max_validation_batches:
break
validation_loss = validation_total / validation_batches
history.append({"epoch": epoch, "training_loss": training_loss, "validation_loss": validation_loss})
print(f"epoch={epoch} training_loss={training_loss:.8f} validation_loss={validation_loss:.8f}")
if validation_loss < best_loss:
best_loss = validation_loss; best_epoch = epoch; stale_epochs = 0
best_state = {name: value.detach().cpu().clone() for name, value in model.state_dict().items()}
else:
stale_epochs += 1
checkpoint = {
"epoch": epoch, "model": model.state_dict(), "optimizer": optimizer.state_dict(),
"scaler": scaler.state_dict(), "history": history, "best_loss": best_loss,
"best_state": best_state, "best_epoch": best_epoch, "stale_epochs": stale_epochs,
}
temporary_checkpoint = checkpoint_path.with_suffix(".tmp")
torch.save(checkpoint, temporary_checkpoint); temporary_checkpoint.replace(checkpoint_path)
if stale_epochs >= args.early_stopping_patience:
print(f"early_stop epoch={epoch} best_epoch={best_epoch}")
break
if best_state is not None:
model.load_state_dict(best_state)
config = {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items() if key != "handler"}
torch.save({"state_dict": model.state_dict(), "metadata": metadata, "config": config}, args.output / "student.pt")
model.eval()
export_token_map = torch.from_numpy(np.load(args.dataset / "token_id_map.npy")).long()
wrapper = ExportWrapper(model, export_token_map).cpu().eval()
example_ids = torch.tensor([[2, 3]], dtype=torch.long)
example_mask = torch.ones_like(example_ids); example_types = torch.zeros_like(example_ids)
sequence = torch.export.Dim("sequence", min=2, max=args.max_length)
torch.onnx.export(
wrapper, (example_ids, example_mask, example_types), args.output / "bert.student.fp32.onnx",
input_names=["input_ids", "attention_mask", "token_type_ids"], output_names=["logits"],
dynamic_shapes=({1: sequence}, {1: sequence}, {1: sequence}),
opset_version=18, dynamo=True, external_data=False,
)
report = {
"device": str(device), "parameters": sum(parameter.numel() for parameter in model.parameters()),
"training_samples": len(training_data), "validation_samples": len(validation_data) if validation_data is not None else 0,
"best_epoch": best_epoch, "best_validation_loss": best_loss,
"peak_cuda_bytes": torch.cuda.max_memory_allocated() if device.type == "cuda" else 0,
"seconds": time.time() - started, "onnx_bytes": (args.output / "bert.student.fp32.onnx").stat().st_size,
"history": history, "training_batches_last_epoch": batches,
"validation_batches_last_epoch": validation_batches if validation_loader is not None else 0,
}
(args.output / "training.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
print(json.dumps({key: value for key, value in report.items() if key != "history"}, indent=2))
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
commands = parser.add_subparsers(dest="command", required=True)
dataset = commands.add_parser("dataset")
dataset.add_argument("--assets", type=Path, required=True)
dataset.add_argument("--corpus", type=Path, required=True)
dataset.add_argument("--output", type=Path, required=True)
dataset.add_argument("--max-vocab", type=int, default=0)
dataset.add_argument("--teacher-batch-size", type=int, default=16)
dataset.add_argument("--teacher-dtype", choices=("float16", "float32"), default="float16")
dataset.add_argument("--shard-size", type=int, default=256)
dataset.add_argument("--reuse-token-map", type=Path)
dataset.set_defaults(handler=build_dataset)
train = commands.add_parser("train")
train.add_argument("--dataset", type=Path, required=True)
train.add_argument("--output", type=Path, required=True)
train.add_argument("--epochs", type=int, default=10)
train.add_argument("--batch-size", type=int, default=8)
train.add_argument("--hidden", type=int, default=256)
train.add_argument("--heads", type=int, default=8)
train.add_argument("--feed-forward", type=int, default=768)
train.add_argument("--layers", type=int, default=4)
train.add_argument("--max-length", type=int, default=256)
train.add_argument("--dropout", type=float, default=0.1)
train.add_argument("--learning-rate", type=float, default=3e-4)
train.add_argument("--cosine-weight", type=float, default=0.1)
train.add_argument("--validation-fraction", type=float, default=0.05)
train.add_argument("--early-stopping-patience", type=int, default=5)
train.add_argument("--max-training-batches", type=int, default=0)
train.add_argument("--max-validation-batches", type=int, default=0)
train.add_argument("--resume", action="store_true")
train.add_argument("--initial-model", type=Path)
train.add_argument("--seed", type=int, default=20260721)
train.set_defaults(handler=train_student)
return parser.parse_args()
def main() -> int:
args = parse_args(); args.handler(args); return 0
if __name__ == "__main__":
sys.exit(main())
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Build a deterministic punctuation-balanced fine-tuning corpus."""
from __future__ import annotations
import argparse
import hashlib
import heapq
import json
import sys
from pathlib import Path
CATEGORIES = {
"question": lambda text: "?" in text,
"exclamation": lambda text: "!" in text,
"quotes": lambda text: "«" in text or "»" in text,
"ellipsis": lambda text: "" in text or "..." in text,
"neutral": lambda text: not any(mark in text for mark in ("?", "!", "«", "»", "", "...")),
}
def build(args: argparse.Namespace) -> None:
if args.output.exists() or args.metadata.exists():
raise FileExistsError("Refusing to overwrite output or metadata")
excluded = set()
if args.exclude:
excluded = {line.strip() for line in args.exclude.read_text(encoding="utf-8-sig").splitlines() if line.strip()}
heaps: dict[str, list[tuple[int, str]]] = {name: [] for name in CATEGORIES}
scanned = 0
for text in args.source.read_text(encoding="utf-8").splitlines():
text = text.strip()
if not text or text in excluded:
continue
scanned += 1
for name, predicate in CATEGORIES.items():
if not predicate(text):
continue
score = int.from_bytes(hashlib.blake2b(f"{name}\0{text}".encode("utf-8"), digest_size=8).digest(), "big")
heap = heaps[name]
if len(heap) < args.per_category:
heapq.heappush(heap, (-score, text))
elif score < -heap[0][0]:
heapq.heapreplace(heap, (-score, text))
selected: dict[str, set[str]] = {
name: {text for _, text in heap} for name, heap in heaps.items()
}
combined = sorted(set().union(*selected.values()), key=lambda text: hashlib.sha256(text.encode("utf-8")).digest())
args.output.write_text("\n".join(combined) + "\n", encoding="utf-8")
metadata = {
"source": str(args.source), "exclude": str(args.exclude) if args.exclude else None,
"scanned": scanned, "per_category_limit": args.per_category,
"category_counts": {name: len(values) for name, values in selected.items()},
"unique_sentences": len(combined), "output_bytes": args.output.stat().st_size,
}
args.metadata.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(metadata, ensure_ascii=False, indent=2))
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--exclude", type=Path)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--metadata", type=Path, required=True)
parser.add_argument("--per-category", type=int, default=4000)
return parser.parse_args()
if __name__ == "__main__":
sys.exit(build(parse_args()) or 0)
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Compare teacher, FP32 student, and INT8 student embeddings on real text."""
from __future__ import annotations
import argparse
import json
import math
import sys
import time
from pathlib import Path
import numpy as np
import onnxruntime as ort
from vosk_booktts_experiment import WordPieceTokenizer, load_corpus, normalize_text
def cosine_rows(left: np.ndarray, right: np.ndarray) -> np.ndarray:
numerator = np.sum(left * right, axis=-1)
denominator = np.linalg.norm(left, axis=-1) * np.linalg.norm(right, axis=-1)
return numerator / np.maximum(denominator, 1e-12)
def compare(args: argparse.Namespace) -> None:
tokenizer = WordPieceTokenizer(args.vocab)
token_map = np.load(args.token_map)
compact_unk = int(token_map[tokenizer.vocabulary["[UNK]"]])
sessions = {
"teacher": ort.InferenceSession(str(args.teacher), providers=["CPUExecutionProvider"]),
"student_fp32": ort.InferenceSession(str(args.student_fp32), providers=["CPUExecutionProvider"]),
"student_int8": ort.InferenceSession(str(args.student_int8), providers=["CPUExecutionProvider"]),
}
timings = {name: 0.0 for name in sessions}
teacher_cosines: list[np.ndarray] = []
quantized_cosines: list[np.ndarray] = []
teacher_squared_error = 0.0
quantized_squared_error = 0.0
elements = 0
tokens = 0
unseen_tokens = 0
phrases = load_corpus(args.corpus)[: args.limit or None]
for text in phrases:
ids, _ = tokenizer.encode(normalize_text(text))
shape = (1, ids.size)
feeds = {
"input_ids": ids.reshape(shape),
"attention_mask": np.ones(shape, dtype=np.int64),
"token_type_ids": np.zeros(shape, dtype=np.int64),
}
outputs: dict[str, np.ndarray] = {}
for name, session in sessions.items():
started = time.perf_counter()
outputs[name] = session.run(None, feeds)[0].astype(np.float32, copy=False)
timings[name] += time.perf_counter() - started
teacher = outputs["teacher"]
fp32 = outputs["student_fp32"]
int8 = outputs["student_int8"]
if teacher.shape != fp32.shape or fp32.shape != int8.shape:
raise ValueError(f"Output shape mismatch for {text!r}: {teacher.shape}, {fp32.shape}, {int8.shape}")
teacher_cosines.append(cosine_rows(teacher, fp32))
quantized_cosines.append(cosine_rows(fp32, int8))
teacher_squared_error += float(np.sum((teacher - fp32) ** 2))
quantized_squared_error += float(np.sum((fp32 - int8) ** 2))
elements += teacher.size
tokens += ids.size
unseen_tokens += int(np.sum((token_map[ids] == compact_unk) & (ids != tokenizer.vocabulary["[UNK]"])))
teacher_cosine = np.concatenate(teacher_cosines)
quantized_cosine = np.concatenate(quantized_cosines)
report = {
"phrases": len(phrases),
"tokens": tokens,
"unseen_tokens": unseen_tokens,
"unseen_percent": 100.0 * unseen_tokens / tokens,
"teacher_vs_student_fp32": {
"mean_token_cosine": float(np.mean(teacher_cosine)),
"p05_token_cosine": float(np.percentile(teacher_cosine, 5)),
"mse": teacher_squared_error / elements,
},
"student_fp32_vs_int8": {
"mean_token_cosine": float(np.mean(quantized_cosine)),
"p05_token_cosine": float(np.percentile(quantized_cosine, 5)),
"mse": quantized_squared_error / elements,
},
"total_inference_seconds": timings,
"milliseconds_per_phrase": {name: seconds * 1000.0 / len(phrases) for name, seconds in timings.items()},
}
if not all(math.isfinite(value) for value in (teacher_squared_error, quantized_squared_error)):
raise ValueError("Non-finite comparison result")
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.output:
if args.output.exists():
raise FileExistsError(f"Refusing to overwrite {args.output}")
args.output.write_text(rendered, encoding="utf-8")
print(rendered, end="")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--teacher", type=Path, required=True)
parser.add_argument("--student-fp32", type=Path, required=True)
parser.add_argument("--student-int8", type=Path, required=True)
parser.add_argument("--vocab", type=Path, required=True)
parser.add_argument("--token-map", type=Path, required=True)
parser.add_argument("--corpus", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--limit", type=int, default=0)
return parser.parse_args()
if __name__ == "__main__":
sys.exit(compare(parse_args()) or 0)
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Extract clean Russian sentences from an official Wikisource XML dump."""
from __future__ import annotations
import argparse
import bz2
import heapq
import hashlib
import json
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree
SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?…])\s+")
WHITESPACE = re.compile(r"\s+")
RUSSIAN_LETTER = re.compile(r"[А-Яа-яЁё]")
ANY_LETTER = re.compile(r"[^\W\d_]", re.UNICODE)
BAD_MARKUP = ("{{", "}}", "[[", "]]", "http://", "https://", "<math", "</", "{|", "|}")
def local_name(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def descendant_text(element, name: str) -> str:
for child in element.iter():
if local_name(child.tag) == name:
return child.text or ""
return ""
def clean_page_text(source: str) -> str:
import mwparserfromhell
source = re.sub(r"<ref\b[^>]*>.*?</ref>|<ref\b[^>]*/>", " ", source, flags=re.IGNORECASE | re.DOTALL)
source = re.sub(r"\{\|.*?\|\}", " ", source, flags=re.DOTALL)
plain = mwparserfromhell.parse(source).strip_code(normalize=True, collapse=True)
return WHITESPACE.sub(" ", plain).strip()
def acceptable(sentence: str) -> bool:
if not 40 <= len(sentence) <= 240 or any(marker in sentence for marker in BAD_MARKUP):
return False
if sentence[-1:] not in ".!?…":
return False
words = sentence.split()
if not 6 <= len(words) <= 45:
return False
letters = ANY_LETTER.findall(sentence)
if not letters:
return False
russian = RUSSIAN_LETTER.findall(sentence)
return len(russian) / len(letters) >= 0.75
def extract(args: argparse.Namespace) -> None:
temporary = args.output.with_suffix(args.output.suffix + ".part")
if args.output.exists() or args.metadata.exists() or temporary.exists():
raise FileExistsError("Refusing to overwrite output, metadata, or partial output")
args.output.parent.mkdir(parents=True, exist_ok=True)
seen: set[bytes] = set()
sample_heap: list[tuple[int, bytes, str]] = []
pages = 0
accepted = 0
candidates = 0
with bz2.open(args.dump, "rb") as source, temporary.open("w", encoding="utf-8", newline="\n") as output:
for _, element in ElementTree.iterparse(source, events=("end",)):
if local_name(element.tag) != "page":
continue
pages += 1
namespace = descendant_text(element, "ns")
title = descendant_text(element, "title")
wiki_text = descendant_text(element, "text")
if namespace == "0" and wiki_text and not title.startswith(("Категория:", "Шаблон:", "Справка:")):
for sentence in SENTENCE_BOUNDARY.split(clean_page_text(wiki_text)):
# Leading dashes carry dialogue/prosody information and must survive extraction.
sentence = sentence.strip(" \t\r\n")
if not acceptable(sentence):
continue
digest = hashlib.blake2b(sentence.encode("utf-8"), digest_size=16).digest()
candidates += 1
if args.sampling == "first":
if digest in seen:
continue
seen.add(digest)
output.write(sentence + "\n")
accepted += 1
if accepted % 10_000 == 0:
print(f"sentences={accepted} pages={pages}", flush=True)
if accepted >= args.max_sentences:
break
else:
score = int.from_bytes(digest, "big")
if digest in seen:
continue
if len(sample_heap) < args.max_sentences:
heapq.heappush(sample_heap, (-score, digest, sentence)); seen.add(digest)
elif score < -sample_heap[0][0]:
_, removed_digest, _ = heapq.heapreplace(sample_heap, (-score, digest, sentence))
seen.remove(removed_digest); seen.add(digest)
element.clear()
if args.sampling == "first" and accepted >= args.max_sentences:
break
if pages % 10_000 == 0:
print(f"pages={pages} candidates={candidates} retained={len(sample_heap)}", flush=True)
if args.sampling == "hash":
selected = sorted((-negative_score, sentence) for negative_score, _, sentence in sample_heap)
for _, sentence in selected:
output.write(sentence + "\n")
accepted = len(selected)
temporary.replace(args.output)
metadata = {
"source_url": args.source_url,
"dump_file": args.dump.name,
"dump_bytes": args.dump.stat().st_size,
"pages_scanned": pages,
"candidate_sentences": candidates,
"sentences": accepted,
"output_bytes": args.output.stat().st_size,
"created_utc": datetime.now(timezone.utc).isoformat(),
"sampling": args.sampling,
"filters": {"min_chars": 40, "max_chars": 240, "min_words": 6, "max_words": 45, "min_russian_letter_ratio": 0.75},
}
args.metadata.write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(metadata, ensure_ascii=False, indent=2))
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--dump", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--metadata", type=Path, required=True)
parser.add_argument("--max-sentences", type=int, default=200_000)
parser.add_argument(
"--sampling", choices=("first", "hash"), default="hash",
help="hash scans the complete dump and keeps a deterministic min-hash sample",
)
parser.add_argument(
"--source-url",
default="https://dumps.wikimedia.org/ruwikisource/latest/ruwikisource-latest-pages-articles-multistream.xml.bz2",
)
return parser.parse_args()
if __name__ == "__main__":
sys.exit(extract(parse_args()) or 0)
@@ -0,0 +1,198 @@
[CmdletBinding()]
param(
[int[]] $Milestones = @(100, 300, 500),
[int] $PollSeconds = 20
)
$ErrorActionPreference = 'Stop'
$root = $PSScriptRoot
$trainingDir = Join-Path $root 'training'
$checkpointRoot = Join-Path $trainingDir 'checkpoints'
$configSource = Join-Path $trainingDir 'aletheia_ru.onnx.json'
$python = Join-Path $root '.venv\Scripts\python.exe'
$outputRoot = Join-Path $trainingDir 'milestones'
$monitorLog = Join-Path $outputRoot 'monitor.log'
New-Item -ItemType Directory -Force -Path $outputRoot | Out-Null
function Write-MonitorLog([string] $Message) {
$line = '{0} {1}' -f (Get-Date).ToUniversalTime().ToString('o'), $Message
Add-Content -LiteralPath $monitorLog -Value $line -Encoding utf8
}
function Invoke-LoggedProcess(
[string] $FilePath,
[string[]] $Arguments,
[string] $StdoutPath,
[string] $StderrPath
) {
$process = Start-Process -FilePath $FilePath `
-ArgumentList $Arguments `
-RedirectStandardOutput $StdoutPath `
-RedirectStandardError $StderrPath `
-WindowStyle Hidden `
-Wait `
-PassThru
if ($process.ExitCode -ne 0) {
throw "Process failed with exit code $($process.ExitCode): $FilePath"
}
}
function Get-LatestCheckpoint {
Get-ChildItem -LiteralPath $checkpointRoot -Filter '*.ckpt' -File -Recurse -ErrorAction SilentlyContinue |
Where-Object { $_.Name -match '^epoch=(\d+)-step=(\d+)\.ckpt$' } |
Sort-Object LastWriteTimeUtc -Descending |
Select-Object -First 1
}
function Show-TrainingStatus {
$trainingLog = Join-Path $trainingDir 'scheduled-training.stdout.log'
$trainingTask = Get-ScheduledTask -TaskName 'AletheiaTTS-Training' -ErrorAction SilentlyContinue
$progressLine = ''
if (Test-Path -LiteralPath $trainingLog -PathType Leaf) {
$progressLine = Get-Content -LiteralPath $trainingLog -Tail 30 |
Where-Object { $_ -match '^Epoch\s+\d+:' } |
Select-Object -Last 1
}
$checkpoint = Get-LatestCheckpoint
$gpu = (& nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits 2>$null) -join [Environment]::NewLine
try {
$Host.UI.RawUI.WindowTitle = 'Aletheia TTS - training monitor'
Clear-Host
} catch {
# The monitor also works when no interactive console is attached.
}
Write-Host 'Aletheia Russian TTS training'
Write-Host ('Updated: {0}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'))
Write-Host ('Training task: {0}' -f $(if ($trainingTask) { $trainingTask.State } else { 'Not found' }))
Write-Host ('Progress: {0}' -f $(if ($progressLine) { $progressLine.Trim() } else { 'Waiting for log data' }))
Write-Host ('GPU: {0}' -f $(if ($gpu) { $gpu.Trim() } else { 'nvidia-smi unavailable' }))
if ($checkpoint) {
Write-Host ('Latest checkpoint: {0} ({1:N0} bytes)' -f $checkpoint.Name, $checkpoint.Length)
} else {
Write-Host 'Latest checkpoint: waiting'
}
Write-Host ''
Write-Host 'Evaluation milestones:'
foreach ($milestone in $Milestones) {
$completePath = Join-Path $outputRoot ('epoch-{0:d4}\complete.json' -f $milestone)
$state = if (Test-Path -LiteralPath $completePath -PathType Leaf) { 'READY' } else { 'waiting' }
Write-Host (' {0,4} epochs: {1}' -f $milestone, $state)
}
Write-Host ''
Write-Host 'This window may be minimized. Closing it stops milestone exports only.'
}
function Save-Milestone([int] $Milestone, [IO.FileInfo] $SourceCheckpoint) {
$milestoneName = 'epoch-{0:d4}' -f $Milestone
$milestoneDir = Join-Path $outputRoot $milestoneName
$completePath = Join-Path $milestoneDir 'complete.json'
if (Test-Path -LiteralPath $completePath -PathType Leaf) {
return
}
New-Item -ItemType Directory -Force -Path $milestoneDir | Out-Null
$checkpointPath = Join-Path $milestoneDir "aletheia_ru_$milestoneName.ckpt"
if (-not (Test-Path -LiteralPath $checkpointPath -PathType Leaf)) {
$firstLength = $SourceCheckpoint.Length
Start-Sleep -Seconds 10
$refreshed = Get-Item -LiteralPath $SourceCheckpoint.FullName
if ($firstLength -ne $refreshed.Length -or $refreshed.Length -le 0) {
throw "Checkpoint is not stable yet: $($SourceCheckpoint.FullName)"
}
$partialPath = "$checkpointPath.partial"
Copy-Item -LiteralPath $refreshed.FullName -Destination $partialPath -Force
Move-Item -LiteralPath $partialPath -Destination $checkpointPath -Force
}
$modelPath = Join-Path $milestoneDir "aletheia_ru_$milestoneName.onnx"
$configPath = "$modelPath.json"
if (-not (Test-Path -LiteralPath $modelPath -PathType Leaf)) {
Invoke-LoggedProcess $python @(
'-m', 'piper.train.export_onnx',
'--checkpoint', $checkpointPath,
'--output-file', $modelPath
) (Join-Path $milestoneDir 'export.stdout.log') (Join-Path $milestoneDir 'export.stderr.log')
}
Copy-Item -LiteralPath $configSource -Destination $configPath -Force
$inputPath = Join-Path $milestoneDir 'test_sentences.txt'
$testText = @(
'0JIg0YLQuNGI0LjQvdC1INCy0LXRh9C10YDQvdC10Lkg0LHQuNCx0LvQuNC+0YLQtdC60Lgg0YjQtdC70LXRgdGC0LXQu9C4INGB0YLRgNCw0L3QuNGG0YssINC4INC60LDQttC00LDRjyDQvdC+0LLQsNGPINCz0LvQsNCy0LAg0L7RgtC60YDRi9Cy0LDQu9CwINGD0LTQuNCy0LjRgtC10LvRjNC90YvQuSDQvNC40YAu',
'0JrQvtCz0LTQsCDRh9C10LvQvtCy0LXQuiDQtNC10LvQsNC10YIg0LLRi9Cx0L7RgCwg0L7QvSDQvdC1INCy0YHQtdCz0LTQsCDQt9Cw0YDQsNC90LXQtSDQt9C90LDQtdGCLCDQuiDQutCw0LrQuNC8INC/0L7RgdC70LXQtNGB0YLQstC40Y/QvCDQv9GA0LjQstC10LTRkdGCINC10LPQviDRgNC10YjQtdC90LjQtS4=',
'0JfQsCDQvtC60L3QvtC8INC80LXQtNC70LXQvdC90L4g0L3QsNGH0LjQvdCw0LvRgdGPINC00L7QttC00YwsINC90L4g0L/Rg9GC0LXRiNC10YHRgtCy0LjQtSDQs9C10YDQvtC10LIg0YLQvtC70YzQutC+INC90LDQsdC40YDQsNC70L4g0YHQuNC70YMu'
) | ForEach-Object { [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($_)) }
$testText = $testText -join [Environment]::NewLine
[IO.File]::WriteAllText($inputPath, $testText, [Text.UTF8Encoding]::new($false))
$samples = @(
@{ Name = 'neutral'; Length = '1.0'; Noise = '0.667'; Width = '0.8'; Silence = '0.0' },
@{ Name = 'book'; Length = '1.5'; Noise = '0.75'; Width = '0.95'; Silence = '0.25' },
@{ Name = 'fast'; Length = '0.82'; Noise = '0.70'; Width = '0.85'; Silence = '0.0' }
)
foreach ($sample in $samples) {
$wavPath = Join-Path $milestoneDir ("sample_{0}.wav" -f $sample.Name)
if (Test-Path -LiteralPath $wavPath -PathType Leaf) {
continue
}
Invoke-LoggedProcess $python @(
'-m', 'piper',
'--model', $modelPath,
'--config', $configPath,
'--input-file', $inputPath,
'--output-file', $wavPath,
'--length-scale', $sample.Length,
'--noise-scale', $sample.Noise,
'--noise-w-scale', $sample.Width,
'--sentence-silence', $sample.Silence
) (Join-Path $milestoneDir "$($sample.Name).stdout.log") (Join-Path $milestoneDir "$($sample.Name).stderr.log")
}
$sourceMatch = [regex]::Match($SourceCheckpoint.Name, '^epoch=(\d+)-step=(\d+)\.ckpt$')
$artifacts = Get-ChildItem -LiteralPath $milestoneDir -File |
Where-Object { $_.Extension -in @('.ckpt', '.onnx', '.json', '.wav') } |
ForEach-Object {
[ordered]@{
name = $_.Name
bytes = $_.Length
sha256 = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
}
}
[ordered]@{
requested_completed_epochs = $Milestone
source_epoch_zero_based = [int]$sourceMatch.Groups[1].Value
source_step = [int]$sourceMatch.Groups[2].Value
created_utc = (Get-Date).ToUniversalTime().ToString('o')
artifacts = @($artifacts)
} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $completePath -Encoding utf8
Write-MonitorLog "Completed milestone $Milestone from $($SourceCheckpoint.Name)"
}
Write-MonitorLog "Monitor started for milestones: $($Milestones -join ', ')"
while ($true) {
Show-TrainingStatus
$pending = @($Milestones | Where-Object {
-not (Test-Path -LiteralPath (Join-Path $outputRoot ('epoch-{0:d4}\complete.json' -f $_)) -PathType Leaf)
})
if ($pending.Count -eq 0) {
Write-MonitorLog 'All milestones completed'
exit 0
}
try {
$checkpoint = Get-LatestCheckpoint
if ($checkpoint -and $checkpoint.Name -match '^epoch=(\d+)-step=(\d+)\.ckpt$') {
$completedEpochs = [int]$Matches[1] + 1
foreach ($milestone in $pending) {
if ($completedEpochs -ge $milestone) {
Save-Milestone $milestone $checkpoint
}
}
}
} catch {
Write-MonitorLog ("Retryable error: " + ($_ | Out-String).Trim())
}
Start-Sleep -Seconds $PollSeconds
}
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Add the original-to-compact token lookup to an already exported student ONNX."""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
import numpy as np
import onnx
from onnx import helper, numpy_helper
def patch(args: argparse.Namespace) -> None:
if args.output.exists():
raise FileExistsError(f"Refusing to overwrite {args.output}")
model = onnx.load(str(args.model))
if not any(value.name == "input_ids" for value in model.graph.input):
raise ValueError("Model has no input_ids graph input")
if any(item.name == "original_to_compact_token_id" for item in model.graph.initializer):
raise ValueError("Model already contains original_to_compact_token_id")
consumers = 0
for node in model.graph.node:
for index, name in enumerate(node.input):
if name == "input_ids":
node.input[index] = "mapped_input_ids"
consumers += 1
if consumers == 0:
raise ValueError("No input_ids consumers found")
token_map = np.load(args.token_map).astype(np.int64, copy=False)
model.graph.initializer.append(numpy_helper.from_array(token_map, "original_to_compact_token_id"))
model.graph.node.insert(
0,
helper.make_node(
"Gather", ["original_to_compact_token_id", "input_ids"], ["mapped_input_ids"],
axis=0, name="MapOriginalTokenIds",
),
)
onnx.checker.check_model(model)
onnx.save(model, str(args.output))
print(f"token_map_entries={token_map.size} consumers={consumers} output_bytes={args.output.stat().st_size}")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--token-map", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
return parser.parse_args()
if __name__ == "__main__":
sys.exit(patch(parse_args()) or 0)
+11
View File
@@ -0,0 +1,11 @@
diff --git a/src/piper/train/export_onnx.py b/src/piper/train/export_onnx.py
index fe9be60..abfeeb0 100644
--- a/src/piper/train/export_onnx.py
+++ b/src/piper/train/export_onnx.py
@@ -103,5 +103,6 @@ def main() -> None:
"input_lengths": {0: "batch_size"},
"output": {0: "batch_size", 2: "time"},
},
+ dynamo=False,
)
_LOGGER.info("Exported model to %s", output_path)
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Prepare a Piper ONNX export for sherpa-onnx mobile inference."""
import argparse
import json
import shutil
from pathlib import Path
import onnx
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--config", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
config = json.loads(args.config.read_text(encoding="utf-8"))
output_dir: Path = args.output_dir
output_dir.mkdir(parents=True, exist_ok=True)
output_model = output_dir / "aletheia_ru.onnx"
output_config = output_dir / "aletheia_ru.onnx.json"
output_tokens = output_dir / "tokens.txt"
shutil.copy2(args.model, output_model)
shutil.copy2(args.config, output_config)
id_map = config["phoneme_id_map"]
# sherpa-onnx's Piper lexicon maps one Unicode code point to one model ID.
# Newer Piper configs may also contain English diphthong aliases such as
# "aɪ". eSpeak emits their component code points, and sherpa rejects the
# multi-code-point aliases, so only the single-code-point table is mobile-safe.
token_rows = sorted(
((ids[0], symbol) for symbol, ids in id_map.items() if len(symbol) == 1),
key=lambda row: row[0],
)
actual_ids = [row[0] for row in token_rows]
if len(actual_ids) != len(set(actual_ids)):
raise RuntimeError("phoneme_id_map contains duplicate token identifiers")
if not actual_ids or min(actual_ids) < 0 or max(actual_ids) >= config["num_symbols"]:
raise RuntimeError("phoneme_id_map contains a token outside the model symbol range")
output_tokens.write_text(
"".join(f"{symbol} {token_id}\n" for token_id, symbol in token_rows),
encoding="utf-8",
newline="\n",
)
model = onnx.load(str(output_model))
metadata = {
"model_type": "vits",
"comment": "piper",
"language": "Russian",
"voice": config["espeak"]["voice"],
"has_espeak": "1",
"n_speakers": str(config["num_speakers"]),
"sample_rate": str(config["audio"]["sample_rate"]),
}
existing = {item.key: item for item in model.metadata_props}
for key, value in metadata.items():
if key in existing:
existing[key].value = value
else:
item = model.metadata_props.add()
item.key = key
item.value = value
onnx.save(model, str(output_model))
print(json.dumps({
"model": str(output_model),
"model_bytes": output_model.stat().st_size,
"tokens": len(token_rows),
"metadata": metadata,
}, ensure_ascii=False))
if __name__ == "__main__":
main()
+15
View File
@@ -0,0 +1,15 @@
param([string]$Root = 'C:\Users\seven\AletheiaBookTTS')
$ErrorActionPreference = 'Stop'
$env:PYTHONUTF8 = '1'
$env:PYTHONIOENCODING = 'utf-8'
$python = Join-Path $Root '.venv\Scripts\python.exe'
$work = Join-Path $Root 'work'
$candidate = Join-Path $work 'student-100k-v1'
& $python (Join-Path $work 'vosk_booktts_experiment.py') compare-feeds `
--model (Join-Path $Root 'teacher\model.onnx') `
--baseline-calibration (Join-Path $candidate 'feeds-baseline-100-v2') `
--candidate-calibration (Join-Path $candidate 'feeds-student-int8-100-v2') `
--output (Join-Path $candidate 'wav-ab-100-v1') `
--resume
exit $LASTEXITCODE
+89
View File
@@ -0,0 +1,89 @@
param(
[string]$Root = 'C:\Users\seven\AletheiaBookTTS',
[int]$MaxSentences = 100000,
[int]$TeacherBatchSize = 16,
[ValidateSet('first', 'hash')]
[string]$Sampling = 'hash'
)
$ErrorActionPreference = 'Stop'
$ProgressPreference = 'SilentlyContinue'
$env:PYTHONUTF8 = '1'
$env:PYTHONIOENCODING = 'utf-8'
$displayName = 'AletheiaBookTTS-RuWikisource'
$corpusDir = Join-Path $Root 'corpus'
$workDir = Join-Path $Root 'work'
$teacherDir = Join-Path $Root 'teacher'
$python = Join-Path $Root '.venv\Scripts\python.exe'
$dump = Join-Path $corpusDir 'ruwikisource-latest-pages-articles-multistream.xml.bz2'
$sentences = Join-Path $corpusDir "ruwikisource-booktts-$MaxSentences-$Sampling.txt"
$metadata = Join-Path $corpusDir "ruwikisource-booktts-$MaxSentences-$Sampling.json"
$dataset = Join-Path $workDir "student-dataset-$MaxSentences-$Sampling"
$datasetPartial = "$dataset.partial"
$expectedBytes = 2104816879L
$checksumUrl = 'https://dumps.wikimedia.org/ruwikisource/latest/ruwikisource-latest-sha1sums.txt'
Import-Module BitsTransfer
while (-not (Test-Path -LiteralPath $dump)) {
$job = Get-BitsTransfer -ErrorAction SilentlyContinue |
Where-Object DisplayName -eq $displayName |
Select-Object -First 1
if (-not $job) {
throw "BITS job $displayName is absent and dump is not downloaded"
}
if ($job.JobState -eq 'Transferred') {
Complete-BitsTransfer -BitsJob $job
break
}
if ($job.JobState -eq 'Error') {
throw "BITS download failed: $($job.ErrorDescription)"
}
$percent = if ($job.BytesTotal -gt 0 -and $job.BytesTotal -lt [uint64]::MaxValue) {
[math]::Round(100 * $job.BytesTransferred / $job.BytesTotal, 2)
} else { 0 }
Write-Output "download state=$($job.JobState) bytes=$($job.BytesTransferred)/$($job.BytesTotal) percent=$percent"
Start-Sleep -Seconds 20
}
$dumpItem = Get-Item -LiteralPath $dump
if ($dumpItem.Length -ne $expectedBytes) {
throw "Unexpected dump size: $($dumpItem.Length), expected $expectedBytes"
}
$checksumText = (Invoke-WebRequest -UseBasicParsing -Uri $checksumUrl).Content
$checksumLine = ($checksumText -split "`n" | Where-Object { $_ -match 'pages-articles-multistream\.xml\.bz2\s*$' } | Select-Object -First 1).Trim()
if (-not $checksumLine) {
throw 'Cannot find multistream dump in official SHA-1 list'
}
$expectedSha1 = ($checksumLine -split '\s+')[0].ToUpperInvariant()
$actualSha1 = (Get-FileHash -LiteralPath $dump -Algorithm SHA1).Hash
if ($actualSha1 -ne $expectedSha1) {
throw "SHA-1 mismatch: actual=$actualSha1 expected=$expectedSha1"
}
Write-Output "dump verified bytes=$($dumpItem.Length) sha1=$actualSha1"
if (-not (Test-Path -LiteralPath $sentences)) {
& $python (Join-Path $workDir 'extract_wikisource_corpus.py') `
--dump $dump `
--output $sentences `
--metadata $metadata `
--max-sentences $MaxSentences `
--sampling $Sampling
if ($LASTEXITCODE -ne 0) { throw "Corpus extraction failed with exit code $LASTEXITCODE" }
}
if (-not (Test-Path -LiteralPath $dataset)) {
if (Test-Path -LiteralPath $datasetPartial) {
throw "Partial dataset already exists: $datasetPartial"
}
& $python (Join-Path $workDir 'booktts_student.py') dataset `
--assets $teacherDir `
--corpus $sentences `
--output $datasetPartial `
--teacher-batch-size $TeacherBatchSize `
--teacher-dtype float16
if ($LASTEXITCODE -ne 0) { throw "Teacher dataset failed with exit code $LASTEXITCODE" }
Move-Item -LiteralPath $datasetPartial -Destination $dataset
}
Write-Output "pipeline complete corpus=$sentences dataset=$dataset"
@@ -0,0 +1,16 @@
param([string]$Root = 'C:\Users\seven\AletheiaBookTTS')
$ErrorActionPreference = 'Stop'
$env:PYTHONUTF8 = '1'
$env:PYTHONIOENCODING = 'utf-8'
$python = Join-Path $Root '.venv\Scripts\python.exe'
$work = Join-Path $Root 'work'
$output = Join-Path $work 'student-dataset-prosody-v1'
& $python (Join-Path $work 'booktts_student.py') dataset `
--assets (Join-Path $Root 'teacher') `
--corpus (Join-Path $Root 'corpus\booktts-prosody-balanced-v1.txt') `
--output $output `
--teacher-batch-size 16 `
--teacher-dtype float16 `
--reuse-token-map (Join-Path $work 'student-dataset-100000')
exit $LASTEXITCODE
@@ -0,0 +1,17 @@
param([string]$Root = 'C:\Users\seven\AletheiaBookTTS')
$ErrorActionPreference = 'Stop'
$env:PYTHONUTF8 = '1'
$env:PYTHONIOENCODING = 'utf-8'
$python = Join-Path $Root '.venv\Scripts\python.exe'
$work = Join-Path $Root 'work'
& $python (Join-Path $work 'booktts_student.py') train `
--dataset (Join-Path $work 'student-dataset-prosody-v1') `
--initial-model (Join-Path $work 'student-100k-v1\student.pt') `
--output (Join-Path $work 'student-100k-prosody-v1') `
--epochs 8 `
--batch-size 32 `
--learning-rate 0.00005 `
--validation-fraction 0.1 `
--early-stopping-patience 3
exit $LASTEXITCODE
+27
View File
@@ -0,0 +1,27 @@
param(
[string]$Root = 'C:\Users\seven\AletheiaBookTTS',
[string]$DatasetName = 'student-dataset-100000',
[string]$OutputName = 'student-100k-v1',
[int]$Epochs = 20,
[int]$BatchSize = 32,
[double]$ValidationFraction = 0.05,
[int]$EarlyStoppingPatience = 4,
[switch]$Resume
)
$ErrorActionPreference = 'Stop'
$env:PYTHONUTF8 = '1'
$env:PYTHONIOENCODING = 'utf-8'
$python = Join-Path $Root '.venv\Scripts\python.exe'
$trainer = Join-Path $Root 'work\booktts_student.py'
$dataset = Join-Path $Root "work\$DatasetName"
$output = Join-Path $Root "work\$OutputName"
$arguments = @(
$trainer, 'train', '--dataset', $dataset, '--output', $output,
'--epochs', $Epochs, '--batch-size', $BatchSize,
'--validation-fraction', $ValidationFraction,
'--early-stopping-patience', $EarlyStoppingPatience
)
if ($Resume) { $arguments += '--resume' }
& $python @arguments
exit $LASTEXITCODE
+10
View File
@@ -15,6 +15,13 @@ if (Test-Path -LiteralPath $exitPath) {
$exitCode = 1
try {
$latestCheckpoint = Get-ChildItem -LiteralPath (Join-Path $runDir 'checkpoints') `
-Filter '*.ckpt' `
-File `
-Recurse `
-ErrorAction SilentlyContinue |
Sort-Object LastWriteTimeUtc -Descending |
Select-Object -First 1
$arguments = @(
'-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
'-File', (Join-Path $root 'train_piper_windows.ps1'),
@@ -24,6 +31,9 @@ try {
'-PythonExe', (Join-Path $root '.venv\Scripts\python.exe'),
'-BatchSize', '16', '-NumWorkers', '0', '-MaxEpochs', '2000'
)
if ($latestCheckpoint) {
$arguments += @('-CheckpointPath', $latestCheckpoint.FullName)
}
$process = Start-Process -FilePath 'powershell.exe' `
-ArgumentList $arguments `
-RedirectStandardOutput $stdoutPath `
+8
View File
@@ -7,6 +7,7 @@ param(
[Parameter(Mandatory = $true)]
[string] $PythonExe,
[string] $CacheDir,
[string] $CheckpointPath,
[int] $BatchSize = 4,
[int] $NumWorkers = 2,
[int] $MaxEpochs = 2000,
@@ -61,6 +62,13 @@ $fitArgs = @(
if ($SmokeTest) {
$fitArgs += @('--trainer.fast_dev_run', 'true', '--trainer.num_sanity_val_steps', '0')
}
if ($CheckpointPath) {
$checkpointToResume = [IO.Path]::GetFullPath($CheckpointPath)
if (-not (Test-Path -LiteralPath $checkpointToResume -PathType Leaf)) {
throw "Resume checkpoint not found: $checkpointToResume"
}
$fitArgs += @('--ckpt_path', $checkpointToResume)
}
& $python @fitArgs
if ($LASTEXITCODE -ne 0) {
+568
View File
@@ -0,0 +1,568 @@
#!/usr/bin/env python3
"""Prepare real Vosk feeds, quantize acoustic ONNX, and render A/B WAVs.
The frontend mirrors VoskSpeechEngine.kt: number normalization, WordPiece
positions, dictionary lookup and all five phone feature channels.
"""
from __future__ import annotations
import argparse
import bisect
import json
import math
import re
import sys
import time
import wave
from collections import OrderedDict
from dataclasses import dataclass
from pathlib import Path
from typing import Iterator
import numpy as np
SAMPLE_RATE = 22_050
SPEAKER_ID = 4
BERT_DIMENSIONS = 768
MAX_WORD_PIECE_CHARS = 100
CACHE_SIZE = 4_096
MULTISTREAM_PUNCTUATION = set("!(),-.:;?")
BERT_EXCLUDED_PUNCTUATION = {"-", ",", ".", "?", "!", ";", ":", '"'}
PHONEMES = [
"_", "^", "$", " ", "!", "'", "(", ")", ",", "-", ".", "...", ":", ";", "?",
"a0", "a1", "b", "bj", "c", "ch", "d", "dj", "e0", "e1", "f", "fj", "g", "gj",
"h", "hj", "i0", "i1", "j", "k", "kj", "l", "lj", "m", "mj", "n", "nj", "o0",
"o1", "p", "pj", "r", "rj", "s", "sch", "sh", "sj", "t", "tj", "u0", "u1", "v",
"vj", "y0", "y1", "z", "zh", "zj",
]
PHONEME_IDS = {phone: index for index, phone in enumerate(PHONEMES)}
DIGITS = ["ноль", "один", "два", "три", "четыре", "пять", "шесть", "семь", "восемь", "девять"]
TEENS = [
"десять", "одиннадцать", "двенадцать", "тринадцать", "четырнадцать",
"пятнадцать", "шестнадцать", "семнадцать", "восемнадцать", "девятнадцать",
]
TENS = ["", "десять", "двадцать", "тридцать", "сорок", "пятьдесят", "шестьдесят", "семьдесят", "восемьдесят", "девяносто"]
HUNDREDS = ["", "сто", "двести", "триста", "четыреста", "пятьсот", "шестьсот", "семьсот", "восемьсот", "девятьсот"]
SCALES = [
None,
("тысяча", "тысячи", "тысяч"),
("миллион", "миллиона", "миллионов"),
("миллиард", "миллиарда", "миллиардов"),
("триллион", "триллиона", "триллионов"),
("квадриллион", "квадриллиона", "квадриллионов"),
]
NUMBER_PATTERN = re.compile(r"(?<![^\W_])([+-]?)(\d{1,18})(?:[,.](\d+))?(?![^\W_])", re.UNICODE)
YEAR_IN_PREPOSITIONAL_PATTERN = re.compile(r"(?<![^\W_])(\d{4})\s+(году|г\.)", re.IGNORECASE | re.UNICODE)
SOFT_LETTERS = set("яёюиье")
SYLLABLE_STARTS = set("#ъьаёуюэеиы-")
SOFT_HARD_CONSONANTS = {
"б": "b", "в": "v", "г": "g", "д": "d", "з": "z", "к": "k", "л": "l",
"м": "m", "н": "n", "п": "p", "р": "r", "с": "s", "т": "t", "ф": "f", "х": "h",
}
OTHER_CONSONANTS = {"ж": "zh", "ц": "c", "ч": "ch", "ш": "sh", "щ": "sch", "й": "j"}
VOWELS = {"а": "a", "я": "a", "у": "u", "ю": "u", "о": "o", "ё": "o", "э": "e", "е": "e", "и": "i", "ы": "y"}
def scale_form(value: int, forms: tuple[str, str, str]) -> str:
if 11 <= value % 100 <= 14:
return forms[2]
return forms[0] if value % 10 == 1 else forms[1] if value % 10 in (2, 3, 4) else forms[2]
def under_thousand_to_words(value: int, feminine: bool) -> str:
words: list[str] = []
if value // 100:
words.append(HUNDREDS[value // 100])
tail = value % 100
if 10 <= tail <= 19:
words.append(TEENS[tail - 10])
else:
if tail // 10:
words.append(TENS[tail // 10])
units = tail % 10
if units:
words.append("одна" if feminine and units == 1 else "две" if feminine and units == 2 else DIGITS[units])
return " ".join(words)
def integer_to_words(value: int) -> str:
if value == 0:
return DIGITS[0]
groups: list[int] = []
while value:
groups.append(value % 1_000)
value //= 1_000
result: list[str] = []
for group_index in range(len(groups) - 1, -1, -1):
group = groups[group_index]
if not group:
continue
words = under_thousand_to_words(group, group_index == 1)
forms = SCALES[group_index] if group_index < len(SCALES) else None
result.append(words if forms is None else f"{words} {scale_form(group, forms)}")
return " ".join(result)
def normalize_numbers(text: str) -> str:
prepared = re.sub(r"(?<=\d)[\u00a0\u202f](?=\d{3}(?:\D|$))", "", text)
prepared = re.sub(r"\s*(?=\d)", "номер ", prepared)
def replace(match: re.Match[str]) -> str:
sign = "минус " if match.group(1) == "-" else "плюс " if match.group(1) == "+" else ""
integer_digits = match.group(2)
if len(integer_digits) > 1 and integer_digits.startswith("0"):
integer = " ".join(DIGITS[int(char)] for char in integer_digits)
else:
integer = integer_to_words(int(integer_digits))
fraction = match.group(3)
return f"{sign}{integer}" if not fraction else f"{sign}{integer} запятая {' '.join(DIGITS[int(char)] for char in fraction)}"
prepared = YEAR_IN_PREPOSITIONAL_PATTERN.sub(
lambda match: f"{year_in_prepositional_words(int(match.group(1)))} {match.group(2)}",
prepared,
)
return NUMBER_PATTERN.sub(replace, prepared)
ORDINAL_UNITS_PREPOSITIONAL = ("", "первом", "втором", "третьем", "четвёртом", "пятом", "шестом", "седьмом", "восьмом", "девятом")
ORDINAL_TEENS_PREPOSITIONAL = ("десятом", "одиннадцатом", "двенадцатом", "тринадцатом", "четырнадцатом", "пятнадцатом", "шестнадцатом", "семнадцатом", "восемнадцатом", "девятнадцатом")
ORDINAL_TENS_PREPOSITIONAL = ("", "десятом", "двадцатом", "тридцатом", "сороковом", "пятидесятом", "шестидесятом", "семидесятом", "восьмидесятом", "девяностом")
ORDINAL_HUNDREDS_PREPOSITIONAL = ("", "сотом", "двухсотом", "трёхсотом", "четырёхсотом", "пятисотом", "шестисотом", "семисотом", "восьмисотом", "девятисотом")
EXACT_THOUSANDTH_PREPOSITIONAL = ("", "тысячном", "двухтысячном", "трёхтысячном", "четырёхтысячном", "пятитысячном", "шеститысячном", "семитысячном", "восьмитысячном", "девятитысячном")
def ordinal_under_thousand_prepositional(value: int) -> str:
last_two = value % 100
units = value % 10
if units and not 11 <= last_two <= 19:
return f"{integer_to_words(value - units) if value > units else ''} {ORDINAL_UNITS_PREPOSITIONAL[units]}".strip()
if 11 <= last_two <= 19:
return f"{integer_to_words(value - last_two) if value > last_two else ''} {ORDINAL_TEENS_PREPOSITIONAL[last_two - 10]}".strip()
if last_two >= 20:
return f"{integer_to_words(value - last_two) if value > last_two else ''} {ORDINAL_TENS_PREPOSITIONAL[last_two // 10]}".strip()
return ORDINAL_HUNDREDS_PREPOSITIONAL[value // 100]
def year_in_prepositional_words(value: int) -> str:
if not 1000 <= value <= 9999:
raise ValueError(value)
if value % 1000 == 0:
return EXACT_THOUSANDTH_PREPOSITIONAL[value // 1000]
thousands = value // 1000
thousands_words = under_thousand_to_words(thousands, True)
thousands_prefix = "" if thousands_words == "одна" else thousands_words + " "
return f"{thousands_prefix}{scale_form(thousands, SCALES[1])} {ordinal_under_thousand_prepositional(value % 1000)}"
def normalize_text(text: str) -> str:
stressed = re.sub(r"([А-Яа-яЁё])\u0301", lambda match: "+" + match.group(1), text)
translated = normalize_numbers(stressed).lower().replace("", "-").replace("", "-").replace("", "...")
translated = translated.translate(str.maketrans({"«": '"', "»": '"', "": '"', "": '"'}))
return re.sub(r"\s+", " ", translated).strip()
class WordPieceTokenizer:
def __init__(self, path: Path):
self.tokens = path.read_text(encoding="utf-8").splitlines()
self.vocabulary = {token: index for index, token in enumerate(self.tokens)}
for special in ("[CLS]", "[SEP]", "[UNK]"):
if special not in self.vocabulary:
raise ValueError(f"Missing {special} in {path}")
def encode(self, text: str) -> tuple[np.ndarray, list[int]]:
ids = [self.vocabulary["[CLS]"]]
positions = [0]
for token in self._basic_tokens(text):
punctuation = token in BERT_EXCLUDED_PUNCTUATION
pieces = [self.vocabulary.get(token, self.vocabulary["[UNK]"])] if punctuation else self._word_pieces(token)
if not punctuation:
positions.append(len(ids))
ids.extend(pieces)
ids.append(self.vocabulary["[SEP]"])
positions.append(len(ids) - 1)
return np.asarray(ids, dtype=np.int64), positions
@staticmethod
def _basic_tokens(text: str) -> list[str]:
result: list[str] = []
word: list[str] = []
for char in text:
if char.isspace():
if word:
result.append("".join(word)); word.clear()
elif char.isalnum():
word.append(char.lower())
elif char == "+":
continue
else:
if word:
result.append("".join(word)); word.clear()
result.append(char)
if word:
result.append("".join(word))
return result
def _word_pieces(self, token: str) -> list[int]:
if len(token) > MAX_WORD_PIECE_CHARS:
return [self.vocabulary["[UNK]"]]
result: list[int] = []
start = 0
while start < len(token):
found: int | None = None
found_end = start
for end in range(len(token), start, -1):
piece = token[start:end] if start == 0 else f"##{token[start:end]}"
if piece in self.vocabulary:
found = self.vocabulary[piece]; found_end = end; break
if found is None:
return [self.vocabulary["[UNK]"]]
result.append(found); start = found_end
return result
class PronunciationDictionary:
def __init__(self, dictionary_path: Path, index_path: Path):
self.source = dictionary_path.open("rb")
entries = []
for line in index_path.read_text(encoding="utf-8").splitlines():
word, offset = line.rsplit("\t", 1)
entries.append((word, int(offset)))
self.words = [entry[0] for entry in entries]
self.offsets = [entry[1] for entry in entries]
self.length = dictionary_path.stat().st_size
self.cache: OrderedDict[str, str | None] = OrderedDict()
def close(self) -> None:
self.source.close()
def find(self, word: str) -> str | None:
if word in self.cache:
value = self.cache.pop(word); self.cache[word] = value; return value
index = max(0, bisect.bisect_right(self.words, word) - 1)
end = self.offsets[index + 1] if index + 1 < len(self.offsets) else self.length
self.source.seek(self.offsets[index])
value: str | None = None
while self.source.tell() < end:
raw = self.source.readline()
if not raw:
break
entry_word, _, pronunciation = raw.decode("utf-8").rstrip("\r\n").partition("\t")
if entry_word < word:
continue
if entry_word == word:
value = pronunciation.strip() or None
break
self.cache[word] = value
if len(self.cache) > CACHE_SIZE:
self.cache.popitem(last=False)
return value
def russian_g2p(word: str) -> str:
marked: list[tuple[str, int]] = []
stress = 0
for char in f"#{word}#":
if char == "+":
stress = 1
else:
marked.append((char, stress)); stress = 0
phones: list[str] = []
for index, (char, accent) in enumerate(marked):
previous = marked[index - 1][0] if index else "#"
following = marked[index + 1][0] if index + 1 < len(marked) else None
if char in SOFT_HARD_CONSONANTS:
phones.append(SOFT_HARD_CONSONANTS[char] + ("j" if following in SOFT_LETTERS else ""))
elif char in OTHER_CONSONANTS:
phones.append(OTHER_CONSONANTS[char])
elif char in VOWELS:
if previous in SYLLABLE_STARTS and char in set("яюеё"):
phones.append("j")
phones.append(f"{VOWELS[char]}{accent}")
return " ".join(phones)
@dataclass
class RawPhone:
phone: str
punctuation: list[str]
in_quote: int
bert_word_index: int
def build_multistream_phones(text: str, dictionary: PronunciationDictionary) -> tuple[np.ndarray, list[int]]:
raw = [RawPhone("^", [], 0, 0)]
word: list[str] = []
pending: list[str] = []
in_quote = 0
bert_word_index = 1
def flush_word() -> None:
nonlocal bert_word_index
if not word:
return
value = "".join(word)
pronunciation = dictionary.find(value) or russian_g2p(value)
raw.extend(RawPhone(phone, [], in_quote, bert_word_index) for phone in pronunciation.split() if phone)
word.clear(); bert_word_index += 1
def append_space() -> None:
raw.append(RawPhone(" ", list(pending), in_quote, bert_word_index)); pending.clear()
index = 0
while index < len(text):
if text.startswith("...", index):
flush_word(); pending.append("..."); index += 3; continue
char = text[index]
if char in {'"', "«", "»", "", ""}:
flush_word(); in_quote = 1 - in_quote
elif char.isspace():
flush_word(); append_space()
elif char == "-" and index > 0 and index + 1 < len(text) and text[index - 1].isalnum() and text[index + 1].isalnum():
flush_word()
elif char in MULTISTREAM_PUNCTUATION:
flush_word(); pending.append(char)
elif char.isalnum() or char == "+":
word.append(char)
index += 1
flush_word(); append_space(); raw.append(RawPhone("$", [], 0, bert_word_index))
last_punctuation = " "
last_sentence_punctuation = " "
reversed_features: list[list[int]] = []
reversed_positions: list[int] = []
for phone in reversed(raw):
for candidate in ("...", ".", "!", "?", "-"):
if candidate in phone.punctuation:
last_sentence_punctuation = candidate; break
if phone.punctuation:
last_punctuation = phone.punctuation[0]
current = phone.punctuation[0] if phone.punctuation else "_"
values = [PHONEME_IDS[phone.phone], PHONEME_IDS[current], phone.in_quote, PHONEME_IDS[last_punctuation], PHONEME_IDS[last_sentence_punctuation]]
reversed_features.append(values); reversed_positions.append(phone.bert_word_index)
features = np.asarray(list(reversed(reversed_features)), dtype=np.int64).T[np.newaxis, :, :]
return features, list(reversed(reversed_positions))
class Frontend:
def __init__(self, assets: Path, bert_model: Path | None = None):
import onnxruntime as ort
self.tokenizer = WordPieceTokenizer(assets / "vocab.txt")
self.dictionary = PronunciationDictionary(assets / "dict.tsv", assets / "dictionary.index")
self.bert = ort.InferenceSession(str(bert_model or assets / "bert.int8.onnx"), providers=["CPUExecutionProvider"])
def close(self) -> None:
self.dictionary.close()
def prepare(self, text: str, articulation: float = 1.0) -> dict[str, np.ndarray]:
normalized = normalize_text(text)
ids, embedding_positions = self.tokenizer.encode(normalized)
shape = (1, ids.size)
token_embeddings = self.bert.run(None, {
"input_ids": ids.reshape(shape),
"attention_mask": np.ones(shape, dtype=np.int64),
"token_type_ids": np.zeros(shape, dtype=np.int64),
})[0]
selected = token_embeddings[np.asarray(embedding_positions, dtype=np.int64)]
features, phone_positions = build_multistream_phones(normalized, self.dictionary)
phone_embeddings = selected[np.clip(np.asarray(phone_positions), 0, len(selected) - 1)]
bert = phone_embeddings.T[np.newaxis, :, :].astype(np.float32, copy=False)
time_steps = features.shape[2]
if bert.shape != (1, BERT_DIMENSIONS, time_steps):
raise ValueError(f"Unexpected BERT shape {bert.shape} for {text!r}")
return {
"input": features,
"input_lengths": np.asarray([time_steps], dtype=np.int64),
"scales": np.asarray([0.8, articulation, 0.8], dtype=np.float32),
"sid": np.asarray([SPEAKER_ID], dtype=np.int64),
"bert": bert,
}
def load_corpus(path: Path) -> list[str]:
lines = [line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines()]
result = [line for line in lines if line and not line.startswith("#")]
if not result:
raise ValueError(f"No phrases in {path}")
return result
def command_prepare(args: argparse.Namespace) -> None:
args.output.mkdir(parents=True, exist_ok=False)
frontend = Frontend(args.assets, args.bert_model)
manifest = []
try:
for index, text in enumerate(load_corpus(args.corpus)):
feeds = frontend.prepare(text, args.articulation)
name = f"sample-{index:04d}.npz"
np.savez_compressed(args.output / name, **feeds)
manifest.append({"file": name, "text": text, "time_steps": int(feeds["input_lengths"][0])})
print(f"prepared {index + 1}: phones={manifest[-1]['time_steps']} text={text}")
finally:
frontend.close()
(args.output / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(f"samples={len(manifest)} output={args.output}")
class NpzCalibrationReader:
def __init__(self, directory: Path):
self.files = sorted(directory.glob("sample-*.npz"))
self.iterator: Iterator[Path] | None = None
if not self.files:
raise ValueError(f"No sample-*.npz in {directory}")
def get_next(self) -> dict[str, np.ndarray] | None:
if self.iterator is None:
self.iterator = iter(self.files)
try:
path = next(self.iterator)
except StopIteration:
return None
with np.load(path) as data:
return {name: data[name] for name in data.files}
def rewind(self) -> None:
self.iterator = iter(self.files)
def command_quantize(args: argparse.Namespace) -> None:
from onnxruntime.quantization import QuantFormat, QuantType, quantize_static
if args.output.exists():
raise FileExistsError(f"Refusing to overwrite {args.output}")
started = time.time()
quantize_static(
str(args.model), str(args.output), NpzCalibrationReader(args.calibration),
quant_format=QuantFormat.QDQ, op_types_to_quantize=["Conv"], per_channel=True,
activation_type=QuantType.QInt8, weight_type=QuantType.QInt8,
)
source_size = args.model.stat().st_size
target_size = args.output.stat().st_size
print(json.dumps({
"source_bytes": source_size, "target_bytes": target_size,
"saving_bytes": source_size - target_size,
"saving_percent": round((source_size - target_size) * 100 / source_size, 4),
"seconds": round(time.time() - started, 3),
}, indent=2))
def write_wav(path: Path, samples: np.ndarray) -> None:
pcm = (np.clip(samples.reshape(-1), -1.0, 1.0) * 32767.0).astype("<i2").tobytes()
with wave.open(str(path), "wb") as output:
output.setnchannels(1); output.setsampwidth(2); output.setframerate(SAMPLE_RATE); output.writeframes(pcm)
def command_compare(args: argparse.Namespace) -> None:
import onnxruntime as ort
args.output.mkdir(parents=True, exist_ok=False)
options = ort.SessionOptions(); options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL
baseline = ort.InferenceSession(str(args.baseline), sess_options=options, providers=["CPUExecutionProvider"])
candidate = ort.InferenceSession(str(args.candidate), sess_options=options, providers=["CPUExecutionProvider"])
manifest = json.loads((args.calibration / "manifest.json").read_text(encoding="utf-8"))
report = []
for item in manifest[: args.limit]:
with np.load(args.calibration / item["file"]) as data:
feeds = {name: data[name] for name in data.files}
feeds["scales"] = np.asarray([0.0, float(feeds["scales"][1]), 0.0], dtype=np.float32)
base_wav, base_length = baseline.run(None, feeds)
test_wav, test_length = candidate.run(None, feeds)
common = min(base_wav.size, test_wav.size)
difference = base_wav.reshape(-1)[:common] - test_wav.reshape(-1)[:common]
entry = {
"text": item["text"], "baseline_samples": int(base_wav.size), "candidate_samples": int(test_wav.size),
"baseline_length": np.asarray(base_length).tolist(), "candidate_length": np.asarray(test_length).tolist(),
"mae": float(np.mean(np.abs(difference))), "rmse": float(math.sqrt(np.mean(difference * difference))),
}
report.append(entry)
number = len(report)
write_wav(args.output / f"{number:02d}-baseline.wav", base_wav)
write_wav(args.output / f"{number:02d}-candidate.wav", test_wav)
print(f"compared {number}: mae={entry['mae']:.6f} rmse={entry['rmse']:.6f}")
(args.output / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def command_compare_feeds(args: argparse.Namespace) -> None:
import onnxruntime as ort
args.output.mkdir(parents=True, exist_ok=args.resume)
options = ort.SessionOptions(); options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL
acoustic = ort.InferenceSession(str(args.model), sess_options=options, providers=["CPUExecutionProvider"])
baseline_manifest = json.loads((args.baseline_calibration / "manifest.json").read_text(encoding="utf-8"))
candidate_manifest = json.loads((args.candidate_calibration / "manifest.json").read_text(encoding="utf-8"))
if [item["text"] for item in baseline_manifest] != [item["text"] for item in candidate_manifest]:
raise ValueError("Baseline and candidate manifests contain different texts")
report = []
for baseline_item, candidate_item in zip(baseline_manifest[: args.limit or None], candidate_manifest[: args.limit or None]):
with np.load(args.baseline_calibration / baseline_item["file"]) as data:
baseline_feeds = {name: data[name] for name in data.files}
with np.load(args.candidate_calibration / candidate_item["file"]) as data:
candidate_feeds = {name: data[name] for name in data.files}
baseline_feeds["scales"] = np.asarray([0.0, float(baseline_feeds["scales"][1]), 0.0], dtype=np.float32)
candidate_feeds["scales"] = np.asarray([0.0, float(candidate_feeds["scales"][1]), 0.0], dtype=np.float32)
baseline_wav, baseline_length = acoustic.run(None, baseline_feeds)
candidate_wav, candidate_length = acoustic.run(None, candidate_feeds)
common = min(baseline_wav.size, candidate_wav.size)
difference = baseline_wav.reshape(-1)[:common] - candidate_wav.reshape(-1)[:common]
entry = {
"text": baseline_item["text"],
"baseline_samples": int(baseline_wav.size), "candidate_samples": int(candidate_wav.size),
"baseline_length": np.asarray(baseline_length).tolist(),
"candidate_length": np.asarray(candidate_length).tolist(),
"mae": float(np.mean(np.abs(difference))),
"rmse": float(math.sqrt(np.mean(difference * difference))),
}
report.append(entry)
number = len(report)
baseline_path = args.output / f"{number:03d}-baseline.wav"
candidate_path = args.output / f"{number:03d}-student-int8.wav"
if not baseline_path.exists():
write_wav(baseline_path, baseline_wav)
if not candidate_path.exists():
write_wav(candidate_path, candidate_wav)
temporary_report = args.output / "report.json.tmp"
temporary_report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary_report.replace(args.output / "report.json")
print(f"rendered {number}: mae={entry['mae']:.6f} rmse={entry['rmse']:.6f}", flush=True)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
commands = parser.add_subparsers(dest="command", required=True)
prepare = commands.add_parser("prepare")
prepare.add_argument("--assets", type=Path, required=True)
prepare.add_argument("--corpus", type=Path, required=True)
prepare.add_argument("--output", type=Path, required=True)
prepare.add_argument("--articulation", type=float, default=1.0)
prepare.add_argument("--bert-model", type=Path)
prepare.set_defaults(handler=command_prepare)
quantize = commands.add_parser("quantize")
quantize.add_argument("--model", type=Path, required=True)
quantize.add_argument("--calibration", type=Path, required=True)
quantize.add_argument("--output", type=Path, required=True)
quantize.set_defaults(handler=command_quantize)
compare = commands.add_parser("compare")
compare.add_argument("--baseline", type=Path, required=True)
compare.add_argument("--candidate", type=Path, required=True)
compare.add_argument("--calibration", type=Path, required=True)
compare.add_argument("--output", type=Path, required=True)
compare.add_argument("--limit", type=int, default=3)
compare.set_defaults(handler=command_compare)
compare_feeds = commands.add_parser("compare-feeds")
compare_feeds.add_argument("--model", type=Path, required=True)
compare_feeds.add_argument("--baseline-calibration", type=Path, required=True)
compare_feeds.add_argument("--candidate-calibration", type=Path, required=True)
compare_feeds.add_argument("--output", type=Path, required=True)
compare_feeds.add_argument("--limit", type=int, default=0)
compare_feeds.add_argument("--resume", action="store_true")
compare_feeds.set_defaults(handler=command_compare_feeds)
return parser.parse_args()
def main() -> int:
args = parse_args()
args.handler(args)
return 0
if __name__ == "__main__":
sys.exit(main())