From 1f78420555ca13d40c6de7fef9d84149893467a5 Mon Sep 17 00:00:00 2001 From: sdercolin Date: Sun, 11 Feb 2024 20:56:49 +0800 Subject: [PATCH] Implement phonemes conversion --- src/jsMain/kotlin/io/Svp.kt | 3 +- src/jsMain/kotlin/io/Ustx.kt | 20 +- src/jsMain/kotlin/model/Feature.kt | 5 + src/jsMain/kotlin/model/Format.kt | 14 +- .../kotlin/process/lyrics/LyricsMapping.kt | 2 +- .../process/phonemes/PhonemesMapping.kt | 9 +- src/jsMain/kotlin/ui/ConfigurationEditor.kt | 67 ++++++ .../ui/configuration/PhonemesConversion.kt | 202 ++++++++++++++++++ src/jsMain/kotlin/ui/strings/Strings.kt | 48 +++++ 9 files changed, 357 insertions(+), 13 deletions(-) create mode 100644 src/jsMain/kotlin/ui/configuration/PhonemesConversion.kt diff --git a/src/jsMain/kotlin/io/Svp.kt b/src/jsMain/kotlin/io/Svp.kt index cf7337c..5cb999f 100644 --- a/src/jsMain/kotlin/io/Svp.kt +++ b/src/jsMain/kotlin/io/Svp.kt @@ -108,6 +108,7 @@ object Svp { tickOn = tickOn, tickOff = tickOn + note.duration / TICK_RATE, lyric = note.lyrics.takeUnless { it.isNullOrBlank() } ?: DEFAULT_LYRIC, + phoneme = note.phonemes, ) } @@ -250,7 +251,7 @@ object Svp { onset = it.tickOn * TICK_RATE, duration = it.length * TICK_RATE, lyrics = it.lyric, - phonemes = "", + phonemes = it.phoneme ?: "", pitch = it.key, attributes = Attributes(), ) diff --git a/src/jsMain/kotlin/io/Ustx.kt b/src/jsMain/kotlin/io/Ustx.kt index 9055bdb..40dba1a 100644 --- a/src/jsMain/kotlin/io/Ustx.kt +++ b/src/jsMain/kotlin/io/Ustx.kt @@ -94,10 +94,20 @@ object Ustx { val track = trackMap[trackId] ?: continue val tickPrefix = voicePart.position val notes = voicePart.notes.map { + val rawLyrics = it.lyric + // some phonemizers use `lyrics [phonemes]` format + val regex = Regex("\\s\\[([^\\[\\]]*)\\]$") + val match = regex.find(rawLyrics) + val (lyric, phoneme) = if (match == null) { + rawLyrics to null + } else { + rawLyrics.take(match.range.first) to rawLyrics.substring(match.range).trim().trim('[', ']') + } Note( id = 0, key = it.tone, - lyric = it.lyric, + lyric = lyric, + phoneme = phoneme, tickOn = it.position + tickPrefix, tickOff = it.position + it.duration + tickPrefix, ) @@ -265,12 +275,18 @@ object Ustx { if (index == 0) datum.copy(y = firstPitchPointValue) else datum.copy() } val pitch = template.pitch.copy(data = pitchPoints) + val lyric = buildString { + append(thisNote.lyric) + if (thisNote.phoneme?.isNotBlank() == true) { + append(" [${thisNote.phoneme}]") + } + } return Note( position = thisNote.tickOn, duration = thisNote.length, tone = thisNote.key, pitch = pitch, - lyric = thisNote.lyric, + lyric = lyric, vibrato = template.vibrato.copy(), ) } diff --git a/src/jsMain/kotlin/model/Feature.kt b/src/jsMain/kotlin/model/Feature.kt index b373ec6..128df87 100644 --- a/src/jsMain/kotlin/model/Feature.kt +++ b/src/jsMain/kotlin/model/Feature.kt @@ -13,6 +13,11 @@ enum class Feature(val isAvailable: (Project) -> Boolean) { ), SplitProject( isAvailable = { true }, + ), + ConvertPhonemes( + isAvailable = { project -> + project.tracks.any { track -> track.notes.any { note -> note.phoneme != null } } + }, ) } diff --git a/src/jsMain/kotlin/model/Format.kt b/src/jsMain/kotlin/model/Format.kt index e3ab683..88f1e3f 100644 --- a/src/jsMain/kotlin/model/Format.kt +++ b/src/jsMain/kotlin/model/Format.kt @@ -1,6 +1,7 @@ package model import io.VsqLike +import model.Feature.ConvertPhonemes import model.Feature.ConvertPitch import model.Feature.SplitProject import model.JapaneseLyricsType.KanaCv @@ -31,7 +32,7 @@ enum class Format( io.Vsqx.generate(project, features) }, possibleLyricsTypes = listOf(RomajiCv, KanaCv), - availableFeaturesForGeneration = listOf(ConvertPitch), + availableFeaturesForGeneration = listOf(ConvertPitch, ConvertPhonemes), ), Vpr( "vpr", @@ -42,7 +43,7 @@ enum class Format( io.Vpr.generate(project, features) }, possibleLyricsTypes = listOf(RomajiCv, KanaCv), - availableFeaturesForGeneration = listOf(ConvertPitch), + availableFeaturesForGeneration = listOf(ConvertPitch, ConvertPhonemes), ), Ust( "ust", @@ -66,7 +67,7 @@ enum class Format( io.Ustx.generate(project, features) }, possibleLyricsTypes = listOf(RomajiCv, RomajiVcv, KanaCv, KanaVcv), - availableFeaturesForGeneration = listOf(ConvertPitch), + availableFeaturesForGeneration = listOf(ConvertPitch, ConvertPhonemes), ), Ccs( "ccs", @@ -88,7 +89,7 @@ enum class Format( io.Svp.generate(project, features) }, possibleLyricsTypes = listOf(RomajiCv, KanaCv), - availableFeaturesForGeneration = listOf(ConvertPitch, SplitProject), + availableFeaturesForGeneration = listOf(ConvertPitch, SplitProject, ConvertPhonemes), ), S5p( "s5p", @@ -181,7 +182,7 @@ enum class Format( io.UfData.generate(project, features) }, possibleLyricsTypes = listOf(RomajiCv, RomajiVcv, KanaCv, KanaVcv), - availableFeaturesForGeneration = listOf(ConvertPitch), + availableFeaturesForGeneration = listOf(ConvertPitch, ConvertPhonemes), ); private val allExtensions get() = listOf(extension) + otherExtensions @@ -235,5 +236,8 @@ enum class Format( StandardMid, UfData, ) + + val vocaloidFormats: List + get() = listOf(Vsq, Vsqx, VocaloidMid, Vpr) } } diff --git a/src/jsMain/kotlin/process/lyrics/LyricsMapping.kt b/src/jsMain/kotlin/process/lyrics/LyricsMapping.kt index b3be822..dc9f16f 100644 --- a/src/jsMain/kotlin/process/lyrics/LyricsMapping.kt +++ b/src/jsMain/kotlin/process/lyrics/LyricsMapping.kt @@ -44,7 +44,7 @@ fun Project.mapLyrics(request: LyricsMappingRequest) = copy( tracks = tracks.map { it.replaceLyrics(request) }, ) -fun Track.replaceLyrics(request: LyricsMappingRequest) = copy( +private fun Track.replaceLyrics(request: LyricsMappingRequest) = copy( notes = notes.mapNotNull { note -> note.replaceLyrics(request).takeIf { it.lyric.isNotEmpty() } } .validateNotes(), ) diff --git a/src/jsMain/kotlin/process/phonemes/PhonemesMapping.kt b/src/jsMain/kotlin/process/phonemes/PhonemesMapping.kt index 83ca5e9..d314f71 100644 --- a/src/jsMain/kotlin/process/phonemes/PhonemesMapping.kt +++ b/src/jsMain/kotlin/process/phonemes/PhonemesMapping.kt @@ -33,17 +33,18 @@ data class PhonemesMappingRequest( } } -fun Project.mapPhonemes(request: PhonemesMappingRequest) = copy( +fun Project.mapPhonemes(request: PhonemesMappingRequest?) = copy( tracks = tracks.map { it.replacePhonemes(request) }, ) -fun Track.replacePhonemes(request: PhonemesMappingRequest) = copy( - notes = notes.mapNotNull { note -> note.replacePhonemes(request).takeIf { it.lyric.isNotEmpty() } } +fun Track.replacePhonemes(request: PhonemesMappingRequest?) = copy( + notes = notes.mapNotNull { note -> note.replacePhonemes(request) } .validateNotes(), ) -fun Note.replacePhonemes(request: PhonemesMappingRequest): Note { +fun Note.replacePhonemes(request: PhonemesMappingRequest?): Note { val input = phoneme?.split(" ") ?: return this + if (request == null) return copy(phoneme = null) val output = mutableListOf() var pos = 0 while (pos <= input.lastIndex) { diff --git a/src/jsMain/kotlin/ui/ConfigurationEditor.kt b/src/jsMain/kotlin/ui/ConfigurationEditor.kt index 9f2da1f..282993f 100644 --- a/src/jsMain/kotlin/ui/ConfigurationEditor.kt +++ b/src/jsMain/kotlin/ui/ConfigurationEditor.kt @@ -30,6 +30,8 @@ import process.lyrics.chinese.convertChineseLyricsToPinyin import process.lyrics.japanese.convertJapaneseLyrics import process.lyrics.mapLyrics import process.lyrics.replaceLyrics +import process.phonemes.PhonemesMappingRequest +import process.phonemes.mapPhonemes import process.projectZoomFactorOptions import process.zoom import react.ChildrenBuilder @@ -50,6 +52,7 @@ import ui.configuration.ChinesePinyinConversionBlock import ui.configuration.JapaneseLyricsConversionBlock import ui.configuration.LyricsMappingBlock import ui.configuration.LyricsReplacementBlock +import ui.configuration.PhonemesConversionBlock import ui.configuration.PitchConversionBlock import ui.configuration.ProjectSplitBlock import ui.configuration.ProjectZoomBlock @@ -170,6 +173,31 @@ val ConfigurationEditor = scopedFC { props, scope -> ?: FeatureConfig.SplitProject.getDefault(props.outputFormat).maxTrackCount.toString(), ) } + val (phonemesConversion, setPhonemesConversion) = useState { + currentConfigs?.phonemesConversion?.let { + return@useState it + } + + val hasPhonemes = props.projects.any { Feature.ConvertPhonemes.isAvailable(it) } + val isAvailable = hasPhonemes && + props.outputFormat.availableFeaturesForGeneration.contains(Feature.ConvertPhonemes) + // Enable for VOCALOID -> VOCALOID or UfData by default + val enabledFormats = Format.vocaloidFormats.plus(Format.UfData) + val isOn = isAvailable && ( + ( + props.projects.all { it.format in enabledFormats } && + props.outputFormat in enabledFormats + ) || + props.projects.all { it.format == props.outputFormat } + ) + PhonemesConversionState( + isAvailable = isAvailable, + isOn = isOn, + useMapping = false, + mappingPresetName = null, + mappingRequest = PhonemesMappingRequest(), + ) + } var dialogError by useState(DialogErrorState()) fun isReady() = listOf( @@ -206,6 +234,10 @@ val ConfigurationEditor = scopedFC { props, scope -> initialState = lyricsMapping submitState = setLyricsMapping } + if (phonemesConversion.isAvailable) PhonemesConversionBlock { + initialState = phonemesConversion + submitState = setPhonemesConversion + } SlightRestsFillingBlock { initialState = slightRestsFilling submitState = setSlightRestsFilling @@ -235,6 +267,7 @@ val ConfigurationEditor = scopedFC { props, scope -> pitchConversion, projectZoom, projectSplit, + phonemesConversion, setProgress = { progress = it }, onDialogError = { dialogError = it }, ) @@ -261,6 +294,7 @@ private fun ChildrenBuilder.buildNextButton( pitchConversion: PitchConversionState, projectZoom: ProjectZoomState, projectSplit: ProjectSplitState, + phonemesConversion: PhonemesConversionState, setProgress: (ProgressProps) -> Unit, onDialogError: (DialogErrorState) -> Unit, ) { @@ -284,6 +318,7 @@ private fun ChildrenBuilder.buildNextButton( pitchConversion, projectZoom, projectSplit, + phonemesConversion, setProgress, onDialogError, ) @@ -304,6 +339,7 @@ private fun process( pitchConversion: PitchConversionState, projectZoom: ProjectZoomState, projectSplit: ProjectSplitState, + phonemesConversion: PhonemesConversionState, setProgress: (ProgressProps) -> Unit, onDialogError: (DialogErrorState) -> Unit, ) { @@ -335,6 +371,7 @@ private fun process( .runIf(projectZoom.isOn) { zoom(projectZoom.factorValue) } + .mapPhonemes(phonemesConversion.resolvedMappingRequest) val featureConfigs = buildList { if (pitchConversion.isOn) add(FeatureConfig.ConvertPitch) @@ -358,6 +395,7 @@ private fun process( pitchConversion, projectZoom, projectSplit, + phonemesConversion, ) window.localStorage.setItem("currentConfigs", json.encodeToString(currentConfigs)) @@ -453,6 +491,34 @@ data class LyricsMappingState( } } +@Serializable +data class PhonemesConversionState( + val isAvailable: Boolean, + val isOn: Boolean, + val useMapping: Boolean, + val mappingPresetName: String?, + val mappingRequest: PhonemesMappingRequest, +) : SubState() { + override val isReady: Boolean get() = if (isAvailable && isOn && useMapping) mappingRequest.isValid else true + + val resolvedMappingRequest: PhonemesMappingRequest? + get() = if (isOn) { + if (useMapping) { + mappingRequest + } else { + PhonemesMappingRequest("") + } + } else { + null + } + + fun updatePresetName() = when { + mappingPresetName == null -> this + PhonemesMappingRequest.findPreset(mappingPresetName) == mappingRequest -> this + else -> copy(mappingPresetName = null) + } +} + @Serializable data class SlightRestsFillingState( val isOn: Boolean, @@ -502,4 +568,5 @@ data class Configs( val pitchConversion: PitchConversionState, val projectZoom: ProjectZoomState, val projectSplit: ProjectSplitState, + val phonemesConversion: PhonemesConversionState, ) diff --git a/src/jsMain/kotlin/ui/configuration/PhonemesConversion.kt b/src/jsMain/kotlin/ui/configuration/PhonemesConversion.kt new file mode 100644 index 0000000..7834eb1 --- /dev/null +++ b/src/jsMain/kotlin/ui/configuration/PhonemesConversion.kt @@ -0,0 +1,202 @@ +package ui.configuration + +import csstype.AlignItems +import csstype.Display +import csstype.FlexDirection +import csstype.Length +import csstype.Margin +import csstype.VerticalAlign +import csstype.WhiteSpace +import csstype.em +import csstype.px +import emotion.react.css +import kotlinx.js.jso +import mui.icons.material.HelpOutline +import mui.material.BaseTextFieldProps +import mui.material.Button +import mui.material.ButtonColor +import mui.material.ButtonVariant +import mui.material.FormControl +import mui.material.FormControlMargin +import mui.material.FormControlVariant +import mui.material.FormGroup +import mui.material.FormLabel +import mui.material.MenuItem +import mui.material.Paper +import mui.material.StandardTextFieldProps +import mui.material.TextField +import mui.material.Tooltip +import mui.material.TooltipPlacement +import mui.material.Typography +import mui.material.styles.TypographyVariant +import process.phonemes.PhonemesMappingRequest +import react.ChildrenBuilder +import react.create +import react.dom.html.ReactHTML.div +import ui.PhonemesConversionState +import ui.common.SubProps +import ui.common.configurationSwitch +import ui.common.subFC +import ui.strings.Strings +import ui.strings.string + +external interface PhonemesConversionProps : SubProps + +val PhonemesConversionBlock = subFC { _, state, editState -> + FormGroup { + div { + configurationSwitch( + isOn = state.isOn, + onSwitched = { editState { copy(isOn = it) } }, + labelStrings = Strings.PhonemesConversion, + ) + } + } + + if (state.isOn) buildPhonemesConversionDetail(state, editState) +} + +private fun ChildrenBuilder.buildPhonemesConversionDetail( + state: PhonemesConversionState, + editState: (PhonemesConversionState.() -> PhonemesConversionState) -> Unit, +) { + div { + css { + margin = Margin(horizontal = 40.px, vertical = 0.px) + width = Length.maxContent + } + Paper { + elevation = 0 + div { + css { + margin = Margin( + horizontal = 24.px, + top = 16.px, + bottom = 24.px, + ) + } + div { + style = jso { + paddingTop = 16.px + paddingBottom = 8.px + } + configurationSwitch( + isOn = state.useMapping, + onSwitched = { + editState { + copy(useMapping = it).updatePresetName() + } + }, + labelStrings = Strings.PhonemesConversionEnableMapping, + ) + Tooltip { + val text = string(Strings.PhonemesConversionEnableMappingDescription) + title = div.create { + css { whiteSpace = WhiteSpace.preLine } + +text + } + placement = TooltipPlacement.right + disableInteractive = false + HelpOutline { + style = jso { + verticalAlign = VerticalAlign.middle + } + } + } + } + FormGroup { + div { + style = jso { + display = Display.flex + flexDirection = FlexDirection.row + alignItems = AlignItems.flexEnd + } + FormControl { + margin = FormControlMargin.normal + variant = FormControlVariant.standard + disabled = state.useMapping.not() + focused = false + FormLabel { + focused = false + Typography { + variant = TypographyVariant.caption + +string(Strings.PhonemesMappingPreset) + } + } + TextField { + disabled = state.useMapping.not() + style = jso { minWidth = 16.em } + select = true + value = state.mappingPresetName.orEmpty().unsafeCast() + (this.unsafeCast()).variant = FormControlVariant.standard + (this.unsafeCast()).onChange = { event -> + val value = event.target.asDynamic().value as String + editState { + copy( + mappingPresetName = value, + mappingRequest = PhonemesMappingRequest.getPreset(value), + ) + } + } + PhonemesMappingRequest.Presets.forEach { preset -> + MenuItem { + value = preset.first + +(preset.first) + } + } + } + } + Button { + style = jso { + marginLeft = 24.px + marginBottom = 12.px + } + disabled = state.useMapping.not() + variant = ButtonVariant.outlined + color = ButtonColor.secondary + onClick = { + editState { + copy( + mappingPresetName = null, + mappingRequest = PhonemesMappingRequest(), + ) + } + } + div { + +string(Strings.PhonemesMappingPresetClear) + } + } + } + div { + TextField { + disabled = state.useMapping.not() + multiline = true + style = jso { + marginTop = 8.px + marginBottom = 16.px + width = 25.em + } + (this.unsafeCast()).InputProps = jso { + style = jso { + paddingTop = 12.px + paddingBottom = 12.px + } + } + minRows = 10 + maxRows = 10 + placeholder = string(Strings.PhonemesMappingMapPlaceholder) + value = state.mappingRequest.mapText + (this.unsafeCast()).variant = FormControlVariant.filled + (this.unsafeCast()).onChange = { event -> + val value = event.target.asDynamic().value as String + editState { + copy(mappingRequest = mappingRequest.copy(mapText = value)).updatePresetName() + } + } + } + } + } + } + } + } +} diff --git a/src/jsMain/kotlin/ui/strings/Strings.kt b/src/jsMain/kotlin/ui/strings/Strings.kt index c8b2c0c..e1a7e5f 100644 --- a/src/jsMain/kotlin/ui/strings/Strings.kt +++ b/src/jsMain/kotlin/ui/strings/Strings.kt @@ -313,6 +313,54 @@ enum class Strings( ru = "Запишите запись отображения на строку в формате \"{from}={to}\".", fr = "Écrivez une entrée de mappage par ligne au format \"{from}={to}\".", ), + PhonemesConversion( + en = "Convert phonemes", + ja = "発音記号を変換", + zhCN = "转换音素", + ru = "Конвертировать фонемы", + fr = "Convertir les phonèmes", + ), + PhonemesConversionEnableMapping( + en = "Map phonemes", + ja = "発音記号をマッピング", + zhCN = "映射音素", + ru = "Отобразить фонемы", + fr = "Mapper les phonèmes", + ), + PhonemesConversionEnableMappingDescription( + en = "Enable phonemes mapping with a customizable mapping table. If disabled, phonemes are copied as is.", + ja = "カスタマイズ可能なマッピングテーブルを使用して発音記号をマッピングします。無効にすると、発音記号はそのままコピーされます。", + zhCN = "使用可自定义的映射表映射音素。如果禁用，音素将被直接复制。", + ru = "Включить отображение фонем с помощью настраиваемой таблицы отображения. " + + "Если отключено, фонемы копируются как есть.", + fr = "Activer le mappage des phonèmes avec une table de mappage personnalisable. " + + "Si désactivé, les phonèmes sont copiés tels quels.", + ), + PhonemesMappingPreset( + en = "Preset", + ja = "プリセット", + zhCN = "预设", + ru = "Пресет", + fr = "Préréglage", + ), + PhonemesMappingPresetClear( + en = "Clear", + ja = "クリア", + zhCN = "清空", + ru = "Очистить", + fr = "Effacer", + ), + PhonemesMappingMapPlaceholder( + en = "Write a mapping entry per line in the format of \"{from}={to}\". " + + "Whitespace \" \" can be used to combine multiple phonemes as a set.", + ja = "「{from}={to}」の形式で、一行に一つのマッピングエントリーを書き込んでください。" + + "複数の発音記号をセットとして扱う場合は、空白「 」を使用してください。", + zhCN = "请按照“{from}={to}”的格式，每行写入一个映射条目。空格“ ”可以用于将多个音素组合成一个集合。", + ru = "Запишите запись отображения на строку в формате \"{from}={to}\". " + + "Пробел \" \" можно использовать для объединения нескольких фонем в качестве набора.", + fr = "Écrivez une entrée de mappage par ligne au format \"{from}={to}\". " + + "L'espace blanc \" \" peut être utilisé pour combiner plusieurs phonèmes en un ensemble.", + ), ConvertPitchData( en = "Convert pitch parameters", ja = "ピッチパラメータを変換",