From a71bb732da076777edd2dc1f5b8468c4e9e04e80 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 14 Aug 2021 18:14:37 +0300 Subject: [PATCH 01/12] Fix meta node listeners and attachements --- CHANGELOG.md | 1 + build.gradle.kts | 2 +- .../kscience/dataforge/meta/MutableMeta.kt | 54 ++++++++++--------- .../kscience/dataforge/meta/ObservableMeta.kt | 2 +- .../dataforge/meta/ObservableMetaWrapper.kt | 2 +- 5 files changed, 33 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a874fa5..b0063e5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Removed ### Fixed +- MutableMetaImpl attachment and checks ### Security ## [0.5.0] diff --git a/build.gradle.kts b/build.gradle.kts index 82f63b33..d49db3c4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.5.0" + version = "0.5.1" } subprojects { diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 527b6cbe..0c2c653e 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -261,6 +261,13 @@ public operator fun > MutableTypedMeta.set(name: Name } } +private fun ObservableMeta.adoptBy(parent: MutableMetaImpl, key: NameToken) { + if (this === parent) error("Can't attach a node to itself") + onChange(parent) { name -> + parent.invalidate(key + name) + } +} + /** * A general implementation of mutable [Meta] which implements both [MutableTypedMeta] and [ObservableMeta]. * The implementation uses blocking synchronization on mutation on JVM @@ -280,17 +287,11 @@ private class MutableMetaImpl( private val children: LinkedHashMap = LinkedHashMap(children.mapValues { (key, meta) -> - MutableMetaImpl(meta.value, meta.items).apply { adoptBy(this, key) } + MutableMetaImpl(meta.value, meta.items).also { it.adoptBy(this, key) } }) override val items: Map get() = children - private fun ObservableMeta.adoptBy(parent: MutableMetaImpl, key: NameToken) { - onChange(parent) { name -> - parent.invalidate(key + name) - } - } - @DFExperimental override fun attach(name: Name, node: ObservableMutableMeta) { when (name.length) { @@ -338,9 +339,14 @@ private class MutableMetaImpl( } private fun wrapItem(meta: Meta): MutableMetaImpl = - MutableMetaImpl(meta.value, meta.items.mapValuesTo(LinkedHashMap()) { wrapItem(it.value) }) - + meta as? MutableMetaImpl ?: MutableMetaImpl( + meta.value, + meta.items.mapValuesTo(LinkedHashMap()) { + wrapItem(it.value) + } + ) + @Synchronized override fun setMeta(name: Name, node: Meta?) { val oldItem: ObservableMutableMeta? = get(name) if (oldItem != node) { @@ -348,13 +354,24 @@ private class MutableMetaImpl( 0 -> error("Can't set a meta with empty name") 1 -> { val token = name.firstOrNull()!! - replaceItem(token, oldItem, node?.let { wrapItem(node) }) + //remove child and invalidate if argument is null + if (node == null) { + children.remove(token)?.removeListener(this) + // old item is not null otherwise we can't be here + invalidate(name) + } else { + val newNode = wrapItem(node) + newNode.adoptBy(this, token) + children[token] = newNode + } } else -> { val token = name.firstOrNull()!! - //get existing or create new node. Index is ignored for new node + //get existing or create new node. if (items[token] == null) { - replaceItem(token, null, MutableMetaImpl(null)) + val newNode = MutableMetaImpl(null) + newNode.adoptBy(this, token) + children[token] = newNode } items[token]?.setMeta(name.cutFirst(), node) } @@ -384,19 +401,6 @@ public fun MutableMeta.append(name: Name, value: Value): Unit = append(name, Met public fun MutableMeta.append(key: String, value: Value): Unit = append(Name.parse(key), value) -///** -// * Apply existing node with given [builder] or create a new element with it. -// */ -//@DFExperimental -//public fun MutableMeta.edit(name: Name, builder: MutableMeta.() -> Unit) { -// val item = when (val existingItem = get(name)) { -// null -> MutableMeta().also { set(name, it) } -// is MetaItemNode -> existingItem.node -// else -> error("Can't edit value meta item") -// } -// item.apply(builder) -//} - /** * Create a mutable copy of this meta. The copy is created even if the Meta is already mutable */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt index 73ec03b6..f2b80417 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt @@ -48,7 +48,7 @@ public interface ObservableMutableMeta : ObservableMeta, MutableMeta, MutableTyp } internal abstract class AbstractObservableMeta : ObservableMeta { - private val listeners = HashSet() + private val listeners = ArrayList() override fun invalidate(name: Name) { listeners.forEach { it.callback(this, name) } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt index 53e427ef..b19235b6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMetaWrapper.kt @@ -68,7 +68,7 @@ private class ObservableMetaWrapper( override fun attach(name: Name, node: ObservableMutableMeta) { set(name, node) node.onChange(this) { changeName -> - setMeta(name + changeName, node[changeName]) + setMeta(name + changeName, this[changeName]) } } } From 0ad6852e3631636a2c36cbe3df845ba5e6dac827 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 15 Aug 2021 13:20:54 +0300 Subject: [PATCH 02/12] Fix meta node listeners and attachements --- CHANGELOG.md | 1 + .../kotlin/space/kscience/dataforge/meta/ObservableMeta.kt | 2 +- .../kotlin/space/kscience/dataforge/meta/Scheme.kt | 7 +++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0063e5c..14f6ed73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Fixed - MutableMetaImpl attachment and checks +- Listeners in observable meta are replaced by lists ### Security ## [0.5.0] diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt index f2b80417..8fa432cc 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/ObservableMeta.kt @@ -48,7 +48,7 @@ public interface ObservableMutableMeta : ObservableMeta, MutableMeta, MutableTyp } internal abstract class AbstractObservableMeta : ObservableMeta { - private val listeners = ArrayList() + private val listeners: MutableList = mutableListOf() override fun invalidate(name: Name) { listeners.forEach { it.callback(this, name) } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index d4f40eb4..2bc0e234 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -1,6 +1,9 @@ package space.kscience.dataforge.meta -import space.kscience.dataforge.meta.descriptors.* +import space.kscience.dataforge.meta.descriptors.Described +import space.kscience.dataforge.meta.descriptors.MetaDescriptor +import space.kscience.dataforge.meta.descriptors.get +import space.kscience.dataforge.meta.descriptors.validate import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.* import space.kscience.dataforge.values.Value @@ -64,7 +67,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl override fun toMeta(): Laminate = Laminate(meta, descriptor?.defaultNode) - private val listeners = HashSet() + private val listeners: MutableList = mutableListOf() private inner class SchemeMeta(val pathName: Name) : ObservableMutableMeta { override var value: Value? From 81cdd38c4048ab6f9935c20e8bff83bcacb58d3d Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 28 Sep 2021 12:26:50 +0000 Subject: [PATCH 03/12] Build --- .space.kts | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .space.kts diff --git a/.space.kts b/.space.kts new file mode 100644 index 00000000..45b2ed53 --- /dev/null +++ b/.space.kts @@ -0,0 +1,4 @@ +job("Build") { + gradlew("openjdk:11", "build") +} + From b07d281a83c9bbd3731e30c0801405e3812f64d9 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 29 Sep 2021 11:08:55 +0300 Subject: [PATCH 04/12] Kotlin 1.6 --- build.gradle.kts | 6 +++--- .../kotlin/space/kscience/dataforge/data/Goal.kt | 2 -- .../space/kscience/dataforge/meta/Scheme.kt | 1 - .../space/kscience/dataforge/values/Value.kt | 4 +--- .../dataforge/meta/MetaSerializationTest.kt | 16 ++++++++++++++++ gradle/wrapper/gradle-wrapper.properties | 2 +- settings.gradle.kts | 2 +- 7 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index d49db3c4..fe2e7f58 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,6 +5,9 @@ plugins { allprojects { group = "space.kscience" version = "0.5.1" + repositories{ + mavenCentral() + } } subprojects { @@ -22,8 +25,5 @@ ksciencePublish { } apiValidation { - if(project.version.toString().contains("dev")) { - validationDisabled = true - } nonPublicMarkers.add("space.kscience.dataforge.misc.DFExperimental") } \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt index e41c124a..b7ddea96 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt @@ -1,7 +1,6 @@ package space.kscience.dataforge.data import kotlinx.coroutines.* -import space.kscience.dataforge.misc.DFExperimental import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext @@ -67,7 +66,6 @@ public open class LazyGoal( * If [GoalExecutionRestriction] is present in the [coroutineScope] context, the call could produce a error a warning * depending on the settings. */ - @DFExperimental override fun async(coroutineScope: CoroutineScope): Deferred { val log = coroutineScope.coroutineContext[GoalLogger] // Check if context restricts goal computation diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index 2bc0e234..ffc29152 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -123,7 +123,6 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl TODO("Not yet implemented") } - } } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt index e2a83b9c..707c86f6 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt @@ -166,9 +166,7 @@ public class EnumValue>(override val value: E) : Value { override fun toString(): String = value.toString() - override fun equals(other: Any?): Boolean { - return string == (other as? Value)?.string - } + override fun equals(other: Any?): Boolean = string == (other as? Value)?.string override fun hashCode(): Int = value.hashCode() } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt new file mode 100644 index 00000000..10e76f90 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MetaSerializationTest.kt @@ -0,0 +1,16 @@ +package space.kscience.dataforge.meta + +import kotlinx.serialization.json.Json +import space.kscience.dataforge.values.string +import kotlin.test.Test +import kotlin.test.assertEquals + +class MetaSerializationTest { + + @Test + fun singleValueDeserialization(){ + val string = "ddd" + val meta = Json.decodeFromString(MetaSerializer, string) + assertEquals(string, meta.value?.string) + } +} \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f371643e..ffed3a25 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/settings.gradle.kts b/settings.gradle.kts index 6d8e56a0..26f6fcc4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,7 +5,7 @@ pluginManagement { gradlePluginPortal() } - val toolsVersion = "0.10.2" + val toolsVersion = "0.10.4" plugins { id("ru.mipt.npm.gradle.project") version toolsVersion From 00d964eef3bbf3ef00df9c1463f646bc90cf4ab7 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Fri, 8 Oct 2021 09:54:11 +0300 Subject: [PATCH 05/12] Workspace and task updates --- .../dataforge/workspace/SimpleWorkspace.kt | 2 +- .../kscience/dataforge/workspace/Task.kt | 10 +++++--- .../kscience/dataforge/workspace/TaskData.kt | 2 +- .../dataforge/workspace/TaskResult.kt | 11 +++++---- .../kscience/dataforge/workspace/Workspace.kt | 5 ++++ .../dataforge/workspace/WorkspaceBuilder.kt | 13 +++++++++- .../dataforge/workspace/taskBuilders.kt | 9 +++++++ .../dataforge/workspace/taskBuilders.kt | 24 ------------------- .../workspace/workspaceExtensions.kt | 11 ++++++++- 9 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt delete mode 100644 dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt index 7e881a8f..e800d8b3 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/SimpleWorkspace.kt @@ -17,7 +17,7 @@ public class SimpleWorkspace( private val externalTasks: Map>, ) : Workspace { - override val data: TaskResult<*> = internalize(data, Name.EMPTY, Meta.EMPTY) + override val data: TaskResult<*> = wrapResult(data, Name.EMPTY, Meta.EMPTY) override val tasks: Map> get() = context.gather>(Task.TYPE) + externalTasks diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt index 965f859e..dcf63db2 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Task.kt @@ -28,7 +28,7 @@ public interface Task : Described { public suspend fun execute(workspace: Workspace, taskName: Name, taskMeta: Meta): TaskResult public companion object { - public const val TYPE: String = "workspace.stage" + public const val TYPE: String = "workspace.task" } } @@ -42,6 +42,10 @@ public class TaskResultBuilder( /** * Create a [Task] that composes a result using [builder]. Only data from the workspace could be used. * Data dependency cycles are not allowed. + * + * @param resultType the type boundary for data produced by this task + * @param descriptor of meta accepted by this task + * @param builder for resulting data set */ @Suppress("FunctionName") @DFInternal @@ -60,9 +64,9 @@ public fun Task( ): TaskResult = withContext(GoalExecutionRestriction() + workspace.goalLogger) { //TODO use safe builder and check for external data on add and detects cycles val dataset = DataTree(resultType) { - TaskResultBuilder(workspace,taskName, taskMeta, this).apply { builder() } + TaskResultBuilder(workspace, taskName, taskMeta, this).apply { builder() } } - workspace.internalize(dataset, taskName, taskMeta) + workspace.wrapResult(dataset, taskName, taskMeta) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt index ff501600..6d14616d 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt @@ -42,6 +42,6 @@ private class TaskDataImpl( // } } -internal fun Workspace.internalize(data: Data, name: Name, stage: Name, stageMeta: Meta): TaskData = +internal fun Workspace.wrapResult(data: Data, name: Name, stage: Name, stageMeta: Meta): TaskData = TaskDataImpl(this, data, name, stage, stageMeta) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index d0097e55..61e4c650 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -37,13 +37,16 @@ private class TaskResultImpl( ) : TaskResult, DataSet by dataSet { override fun flow(): Flow> = dataSet.flow().map { - workspace.internalize(it, it.name, taskName, taskMeta) + workspace.wrapResult(it, it.name, taskName, taskMeta) } override suspend fun getData(name: Name): TaskData? = dataSet.getData(name)?.let { - workspace.internalize(it, name, taskName, taskMeta) + workspace.wrapResult(it, name, taskName, taskMeta) } } -internal fun Workspace.internalize(dataSet: DataSet, stage: Name, stageMeta: Meta): TaskResult = - TaskResultImpl(this, dataSet, stage, stageMeta) \ No newline at end of file +/** + * Wrap data into [TaskResult] + */ +public fun Workspace.wrapResult(dataSet: DataSet, taskName: Name, taskMeta: Meta): TaskResult = + TaskResultImpl(this, dataSet, taskName, taskMeta) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt index e5458258..bc2eb7dd 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/Workspace.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.context.ContextAware +import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.MutableMeta import space.kscience.dataforge.misc.Type @@ -8,6 +9,10 @@ import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider +public interface DataSelector{ + public suspend fun select(workspace: Workspace, meta: Meta): DataSet +} + @Type(Workspace.TYPE) public interface Workspace : ContextAware, Provider { /** diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 423a1807..72a35da7 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -17,7 +17,18 @@ import space.kscience.dataforge.names.Name import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty -public data class TaskReference(public val taskName: Name, public val task: Task) +public data class TaskReference(public val taskName: Name, public val task: Task): DataSelector{ + + @Suppress("UNCHECKED_CAST") + override suspend fun select(workspace: Workspace, meta: Meta): DataSet { + if (workspace.tasks[taskName] == task) { + return workspace.produce(taskName, meta) as TaskResult + } else { + error("Task $taskName does not belong to the workspace") + } + } + +} public interface TaskContainer { public fun registerTask(taskName: Name, task: Task<*>) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt new file mode 100644 index 00000000..bcfeae32 --- /dev/null +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -0,0 +1,9 @@ +package space.kscience.dataforge.workspace + +import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.meta.Meta + +public suspend fun TaskResultBuilder<*>.from( + selector: DataSelector, + meta: Meta = Meta.EMPTY, +): DataSet = selector.select(workspace, meta) diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt deleted file mode 100644 index d5ab5313..00000000 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ /dev/null @@ -1,24 +0,0 @@ -package space.kscience.dataforge.workspace - -import space.kscience.dataforge.data.DataSet -import space.kscience.dataforge.data.select -import space.kscience.dataforge.meta.Meta -import space.kscience.dataforge.names.Name - -public suspend inline fun TaskResultBuilder.from( - task: Name, - taskMeta: Meta = Meta.EMPTY, -): DataSet = workspace.produce(task, taskMeta).select() - - -@Suppress("UNCHECKED_CAST") -public suspend fun TaskResultBuilder<*>.from( - reference: TaskReference, - taskMeta: Meta = Meta.EMPTY, -): DataSet { - if (workspace.tasks[reference.taskName] == reference.task) { - return workspace.produce(reference.taskName, taskMeta) as TaskResult - } else { - throw error("Task ${reference.taskName} does not belong to the workspace") - } -} diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt index 543b0da1..2690a8f6 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt @@ -1,8 +1,17 @@ package space.kscience.dataforge.workspace import kotlinx.coroutines.runBlocking +import space.kscience.dataforge.data.DataSet import space.kscience.dataforge.data.DataSetBuilder +import space.kscience.dataforge.data.select +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.Name public fun WorkspaceBuilder.data(builder: suspend DataSetBuilder.() -> Unit): Unit = runBlocking { buildData(builder) -} \ No newline at end of file +} + +public suspend inline fun TaskResultBuilder<*>.from( + task: Name, + taskMeta: Meta = Meta.EMPTY, +): DataSet = workspace.produce(task, taskMeta).select() \ No newline at end of file From aded38254e647d7cb9ffad6acea653060007ce46 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 9 Oct 2021 10:49:38 +0300 Subject: [PATCH 06/12] Workspace and task updates --- .../space/kscience/dataforge/data/select.kt | 10 +++-- .../kscience/dataforge/workspace/TaskData.kt | 4 +- .../dataforge/workspace/TaskResult.kt | 4 +- .../dataforge/workspace/taskBuilders.kt | 40 ++++++++++++++++++- ...workspaceExtensions.kt => workspaceJvm.kt} | 4 ++ .../workspace/SimpleWorkspaceTest.kt | 27 +++++++++---- 6 files changed, 71 insertions(+), 18 deletions(-) rename dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/{workspaceExtensions.kt => workspaceJvm.kt} (70%) diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt index 7d311159..2cbbc1c1 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt @@ -16,8 +16,12 @@ import kotlin.reflect.typeOf */ @Suppress("UNCHECKED_CAST") private fun Data<*>.castOrNull(type: KType): Data? = - if (!this.type.isSubtypeOf(type)) null else object : Data by (this as Data) { - override val type: KType = type + if (!this.type.isSubtypeOf(type)) { + null + } else { + object : Data by (this as Data) { + override val type: KType = type + } } /** @@ -31,7 +35,6 @@ internal fun DataSet<*>.select( ): ActiveDataSet = object : ActiveDataSet { override val dataType = type - override fun flow(): Flow> = this@select.flow().filter { datum -> datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern)) }.map { @@ -45,7 +48,6 @@ internal fun DataSet<*>.select( val datum = this@select.getData(it) datum?.type?.isSubtypeOf(type) ?: false } - } /** diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt index 6d14616d..bd2bdb48 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskData.kt @@ -42,6 +42,6 @@ private class TaskDataImpl( // } } -internal fun Workspace.wrapResult(data: Data, name: Name, stage: Name, stageMeta: Meta): TaskData = - TaskDataImpl(this, data, name, stage, stageMeta) +public fun Workspace.wrapData(data: Data, name: Name, taskName: Name, stageMeta: Meta): TaskData = + TaskDataImpl(this, data, name, taskName, stageMeta) diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index 61e4c650..5a65f219 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -37,11 +37,11 @@ private class TaskResultImpl( ) : TaskResult, DataSet by dataSet { override fun flow(): Flow> = dataSet.flow().map { - workspace.wrapResult(it, it.name, taskName, taskMeta) + workspace.wrapData(it, it.name, taskName, taskMeta) } override suspend fun getData(name: Name): TaskData? = dataSet.getData(name)?.let { - workspace.wrapResult(it, name, taskName, taskMeta) + workspace.wrapData(it, name, taskName, taskMeta) } } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt index bcfeae32..4744d415 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/taskBuilders.kt @@ -1,9 +1,45 @@ package space.kscience.dataforge.workspace import space.kscience.dataforge.data.DataSet +import space.kscience.dataforge.data.forEach +import space.kscience.dataforge.data.map import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.meta.toMutableMeta +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +/** + * Select data using given [selector] + */ public suspend fun TaskResultBuilder<*>.from( selector: DataSelector, - meta: Meta = Meta.EMPTY, -): DataSet = selector.select(workspace, meta) +): DataSet = selector.select(workspace, taskMeta) + +public val TaskResultBuilder<*>.allData: DataSelector<*> + get() = object : DataSelector { + override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data + } + +/** + * Perform a lazy mapping task using given [selector] and [action]. The meta of resulting + * TODO move selector to receiver with multi-receivers + */ +@DFExperimental +public suspend inline fun TaskResultBuilder.pipeFrom( + selector: DataSelector, + crossinline action: suspend (arg: T, name: Name, meta: Meta) -> R +) { + from(selector).forEach { data -> + val meta = data.meta.toMutableMeta().apply { + taskName put taskMeta + } + + val res = data.map(workspace.context.coroutineContext, meta) { + action(it, data.name, meta) + } + + emit(data.name, res) + } +} + + diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt similarity index 70% rename from dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt rename to dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt index 2690a8f6..61e1dbec 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceExtensions.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt @@ -11,6 +11,10 @@ public fun WorkspaceBuilder.data(builder: suspend DataSetBuilder.() -> Unit buildData(builder) } +public inline fun TaskResultBuilder<*>.selectData(namePattern: Name? = null): DataSelector = object : DataSelector { + override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data.select(namePattern) +} + public suspend inline fun TaskResultBuilder<*>.from( task: Name, taskMeta: Meta = Meta.EMPTY, diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index e5657444..edbe3a4b 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -71,21 +71,32 @@ class SimpleWorkspaceTest { } val square by task { - workspace.data.select().forEach { data -> - if (data.meta["testFlag"].boolean == true) { + pipeFrom(selectData()) { arg, name, meta -> + if (meta["testFlag"].boolean == true) { println("flag") } - val value = data.await() - workspace.logger.info { "Starting square on $value" } - emit(data.name, data.map { it * it }) + workspace.logger.info { "Starting square on $name" } + arg * arg } +// workspace.data.select().forEach { data -> +// if (data.meta["testFlag"].boolean == true) { +// println("flag") +// } +// val value = data.await() +// workspace.logger.info { "Starting square on $value" } +// emit(data.name, data.map { it * it }) +// } } val linear by task { - workspace.data.select().forEach { data -> - workspace.logger.info { "Starting linear on $data" } - emit(data.name, data.data.map { it * 2 + 1 }) + pipeFrom(selectData()) { arg, name, _ -> + workspace.logger.info { "Starting linear on $name" } + arg * 2 + 1 } +// workspace.data.select().forEach { data -> +// workspace.logger.info { "Starting linear on $data" } +// emit(data.name, data.data.map { it * 2 + 1 }) +// } } val fullSquare by task { From 3f54eee578c9f85d36a713b0e603d78d0fa52fd5 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sun, 10 Oct 2021 14:32:32 +0300 Subject: [PATCH 07/12] Workaround for https://youtrack.jetbrains.com/issue/KT-48988. Smart building of child contexts --- build.gradle.kts | 2 +- .../kscience/dataforge/context/Context.kt | 17 ++++----- .../dataforge/context/ContextBuilder.kt | 14 ++++---- .../kscience/dataforge/context/Global.kt | 6 ++-- .../kscience/dataforge/context/LogManager.kt | 2 +- .../dataforge/context/PluginManager.kt | 3 +- .../kscience/dataforge/context/ContextTest.kt | 3 +- .../kscience/dataforge/context/loggingJs.kt | 2 +- .../kscience/dataforge/context/loggingJvm.kt | 2 +- .../dataforge/context/loggingNative.kt | 2 +- .../space/kscience/dataforge/data/DataSet.kt | 4 +-- .../space/kscience/dataforge/data/select.kt | 36 +++++++++++++------ .../space/kscience/dataforge/io/IOPlugin.kt | 18 +++++----- .../dataforge/io/TaggedEnvelopeFormat.kt | 2 +- .../dataforge/io/TaglessEnvelopeFormat.kt | 6 ++-- .../dataforge/scripting/BuildersKtTest.kt | 4 +-- .../dataforge/workspace/WorkspaceBuilder.kt | 3 +- gradle.properties | 4 +-- 18 files changed, 72 insertions(+), 58 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index fe2e7f58..42505cb9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.5.1" + version = "0.5.2-dev-1" repositories{ mavenCentral() } diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt index 97efc76a..427d8611 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Context.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import space.kscience.dataforge.meta.* +import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.misc.Named import space.kscience.dataforge.names.Name import space.kscience.dataforge.provider.Provider @@ -71,16 +72,16 @@ public open class Context internal constructor( private val childrenContexts = HashMap() /** - * Build and register a child context + * Get and validate existing context or build and register a new child context. + * @param name the relative (tail) name of the new context. If null, uses context hash code as a marker. */ + @OptIn(DFExperimental::class) @Synchronized - public fun buildContext(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context { - val newContext = ContextBuilder(this) - .apply { name?.let { name(it) } } - .apply(block) - .build() - childrenContexts[newContext.name] = newContext - return newContext + public fun buildContext(name: Name? = null, block: ContextBuilder.() -> Unit = {}): Context { + val existing = name?.let { childrenContexts[name] } + return existing?.modify(block)?: ContextBuilder(this, name).apply(block).build().also { + childrenContexts[it.name] = it + } } /** diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt index 0de5fb7a..80329176 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/ContextBuilder.kt @@ -20,7 +20,7 @@ import kotlin.collections.set @DFBuilder public class ContextBuilder internal constructor( private val parent: Context, - public var name: Name? = null, + public val name: Name? = null, meta: Meta = Meta.EMPTY, ) { internal val factories = HashMap, Meta>() @@ -30,10 +30,6 @@ public class ContextBuilder internal constructor( meta.action() } - public fun name(string: String) { - this.name = Name.parse(string) - } - @OptIn(DFExperimental::class) private fun findPluginFactory(tag: PluginTag): PluginFactory<*> = parent.gatherInSequence>(PluginFactory.TYPE).values @@ -95,19 +91,21 @@ public class ContextBuilder internal constructor( } /** - * Check if current context contains all plugins required by the builder and return it it does or forks to a new context + * Check if current context contains all plugins required by the builder and return it does or forks to a new context * if it does not. */ -public fun Context.withEnv(block: ContextBuilder.() -> Unit): Context { +@DFExperimental +public fun Context.modify(block: ContextBuilder.() -> Unit): Context { fun Context.contains(factory: PluginFactory<*>, meta: Meta): Boolean { val loaded = plugins[factory.tag] ?: return false return loaded.meta == meta } - val builder = ContextBuilder(this, name + "env", properties).apply(block) + val builder = ContextBuilder(this, name + "mod", properties).apply(block) val requiresFork = builder.factories.any { (factory, meta) -> !contains(factory, meta) } || ((properties as Meta) == builder.meta) + return if (requiresFork) builder.build() else this } \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt index 9ace5800..05bce836 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/Global.kt @@ -3,11 +3,13 @@ package space.kscience.dataforge.context import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.Job import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.Name.Companion.parse import space.kscience.dataforge.names.asName import kotlin.coroutines.CoroutineContext import kotlin.native.concurrent.ThreadLocal -internal expect val globalLoggerFactory: PluginFactory +internal expect fun getGlobalLoggerFactory(): PluginFactory /** * A global root context. Closing [Global] terminates the framework. @@ -20,4 +22,4 @@ private object GlobalContext : Context("GLOBAL".asName(), null, emptySet(), Meta public val Global: Context get() = GlobalContext public fun Context(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context = - Global.buildContext(name, block) \ No newline at end of file + Global.buildContext(name?.let(Name::parse), block) \ No newline at end of file diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt index d3efed6f..84a2423d 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/LogManager.kt @@ -75,7 +75,7 @@ public class DefaultLogManager : AbstractPlugin(), LogManager { */ public val Context.logger: LogManager get() = plugins.find(inherit = true) { it is LogManager } as? LogManager - ?: globalLoggerFactory(context = Global).apply { attach(Global) } + ?: getGlobalLoggerFactory()(context = Global).apply { attach(Global) } /** * The named proxy logger for a context member diff --git a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt index 81d455c7..d8d7b602 100644 --- a/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt +++ b/dataforge-context/src/commonMain/kotlin/space/kscience/dataforge/context/PluginManager.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.context import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.names.plus import kotlin.reflect.KClass @@ -84,7 +85,7 @@ public inline fun Context.fetch(factory: PluginFactory, val existing = plugins[factory] return if (existing != null && existing.meta == meta) existing else { - buildContext { + buildContext(name = this@fetch.name + factory.tag.name) { plugin(factory, meta) }.plugins[factory]!! } diff --git a/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt b/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt index 8e39eb3f..c00921cd 100644 --- a/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt +++ b/dataforge-context/src/commonTest/kotlin/space/kscience/dataforge/context/ContextTest.kt @@ -20,8 +20,7 @@ class ContextTest { @Test fun testPluginManager() { - val context = Global.buildContext { - name("test") + val context = Context("test") { plugin(DummyPlugin()) } val members = context.gather("test") diff --git a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt b/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt index 406929ca..0aae274a 100644 --- a/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt +++ b/dataforge-context/src/jsMain/kotlin/space/kscience/dataforge/context/loggingJs.kt @@ -29,4 +29,4 @@ public class ConsoleLogManager : AbstractPlugin(), LogManager { } } -internal actual val globalLoggerFactory: PluginFactory = ConsoleLogManager +internal actual fun getGlobalLoggerFactory(): PluginFactory = ConsoleLogManager diff --git a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt index 05763e00..445e04c9 100644 --- a/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt +++ b/dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/context/loggingJvm.kt @@ -31,4 +31,4 @@ public class SlfLogManager : AbstractPlugin(), LogManager { } } -internal actual val globalLoggerFactory: PluginFactory = SlfLogManager +internal actual fun getGlobalLoggerFactory(): PluginFactory = SlfLogManager diff --git a/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt b/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt index 62512132..ff4155f0 100644 --- a/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt +++ b/dataforge-context/src/nativeMain/kotlin/space/kscience/dataforge/context/loggingNative.kt @@ -1,4 +1,4 @@ package space.kscience.dataforge.context -internal actual val globalLoggerFactory: PluginFactory = DefaultLogManager +internal actual fun getGlobalLoggerFactory(): PluginFactory = DefaultLogManager diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt index 305f50f8..d4ce7990 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt @@ -32,7 +32,7 @@ public interface DataSet { */ public suspend fun listTop(prefix: Name = Name.EMPTY): List = flow().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList() - // By default traverses the whole tree. Could be optimized in descendants + // By default, traverses the whole tree. Could be optimized in descendants public companion object { public val META_KEY: Name = "@meta".asName() @@ -43,7 +43,7 @@ public interface DataSet { public val EMPTY: DataSet = object : DataSet { override val dataType: KType = TYPE_OF_NOTHING - private val nothing: Nothing get() = error("this is nothing") + //private val nothing: Nothing get() = error("this is nothing") override fun flow(): Flow> = emptyFlow() diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt index 2cbbc1c1..5b09468d 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt @@ -3,6 +3,7 @@ package space.kscience.dataforge.data import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.map +import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name import space.kscience.dataforge.names.matches @@ -26,40 +27,55 @@ private fun Data<*>.castOrNull(type: KType): Data? = /** * Select all data matching given type and filters. Does not modify paths + * + * @param namePattern a name match patter according to [Name.matches] + * @param filter addition filtering condition based on item name and meta. By default, accepts all */ @OptIn(DFExperimental::class) -@PublishedApi -internal fun DataSet<*>.select( +public fun DataSet<*>.select( type: KType, namePattern: Name? = null, + filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true } ): ActiveDataSet = object : ActiveDataSet { override val dataType = type - override fun flow(): Flow> = this@select.flow().filter { datum -> - datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern)) + private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type) + && (namePattern == null || name.matches(namePattern)) + && filter(name, datum.meta) + + override fun flow(): Flow> = this@select.flow().filter { + checkDatum(it.name, it.data) }.map { @Suppress("UNCHECKED_CAST") it as NamedData } - override suspend fun getData(name: Name): Data? = this@select.getData(name)?.castOrNull(type) + override suspend fun getData(name: Name): Data? = this@select.getData(name)?.let { datum -> + if (checkDatum(name, datum)) datum.castOrNull(type) else null + } override val updates: Flow = this@select.updates.filter { - val datum = this@select.getData(it) - datum?.type?.isSubtypeOf(type) ?: false + val datum = this@select.getData(it) ?: return@filter false + checkDatum(it, datum) } } /** * Select a single datum of the appropriate type */ -public inline fun DataSet<*>.select(namePattern: Name? = null): DataSet = - select(typeOf(), namePattern) +public inline fun DataSet<*>.select( + namePattern: Name? = null, + noinline filter: (name: Name, meta: Meta) -> Boolean = { _, _ -> true } +): DataSet = select(typeOf(), namePattern, filter) +/** + * Select a single datum if it is present and of given [type] + */ public suspend fun DataSet<*>.selectOne(type: KType, name: Name): NamedData? = getData(name)?.castOrNull(type)?.named(name) -public suspend inline fun DataSet<*>.selectOne(name: Name): NamedData? = selectOne(typeOf(), name) +public suspend inline fun DataSet<*>.selectOne(name: Name): NamedData? = + selectOne(typeOf(), name) public suspend inline fun DataSet<*>.selectOne(name: String): NamedData? = selectOne(typeOf(), Name.parse(name)) \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index d7ed301f..66281f19 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -10,8 +10,7 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.names.Name - -import kotlin.native.concurrent.ThreadLocal +import space.kscience.dataforge.names.asName import kotlin.reflect.KClass public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { @@ -75,10 +74,11 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } } -@ThreadLocal -internal val ioContext = Global.withEnv { - name("IO") - plugin(IOPlugin) -} - -public val Context.io: IOPlugin get() = (if (this == Global) ioContext else this).fetch(IOPlugin) \ No newline at end of file +public val Context.io: IOPlugin + get() = if (this == Global) { + Global.buildContext("IO".asName()) { + plugin(IOPlugin) + }.fetch(IOPlugin) + } else { + fetch(IOPlugin) + } \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt index 8347118c..96c62db2 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaggedEnvelopeFormat.kt @@ -161,7 +161,7 @@ public class TaggedEnvelopeFormat( } } - private val default by lazy { invoke(context = ioContext) } + private val default by lazy { invoke() } override fun readPartial(input: Input): PartialEnvelope = default.run { readPartial(input) } diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt index e1a55607..bc8031d7 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/TaglessEnvelopeFormat.kt @@ -193,11 +193,9 @@ public class TaglessEnvelopeFormat( override val name: Name = TAGLESS_ENVELOPE_TYPE.asName() - override fun invoke(meta: Meta, context: Context): EnvelopeFormat { - return TaglessEnvelopeFormat(context.io, meta) - } + override fun invoke(meta: Meta, context: Context): EnvelopeFormat = TaglessEnvelopeFormat(context.io, meta) - private val default by lazy { invoke(context = ioContext) } + private val default by lazy { invoke() } override fun readPartial(input: Input): PartialEnvelope = default.run { readPartial(input) } diff --git a/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt b/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt index 36ad16fb..345faf14 100644 --- a/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt +++ b/dataforge-scripting/src/jvmTest/kotlin/space/kscience/dataforge/scripting/BuildersKtTest.kt @@ -15,7 +15,7 @@ class BuildersKtTest { Workspace(Global){ println("I am working") - context { name("test") } + //context { name("test") } target("testTarget") { "a" put 12 @@ -28,8 +28,6 @@ class BuildersKtTest { val script = """ println("I am working") - context{ name("test") } - target("testTarget"){ "a" put 12 } diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 72a35da7..2d5715de 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -14,6 +14,7 @@ import space.kscience.dataforge.meta.descriptors.MetaDescriptorBuilder import space.kscience.dataforge.misc.DFBuilder import space.kscience.dataforge.misc.DFExperimental import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.asName import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty @@ -62,7 +63,7 @@ public class WorkspaceBuilder(private val parentContext: Context = Global) : Tas * Define a context for the workspace */ public fun context(block: ContextBuilder.() -> Unit = {}) { - this.context = parentContext.buildContext("workspace", block) + this.context = parentContext.buildContext("workspace".asName(), block) } /** diff --git a/gradle.properties b/gradle.properties index 2bae1b5e..5c9b6463 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,6 +4,6 @@ org.gradle.parallel=true kotlin.code.style=official kotlin.parallel.tasks.in.project=true -kotlin.mpp.enableGranularSourceSetsMetadata=true -kotlin.native.enableDependencyPropagation=false +#kotlin.mpp.enableGranularSourceSetsMetadata=true +#kotlin.native.enableDependencyPropagation=false kotlin.mpp.stability.nowarn=true From d178c4ff0d8936c59ad040bc6600f6c3d6c308ab Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 2 Nov 2021 10:50:38 +0300 Subject: [PATCH 08/12] Fix number comparison bug in value --- CHANGELOG.md | 1 + build.gradle.kts | 2 +- dataforge-context/build.gradle.kts | 4 +-- .../space/kscience/dataforge/data/Goal.kt | 2 ++ .../space/kscience/dataforge/io/IOPlugin.kt | 9 +++--- .../space/kscience/dataforge/values/Value.kt | 17 +++++----- .../kscience/dataforge/values/exoticValues.kt | 4 +-- .../kscience/dataforge/meta/SchemeTest.kt | 15 +++++++++ .../dataforge/meta/SpecificationTest.kt | 2 ++ .../dataforge/values/ValueEqualityTest.kt | 32 +++++++++++++++++++ .../dataforge/workspace/WorkspaceBuilder.kt | 12 ++++--- settings.gradle.kts | 2 +- 12 files changed, 79 insertions(+), 23 deletions(-) create mode 100644 dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/ValueEqualityTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 14f6ed73..c2e63dc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Fixed - MutableMetaImpl attachment and checks - Listeners in observable meta are replaced by lists +- JS number comparison bug. ### Security ## [0.5.0] diff --git a/build.gradle.kts b/build.gradle.kts index 42505cb9..97c9b3f8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.5.2-dev-1" + version = "0.5.2-dev-3" repositories{ mavenCentral() } diff --git a/dataforge-context/build.gradle.kts b/dataforge-context/build.gradle.kts index d090e9de..4d476c50 100644 --- a/dataforge-context/build.gradle.kts +++ b/dataforge-context/build.gradle.kts @@ -11,7 +11,7 @@ kscience { kotlin { sourceSets { - val commonMain by getting{ + val commonMain by getting { dependencies { api(project(":dataforge-meta")) } @@ -30,6 +30,6 @@ kotlin { } } -readme{ +readme { maturity = ru.mipt.npm.gradle.Maturity.DEVELOPMENT } \ No newline at end of file diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt index b7ddea96..678711c1 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Goal.kt @@ -1,6 +1,7 @@ package space.kscience.dataforge.data import kotlinx.coroutines.* +import space.kscience.dataforge.misc.DFExperimental import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext @@ -66,6 +67,7 @@ public open class LazyGoal( * If [GoalExecutionRestriction] is present in the [coroutineScope] context, the call could produce a error a warning * depending on the settings. */ + @OptIn(DFExperimental::class) override fun async(coroutineScope: CoroutineScope): Deferred { val log = coroutineScope.coroutineContext[GoalLogger] // Check if context restricts goal computation diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index 66281f19..daf5ba41 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -10,7 +10,6 @@ import space.kscience.dataforge.meta.Meta import space.kscience.dataforge.meta.get import space.kscience.dataforge.meta.string import space.kscience.dataforge.names.Name -import space.kscience.dataforge.names.asName import kotlin.reflect.KClass public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { @@ -74,11 +73,13 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } } +internal val ioContext = Context("IO") { + plugin(IOPlugin) +} + public val Context.io: IOPlugin get() = if (this == Global) { - Global.buildContext("IO".asName()) { - plugin(IOPlugin) - }.fetch(IOPlugin) + ioContext.fetch(IOPlugin) } else { fetch(IOPlugin) } \ No newline at end of file diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt index 707c86f6..93ea1994 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/Value.kt @@ -140,14 +140,17 @@ public class NumberValue(public val number: Number) : Value { val otherNumber = other.numberOrNull ?: return false + if(number == otherNumber) return true + + //Do not change the order of comparison. On JS number is the instance of all types return when (numberOrNull) { - is Short -> number.toShort() == otherNumber.toShort() + is Double -> number.toDouble() == otherNumber.toDouble() + is Float -> number.toFloat() == otherNumber.toFloat() is Long -> number.toLong() == otherNumber.toLong() - is Byte -> number.toByte() == otherNumber.toByte() + is Short -> number.toShort() == otherNumber.toShort() is Int -> number.toInt() == otherNumber.toInt() - is Float -> number.toFloat() == otherNumber.toFloat() - is Double -> number.toDouble() == otherNumber.toDouble() - else -> number.toString() == otherNumber.toString() + is Byte -> number.toByte() == otherNumber.toByte() + else -> false } } @@ -188,9 +191,7 @@ public class ListValue(override val list: List) : Value, Iterable return list == other.list } - override fun hashCode(): Int { - return list.hashCode() - } + override fun hashCode(): Int = list.hashCode() public companion object { public val EMPTY: ListValue = ListValue(emptyList()) diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/exoticValues.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/exoticValues.kt index d3fc66b5..218ef5ad 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/exoticValues.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/values/exoticValues.kt @@ -36,9 +36,7 @@ public class DoubleArrayValue(override val value: DoubleArray) : Value, Iterable } } - override fun hashCode(): Int { - return value.contentHashCode() - } + override fun hashCode(): Int = value.contentHashCode() override fun toString(): String = list.joinToString(prefix = "[", postfix = "]") diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt index 87948ec9..ddf81780 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SchemeTest.kt @@ -1,8 +1,11 @@ package space.kscience.dataforge.meta import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.values.ListValue +import space.kscience.dataforge.values.Value import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertNotNull @DFExperimental class SchemeTest { @@ -33,4 +36,16 @@ class SchemeTest { scheme.a = 2 assertEquals(2, flag) } + + @Test + fun testListSubscription(){ + val scheme = TestScheme.empty() + var value: Value? = null + scheme.v = ListValue(0.0,0.0,0.0) + scheme.useProperty(TestScheme::v){ + value = it + } + scheme.v = ListValue(1.0, 2.0, 3.0) + assertNotNull(value) + } } \ No newline at end of file diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt index 3a61363c..8d4d3537 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/SpecificationTest.kt @@ -9,6 +9,8 @@ internal class TestScheme : Scheme() { var a by int() var b by string() + var v by value() + companion object : SchemeSpec(::TestScheme) } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/ValueEqualityTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/ValueEqualityTest.kt new file mode 100644 index 00000000..cfc61e01 --- /dev/null +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/values/ValueEqualityTest.kt @@ -0,0 +1,32 @@ +package space.kscience.dataforge.values + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals + +class ValueEqualityTest { + @Test + fun numberValueNotEquals(){ + val a = 0.33.asValue() + val b = 0.34.asValue() + + println(a.number == b.number) + + assertNotEquals(a,b) + } + + @Test + fun arrayEqualsList() { + val v1 = doubleArrayOf(1.0, 2.0, 3.0).asValue() + val v2 = listOf(1, 2, 3).map { it.asValue() }.asValue() + assertEquals(v1, v2) + } + + @Test + fun notEquals() { + val v1 = doubleArrayOf(1.0, 2.0, 3.0).asValue() + val v2 = listOf(1, 2, 6).map { it.asValue() }.asValue() + assertNotEquals(v1, v2) + } + +} \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt index 2d5715de..ec078020 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/WorkspaceBuilder.kt @@ -18,7 +18,7 @@ import space.kscience.dataforge.names.asName import kotlin.properties.PropertyDelegateProvider import kotlin.properties.ReadOnlyProperty -public data class TaskReference(public val taskName: Name, public val task: Task): DataSelector{ +public data class TaskReference(public val taskName: Name, public val task: Task) : DataSelector { @Suppress("UNCHECKED_CAST") override suspend fun select(workspace: Workspace, meta: Meta): DataSet { @@ -35,7 +35,6 @@ public interface TaskContainer { public fun registerTask(taskName: Name, task: Task<*>) } - public inline fun TaskContainer.registerTask( name: String, noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, @@ -43,15 +42,20 @@ public inline fun TaskContainer.registerTask( ): Unit = registerTask(Name.parse(name), Task(MetaDescriptor(descriptorBuilder), builder)) public inline fun TaskContainer.task( - noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, + descriptor: MetaDescriptor, noinline builder: suspend TaskResultBuilder.() -> Unit, ): PropertyDelegateProvider>> = PropertyDelegateProvider { _, property -> val taskName = Name.parse(property.name) - val task = Task(MetaDescriptor(descriptorBuilder), builder) + val task = Task(descriptor, builder) registerTask(taskName, task) ReadOnlyProperty { _, _ -> TaskReference(taskName, task) } } +public inline fun TaskContainer.task( + noinline descriptorBuilder: MetaDescriptorBuilder.() -> Unit = {}, + noinline builder: suspend TaskResultBuilder.() -> Unit, +): PropertyDelegateProvider>> = + task(MetaDescriptor(descriptorBuilder), builder) public class WorkspaceBuilder(private val parentContext: Context = Global) : TaskContainer { private var context: Context? = null diff --git a/settings.gradle.kts b/settings.gradle.kts index 26f6fcc4..fcf4a710 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,7 +5,7 @@ pluginManagement { gradlePluginPortal() } - val toolsVersion = "0.10.4" + val toolsVersion = "0.10.5" plugins { id("ru.mipt.npm.gradle.project") version toolsVersion From c423dc214ed034b177e575bebbf425cbf27512c4 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Wed, 24 Nov 2021 20:26:29 +0300 Subject: [PATCH 09/12] Some name refactoring --- build.gradle.kts | 2 +- .../space/kscience/dataforge/actions/MapAction.kt | 2 +- .../kscience/dataforge/actions/ReduceAction.kt | 2 +- .../kscience/dataforge/actions/SplitAction.kt | 2 +- .../space/kscience/dataforge/data/DataSet.kt | 14 ++++++-------- .../kscience/dataforge/data/DataSetBuilder.kt | 4 ++-- .../space/kscience/dataforge/data/DataTree.kt | 4 ++-- .../space/kscience/dataforge/data/GroupRule.kt | 2 +- .../kscience/dataforge/data/StaticDataTree.kt | 2 +- .../space/kscience/dataforge/data/dataFilter.kt | 8 ++++---- .../space/kscience/dataforge/data/dataTransform.kt | 8 ++++---- .../kotlin/space/kscience/dataforge/data/select.kt | 2 +- .../kotlin/space/kscience/dataforge/io/Binary.kt | 1 + .../kotlin/space/kscience/dataforge/io/IOPlugin.kt | 1 - .../space/kscience/dataforge/meta/MutableMeta.kt | 2 +- .../kotlin/space/kscience/dataforge/meta/Scheme.kt | 14 ++++++++++++-- .../kotlin/space/kscience/dataforge/names/Name.kt | 4 +++- .../kscience/dataforge/workspace/TaskResult.kt | 4 ++-- .../kscience/dataforge/workspace/workspaceJvm.kt | 4 ++-- .../dataforge/workspace/DataPropagationTest.kt | 6 +++--- .../dataforge/workspace/SimpleWorkspaceTest.kt | 10 +++++----- settings.gradle.kts | 2 +- 22 files changed, 55 insertions(+), 45 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 97c9b3f8..8c9f0b6e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.5.2-dev-3" + version = "0.5.2-dev-4" repositories{ mavenCentral() } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt index 0e6d6f0f..962d4b4d 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/MapAction.kt @@ -76,7 +76,7 @@ internal class MapAction( return newData.named(newName) } - val flow = dataSet.flow().map(::mapOne) + val flow = dataSet.flowData().map(::mapOne) return ActiveDataTree(outputType) { populate(flow) diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt index 7186504d..39f26dd6 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/ReduceAction.kt @@ -84,7 +84,7 @@ internal class ReduceAction( override fun CoroutineScope.transform(set: DataSet, meta: Meta, key: Name): Flow> = flow { ReduceGroupBuilder(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group -> - val dataFlow: Map> = group.set.flow().fold(HashMap()) { acc, value -> + val dataFlow: Map> = group.set.flowData().fold(HashMap()) { acc, value -> acc.apply { acc[value.name] = value.data } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt index e8ed7961..51a47c4d 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/SplitAction.kt @@ -72,7 +72,7 @@ internal class SplitAction( } return ActiveDataTree(outputType) { - populate(dataSet.flow().flatMapConcat(transform = ::splitOne)) + populate(dataSet.flowData().flatMapConcat(transform = ::splitOne)) scope?.launch { dataSet.updates.collect { name -> //clear old nodes diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt index d4ce7990..64f1f521 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSet.kt @@ -17,10 +17,8 @@ public interface DataSet { /** * Traverse this provider or its child. The order is not guaranteed. - * [root] points to a root name for traversal. If it is empty, traverse this source, if it points to a [Data], - * return flow, that contains single [Data], if it points to a node with children, return children. */ - public fun flow(): Flow> + public fun flowData(): Flow> /** * Get data with given name. @@ -31,7 +29,7 @@ public interface DataSet { * Get a snapshot of names of top level children of given node. Empty if node does not exist or is a leaf. */ public suspend fun listTop(prefix: Name = Name.EMPTY): List = - flow().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList() + flowData().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList() // By default, traverses the whole tree. Could be optimized in descendants public companion object { @@ -45,7 +43,7 @@ public interface DataSet { //private val nothing: Nothing get() = error("this is nothing") - override fun flow(): Flow> = emptyFlow() + override fun flowData(): Flow> = emptyFlow() override suspend fun getData(name: Name): Data? = null } @@ -67,7 +65,7 @@ public val DataSet.updates: Flow get() = if (this is ActiveDa /** * Flow all data nodes with names starting with [branchName] */ -public fun DataSet.flowChildren(branchName: Name): Flow> = this@flowChildren.flow().filter { +public fun DataSet.flowChildren(branchName: Name): Flow> = this@flowChildren.flowData().filter { it.name.startsWith(branchName) } @@ -75,7 +73,7 @@ public fun DataSet.flowChildren(branchName: Name): Flow DataSet.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch { - flow().map { + flowData().map { it.launch(this@launch) }.toList().joinAll() } @@ -83,7 +81,7 @@ public fun DataSet.startAll(coroutineScope: CoroutineScope): Job = public suspend fun DataSet.join(): Unit = coroutineScope { startAll(this).join() } public suspend fun DataSet<*>.toMeta(): Meta = Meta { - flow().collect { + flowData().collect { if (it.name.endsWith(DataSet.META_KEY)) { set(it.name, it.meta) } else { diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt index d966d8ae..ea8e3e38 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataSetBuilder.kt @@ -30,7 +30,7 @@ public interface DataSetBuilder { } //Set new items - dataSet.flow().collect { + dataSet.flowData().collect { emit(name + it.name, it.data) } } @@ -139,7 +139,7 @@ public suspend inline fun DataSetBuilder.static( */ @DFExperimental public suspend fun DataSetBuilder.populate(tree: DataSet): Unit = coroutineScope { - tree.flow().collect { + tree.flowData().collect { //TODO check if the place is occupied emit(it.name, it.data) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt index 87397f3e..ea9d67e2 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/DataTree.kt @@ -32,12 +32,12 @@ public interface DataTree : DataSet { */ public suspend fun items(): Map> - override fun flow(): Flow> = flow { + override fun flowData(): Flow> = flow { items().forEach { (token, childItem: DataTreeItem) -> if(!token.body.startsWith("@")) { when (childItem) { is DataTreeItem.Leaf -> emit(childItem.data.named(token.asName())) - is DataTreeItem.Node -> emitAll(childItem.tree.flow().map { it.named(token + it.name) }) + is DataTreeItem.Node -> emitAll(childItem.tree.flowData().map { it.named(token + it.name) }) } } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt index a9050b3c..d26cbfb1 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/GroupRule.kt @@ -44,7 +44,7 @@ public interface GroupRule { ): Map> { val map = HashMap>() - set.flow().collect { data -> + set.flowData().collect { data -> val tagValue = data.meta[key]?.string ?: defaultTagValue map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt index f57af184..e68c16e2 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/StaticDataTree.kt @@ -53,7 +53,7 @@ internal class StaticDataTree( set(name, DataTreeItem.Node(dataSet)) } else { coroutineScope { - dataSet.flow().collect { + dataSet.flowData().collect { emit(name + it.name, it.data) } } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt index 469594fe..a526cdd7 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataFilter.kt @@ -20,8 +20,8 @@ public fun DataSet.filter( ): ActiveDataSet = object : ActiveDataSet { override val dataType: KType get() = this@filter.dataType - override fun flow(): Flow> = - this@filter.flow().filter { predicate(it.name, it.data) } + override fun flowData(): Flow> = + this@filter.flowData().filter { predicate(it.name, it.data) } override suspend fun getData(name: Name): Data? = this@filter.getData(name)?.takeIf { predicate(name, it) @@ -40,7 +40,7 @@ public fun DataSet.withNamePrefix(prefix: Name): DataSet = if (p else object : ActiveDataSet { override val dataType: KType get() = this@withNamePrefix.dataType - override fun flow(): Flow> = this@withNamePrefix.flow().map { it.data.named(prefix + it.name) } + override fun flowData(): Flow> = this@withNamePrefix.flowData().map { it.data.named(prefix + it.name) } override suspend fun getData(name: Name): Data? = name.removeHeadOrNull(name)?.let { this@withNamePrefix.getData(it) } @@ -56,7 +56,7 @@ public fun DataSet.branch(branchName: Name): DataSet = if (branc } else object : ActiveDataSet { override val dataType: KType get() = this@branch.dataType - override fun flow(): Flow> = this@branch.flow().mapNotNull { + override fun flowData(): Flow> = this@branch.flowData().mapNotNull { it.name.removeHeadOrNull(branchName)?.let { name -> it.data.named(name) } diff --git a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt index f4c4a710..14b90729 100644 --- a/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt +++ b/dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt @@ -144,7 +144,7 @@ public suspend fun DataSet.map( block: suspend (T) -> R, ): DataTree = DataTree(outputType) { populate( - flow().map { + flowData().map { val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal() Data(outputType, newMeta, coroutineContext, listOf(it)) { block(it.await()) @@ -162,7 +162,7 @@ public suspend inline fun DataSet.map( public suspend fun DataSet.forEach(block: suspend (NamedData) -> Unit) { contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) } - flow().collect { + flowData().collect { block(it) } } @@ -171,11 +171,11 @@ public suspend inline fun DataSet.reduceToData( coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline transformation: suspend (Flow>) -> R, -): Data = flow().reduceToData(coroutineContext, meta, transformation) +): Data = flowData().reduceToData(coroutineContext, meta, transformation) public suspend inline fun DataSet.foldToData( initial: R, coroutineContext: CoroutineContext = EmptyCoroutineContext, meta: Meta = Meta.EMPTY, noinline block: suspend (result: R, data: NamedData) -> R, -): Data = flow().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file +): Data = flowData().foldToData(initial, coroutineContext, meta, block) \ No newline at end of file diff --git a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt index 5b09468d..5e65cee7 100644 --- a/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt +++ b/dataforge-data/src/jvmMain/kotlin/space/kscience/dataforge/data/select.kt @@ -43,7 +43,7 @@ public fun DataSet<*>.select( && (namePattern == null || name.matches(namePattern)) && filter(name, datum.meta) - override fun flow(): Flow> = this@select.flow().filter { + override fun flowData(): Flow> = this@select.flowData().filter { checkDatum(it.name, it.data) }.map { @Suppress("UNCHECKED_CAST") diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt index 0a375ab1..54353a15 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/Binary.kt @@ -54,6 +54,7 @@ public fun Binary.toByteArray(): ByteArray = if (this is ByteArrayBinary) { } } +//TODO optimize for file-based Inputs public fun Input.readBinary(size: Int): Binary { val array = readBytes(size) return ByteArrayBinary(array) diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index daf5ba41..fafa3ce3 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -29,7 +29,6 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { } } - public val metaFormatFactories: Collection by lazy { context.gather(META_FORMAT_TYPE).values } diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index 0c2c653e..e1661e53 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -155,7 +155,7 @@ public fun MutableMeta.getOrCreate(key: String): MutableMeta = getOrCreate(Name. public interface MutableTypedMeta> : TypedMeta, MutableMeta { /** - * Zero-copy attach or replace existing node. Node is used with any additional state, listeners, etc. + * Zero-copy (if possible) attach or replace existing node. Node is used with any additional state, listeners, etc. * In some cases it is possible to have the same node as a child to several others */ @DFExperimental diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt index ffc29152..fa829d29 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/Scheme.kt @@ -32,7 +32,7 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl internal fun wrap( newMeta: MutableMeta, - preserveDefault: Boolean = false + preserveDefault: Boolean = false, ) { if (preserveDefault) { defaultMeta = targetMeta.seal() @@ -120,7 +120,11 @@ public open class Scheme : Described, MetaRepr, MutableMetaProvider, Configurabl @DFExperimental override fun attach(name: Name, node: ObservableMutableMeta) { - TODO("Not yet implemented") + //TODO implement zero-copy attachment + setMeta(name, meta) + node.onChange(this) { changeName -> + setMeta(name + changeName, this[changeName]) + } } } @@ -139,6 +143,12 @@ public fun T.retarget(provider: MutableMeta): T = apply { */ public inline operator fun T.invoke(block: T.() -> Unit): T = apply(block) +/** + * Create a copy of given [Scheme] + */ +public inline fun T.copy(spec: SchemeSpec, block: T.() -> Unit = {}): T = + spec.read(meta.copy()).apply(block) + /** * A specification for simplified generation of wrappers */ diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt index 7bf8d14e..debe5e17 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/names/Name.kt @@ -201,4 +201,6 @@ public fun Name.removeHeadOrNull(head: Name): Name? = if (startsWith(head)) { Name(tokens.subList(head.length, length)) } else { null -} \ No newline at end of file +} + +public fun String.parseAsName(): Name = Name.parse(this) \ No newline at end of file diff --git a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt index 5a65f219..6a12e3bc 100644 --- a/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt +++ b/dataforge-workspace/src/commonMain/kotlin/space/kscience/dataforge/workspace/TaskResult.kt @@ -25,7 +25,7 @@ public interface TaskResult : DataSet { */ public val taskMeta: Meta - override fun flow(): Flow> + override fun flowData(): Flow> override suspend fun getData(name: Name): TaskData? } @@ -36,7 +36,7 @@ private class TaskResultImpl( override val taskMeta: Meta, ) : TaskResult, DataSet by dataSet { - override fun flow(): Flow> = dataSet.flow().map { + override fun flowData(): Flow> = dataSet.flowData().map { workspace.wrapData(it, it.name, taskName, taskMeta) } diff --git a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt index 61e1dbec..ffa397d3 100644 --- a/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt +++ b/dataforge-workspace/src/jvmMain/kotlin/space/kscience/dataforge/workspace/workspaceJvm.kt @@ -11,11 +11,11 @@ public fun WorkspaceBuilder.data(builder: suspend DataSetBuilder.() -> Unit buildData(builder) } -public inline fun TaskResultBuilder<*>.selectData(namePattern: Name? = null): DataSelector = object : DataSelector { +public inline fun TaskResultBuilder<*>.data(namePattern: Name? = null): DataSelector = object : DataSelector { override suspend fun select(workspace: Workspace, meta: Meta): DataSet = workspace.data.select(namePattern) } -public suspend inline fun TaskResultBuilder<*>.from( +public suspend inline fun TaskResultBuilder<*>.fromTask( task: Name, taskMeta: Meta = Meta.EMPTY, ): DataSet = workspace.produce(task, taskMeta).select() \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt index effc43e0..f88c6f59 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/DataPropagationTest.kt @@ -16,7 +16,7 @@ class DataPropagationTestPlugin : WorkspacePlugin() { val allData by task { val selectedData = workspace.data.select() - val result: Data = selectedData.flow().foldToData(0) { result, data -> + val result: Data = selectedData.flowData().foldToData(0) { result, data -> result + data.await() } emit("result", result) @@ -58,7 +58,7 @@ class DataPropagationTest { fun testAllData() { runBlocking { val node = testWorkspace.produce("Test.allData") - assertEquals(4950, node.flow().single().await()) + assertEquals(4950, node.flowData().single().await()) } } @@ -66,7 +66,7 @@ class DataPropagationTest { fun testSingleData() { runBlocking { val node = testWorkspace.produce("Test.singleData") - assertEquals(12, node.flow().single().await()) + assertEquals(12, node.flowData().single().await()) } } } \ No newline at end of file diff --git a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt index edbe3a4b..a1ce6ae1 100644 --- a/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt +++ b/dataforge-workspace/src/jvmTest/kotlin/space/kscience/dataforge/workspace/SimpleWorkspaceTest.kt @@ -71,7 +71,7 @@ class SimpleWorkspaceTest { } val square by task { - pipeFrom(selectData()) { arg, name, meta -> + pipeFrom(data()) { arg, name, meta -> if (meta["testFlag"].boolean == true) { println("flag") } @@ -89,7 +89,7 @@ class SimpleWorkspaceTest { } val linear by task { - pipeFrom(selectData()) { arg, name, _ -> + pipeFrom(data()) { arg, name, _ -> workspace.logger.info { "Starting linear on $name" } arg * 2 + 1 } @@ -162,7 +162,7 @@ class SimpleWorkspaceTest { fun testWorkspace() { runBlocking { val node = workspace.runBlocking("sum") - val res = node.flow().single() + val res = node.flowData().single() assertEquals(328350, res.await()) } } @@ -172,7 +172,7 @@ class SimpleWorkspaceTest { fun testMetaPropagation() { runBlocking { val node = workspace.produce("sum") { "testFlag" put true } - val res = node.flow().single().await() + val res = node.flowData().single().await() } } @@ -195,7 +195,7 @@ class SimpleWorkspaceTest { fun testFilter() { runBlocking { val node = workspace.produce("filterOne") - assertEquals(12, node.flow().first().await()) + assertEquals(12, node.flowData().first().await()) } } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index fcf4a710..2e5ee74f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,7 +5,7 @@ pluginManagement { gradlePluginPortal() } - val toolsVersion = "0.10.5" + val toolsVersion = "0.10.7" plugins { id("ru.mipt.npm.gradle.project") version toolsVersion From 532e0c253bd7ad04fbc536ac9b9a6073f72d1343 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 27 Nov 2021 13:36:07 +0300 Subject: [PATCH 10/12] Add Yaml meta format plugin --- .../io/yaml/FrontMatterEnvelopeFormat.kt | 32 +++++++++---------- .../dataforge/io/yaml/YamlMetaFormat.kt | 2 +- .../kscience/dataforge/io/yaml/YamlPlugin.kt | 32 +++++++++++++++++++ .../space/kscience/dataforge/io/IOPlugin.kt | 10 +++--- 4 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt index 93b9300d..6c96b29d 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/FrontMatterEnvelopeFormat.kt @@ -23,11 +23,11 @@ public class FrontMatterEnvelopeFormat( do { line = input.readUTF8Line() ?: error("Input does not contain front matter separator") offset += line.encodeToByteArray().size.toUInt() - } while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR)) + } while (!line.startsWith(SEPARATOR)) val readMetaFormat = - space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.resolveMetaFormat(it) } ?: space.kscience.dataforge.io.yaml.YamlMetaFormat + metaTypeRegex.matchEntire(line)?.groupValues?.first() + ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat //TODO replace by preview val meta = Binary { @@ -35,7 +35,7 @@ public class FrontMatterEnvelopeFormat( line = input.readSafeUtf8Line() writeUtf8String(line + "\r\n") offset += line.encodeToByteArray().size.toUInt() - } while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR)) + } while (!line.startsWith(SEPARATOR)) }.read { readMetaFormat.readMeta(input) @@ -47,16 +47,16 @@ public class FrontMatterEnvelopeFormat( var line: String do { line = input.readSafeUtf8Line() //?: error("Input does not contain front matter separator") - } while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR)) + } while (!line.startsWith(SEPARATOR)) val readMetaFormat = - space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.metaTypeRegex.matchEntire(line)?.groupValues?.first() - ?.let { io.resolveMetaFormat(it) } ?: space.kscience.dataforge.io.yaml.YamlMetaFormat + metaTypeRegex.matchEntire(line)?.groupValues?.first() + ?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat val meta = Binary { do { writeUtf8String(input.readSafeUtf8Line() + "\r\n") - } while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR)) + } while (!line.startsWith(SEPARATOR)) }.read { readMetaFormat.readMeta(input) } @@ -72,9 +72,9 @@ public class FrontMatterEnvelopeFormat( formatMeta: Meta, ) { val metaFormat = metaFormatFactory(formatMeta, this@FrontMatterEnvelopeFormat.io.context) - output.writeRawString("${space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n") + output.writeRawString("$SEPARATOR\r\n") metaFormat.run { this.writeObject(output, envelope.meta) } - output.writeRawString("${space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n") + output.writeRawString("$SEPARATOR\r\n") //Printing data envelope.data?.let { data -> output.writeBinary(data) @@ -92,32 +92,32 @@ public class FrontMatterEnvelopeFormat( private val metaTypeRegex = "---(\\w*)\\s*".toRegex() override fun invoke(meta: Meta, context: Context): EnvelopeFormat { - return space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat(context.io, meta) + return FrontMatterEnvelopeFormat(context.io, meta) } override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = binary.read { val line = readSafeUtf8Line() return@read if (line.startsWith("---")) { - space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.invoke() + invoke() } else { null } } - private val default by lazy { space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.invoke() } + private val default by lazy { invoke() } override fun readPartial(input: Input): PartialEnvelope = - space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.readPartial(input) + default.readPartial(input) override fun writeEnvelope( output: Output, envelope: Envelope, metaFormatFactory: MetaFormatFactory, formatMeta: Meta, - ): Unit = space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta) + ): Unit = FrontMatterEnvelopeFormat.default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta) - override fun readObject(input: Input): Envelope = space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.readObject(input) + override fun readObject(input: Input): Envelope = default.readObject(input) } } \ No newline at end of file diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt index 96c57c58..f34da12d 100644 --- a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlMetaFormat.kt @@ -110,7 +110,7 @@ public class YamlMetaFormat(private val meta: Meta) : MetaFormat { } override fun toMeta(): Meta = Meta { - NAME_KEY put space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.name.toString() + NAME_KEY put FrontMatterEnvelopeFormat.name.toString() META_KEY put meta } diff --git a/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt new file mode 100644 index 00000000..6d2215dc --- /dev/null +++ b/dataforge-io/dataforge-io-yaml/src/commonMain/kotlin/space/kscience/dataforge/io/yaml/YamlPlugin.kt @@ -0,0 +1,32 @@ +package space.kscience.dataforge.io.yaml + +import space.kscience.dataforge.context.AbstractPlugin +import space.kscience.dataforge.context.Context +import space.kscience.dataforge.context.PluginFactory +import space.kscience.dataforge.context.PluginTag +import space.kscience.dataforge.io.IOPlugin +import space.kscience.dataforge.io.MetaFormatFactory +import space.kscience.dataforge.meta.Meta +import space.kscience.dataforge.misc.DFExperimental +import space.kscience.dataforge.names.Name +import space.kscience.dataforge.names.asName +import kotlin.reflect.KClass + +@DFExperimental +public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) { + public val io: IOPlugin by require(IOPlugin) + + override val tag: PluginTag get() = Companion.tag + + override fun content(target: String): Map = when (target) { + MetaFormatFactory.META_FORMAT_TYPE -> mapOf("yaml".asName() to YamlMetaFormat) + else -> super.content(target) + } + + public companion object : PluginFactory { + override val tag: PluginTag = PluginTag("io.yaml", group = PluginTag.DATAFORGE_GROUP) + + override val type: KClass = YamlPlugin::class + override fun invoke(meta: Meta, context: Context): YamlPlugin = YamlPlugin(meta) + } +} \ No newline at end of file diff --git a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt index fafa3ce3..c72aeff6 100644 --- a/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt +++ b/dataforge-io/src/commonMain/kotlin/space/kscience/dataforge/io/IOPlugin.kt @@ -52,12 +52,10 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) { return resolveEnvelopeFormat(Name.parse(name), meta) } - override fun content(target: String): Map { - return when (target) { - META_FORMAT_TYPE -> defaultMetaFormats.toMap() - ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() - else -> super.content(target) - } + override fun content(target: String): Map = when (target) { + META_FORMAT_TYPE -> defaultMetaFormats.toMap() + ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap() + else -> super.content(target) } public companion object : PluginFactory { From 64e0c554ccd71545f840210a0867eccf4b66d6ad Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 30 Nov 2021 11:58:23 +0300 Subject: [PATCH 11/12] 0.5.2 release --- CHANGELOG.md | 46 ++++++++++++------- build.gradle.kts | 2 +- dataforge-context/api/dataforge-context.api | 7 +-- dataforge-meta/api/dataforge-meta.api | 3 ++ .../kscience/dataforge/meta/MutableMeta.kt | 4 +- .../dataforge/meta/MutableMetaTest.kt | 12 ++++- 6 files changed, 48 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2e63dc1..b078be98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ ## [Unreleased] ### Added +### Changed + +### Deprecated + +### Removed + +### Fixed + +### Security + +## [0.5.2] +### Added +- Yaml plugin +- Partial fix to #53 + + ### Changed ### Deprecated @@ -14,11 +30,14 @@ - Listeners in observable meta are replaced by lists - JS number comparison bug. + ### Security + ## [0.5.0] ### Added - Experimental `listOfSpec` delegate. + ### Changed - **API breaking** Config is deprecated, use `ObservableMeta` instead. - **API breaking** Descriptor no has a member property `defaultValue` instead of `defaultItem()` extension. It caches default value state on the first call. It is done because computing default on each call is too expensive. @@ -29,17 +48,18 @@ - **API breaking** `String.toName()` is replaced by `Name.parse()` - **API breaking** Configurable`config` changed to `meta` + ### Removed - `Config` - Public PluginManager mutability - Tables and tables-exposed moved to the separate project `tables.kt` - BinaryMetaFormat. Use CBOR encoding instead + ### Fixed - Proper json array index treatment. - Proper json index for single-value array. -### Security ## [0.4.0] ### Added - LogManager plugin @@ -47,6 +67,7 @@ - Context `withEnv` and `fetch` methods to manipulate plugins without changing plugins after creation. - Split `ItemDescriptor` into builder and read-only part + ### Changed - Kotlin-logging moved from common to JVM and JS. Replaced by console for native. - Package changed to `space.kscience` @@ -56,18 +77,19 @@ - Added blank builders for children context. - Refactor loggers + ### Deprecated - Direct use of PluginManager + ### Removed - Common dependency on Kotlin-logging - Kotlinx-io fork dependency. Replaced by Ktor-io. + ### Fixed - Scheme properties properly handle children property change. -### Security - ## [0.3.0] ### Added - Yaml meta format based on yaml.kt @@ -76,6 +98,7 @@ - `copy` method to descriptors - Multiplatform yaml meta + ### Changed - `ListValue` and `DoubleArrayValue` implement `Iterable`. - Changed the logic of `Value::isList` to check for type instead of size @@ -90,17 +113,7 @@ - \[Major breaking change\] Full refactor of DataTree/DataSource - \[Major Breaking change\] Replace KClass with KType in data. Remove direct access to constructors with types. -### Deprecated - -### Removed - -### Fixed - -### Security - ## [0.2.0] -### Added - ### Changed - Context content resolution refactor - Kotlin 1.4.10 (build tools 0.6.0) @@ -110,16 +123,17 @@ - Removed io depdendency from `dataforge-output`. Replaced Output by Appendable. - Configurable is no longer MutableItemProvider. All functionality moved to Scheme. + ### Deprecated - Context activation API - TextRenderer + ### Removed - Functional server prototype - `dataforge-output` module + ### Fixed - Global context CoroutineScope resolution -- Library mode compliance - -### Security +- Library mode compliance \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 8c9f0b6e..61588f0a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ plugins { allprojects { group = "space.kscience" - version = "0.5.2-dev-4" + version = "0.5.2" repositories{ mavenCentral() } diff --git a/dataforge-context/api/dataforge-context.api b/dataforge-context/api/dataforge-context.api index 65bcc846..c38ca6e5 100644 --- a/dataforge-context/api/dataforge-context.api +++ b/dataforge-context/api/dataforge-context.api @@ -33,8 +33,8 @@ public final class space/kscience/dataforge/context/ClassLoaderPluginKt { public class space/kscience/dataforge/context/Context : kotlinx/coroutines/CoroutineScope, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/misc/Named, space/kscience/dataforge/provider/Provider { public static final field Companion Lspace/kscience/dataforge/context/Context$Companion; public static final field PROPERTY_TARGET Ljava/lang/String; - public final fun buildContext (Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context; - public static synthetic fun buildContext$default (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context; + public final fun buildContext (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context; + public static synthetic fun buildContext$default (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context; public fun close ()V public fun content (Ljava/lang/String;)Ljava/util/Map; public final fun content (Ljava/lang/String;Z)Ljava/util/Map; @@ -57,7 +57,6 @@ public abstract interface class space/kscience/dataforge/context/ContextAware { public final class space/kscience/dataforge/context/ContextBuilder { public final fun build ()Lspace/kscience/dataforge/context/Context; public final fun getName ()Lspace/kscience/dataforge/names/Name; - public final fun name (Ljava/lang/String;)V public final fun plugin (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)V public final fun plugin (Lspace/kscience/dataforge/context/Plugin;)V public final fun plugin (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;)V @@ -67,11 +66,9 @@ public final class space/kscience/dataforge/context/ContextBuilder { public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V public final fun properties (Lkotlin/jvm/functions/Function1;)V - public final fun setName (Lspace/kscience/dataforge/names/Name;)V } public final class space/kscience/dataforge/context/ContextBuilderKt { - public static final fun withEnv (Lspace/kscience/dataforge/context/Context;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context; } public final class space/kscience/dataforge/context/DefaultLogManager : space/kscience/dataforge/context/AbstractPlugin, space/kscience/dataforge/context/LogManager { diff --git a/dataforge-meta/api/dataforge-meta.api b/dataforge-meta/api/dataforge-meta.api index d1792538..03075958 100644 --- a/dataforge-meta/api/dataforge-meta.api +++ b/dataforge-meta/api/dataforge-meta.api @@ -365,6 +365,8 @@ public class space/kscience/dataforge/meta/Scheme : space/kscience/dataforge/met } public final class space/kscience/dataforge/meta/SchemeKt { + public static final fun copy (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; + public static synthetic fun copy$default (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/SchemeSpec;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/meta/Scheme; public static final fun invoke (Lspace/kscience/dataforge/meta/Scheme;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/meta/Scheme; public static final fun retarget (Lspace/kscience/dataforge/meta/Scheme;Lspace/kscience/dataforge/meta/MutableMeta;)Lspace/kscience/dataforge/meta/Scheme; } @@ -726,6 +728,7 @@ public final class space/kscience/dataforge/names/NameKt { public static final fun getLength (Lspace/kscience/dataforge/names/Name;)I public static final fun isEmpty (Lspace/kscience/dataforge/names/Name;)Z public static final fun lastOrNull (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/names/NameToken; + public static final fun parseAsName (Ljava/lang/String;)Lspace/kscience/dataforge/names/Name; public static final fun plus (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;)Lspace/kscience/dataforge/names/Name; public static final fun plus (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/names/Name; public static final fun plus (Lspace/kscience/dataforge/names/Name;Lspace/kscience/dataforge/names/NameToken;)Lspace/kscience/dataforge/names/Name; diff --git a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt index e1661e53..d39a6dbf 100644 --- a/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt +++ b/dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt @@ -296,9 +296,7 @@ private class MutableMetaImpl( override fun attach(name: Name, node: ObservableMutableMeta) { when (name.length) { 0 -> error("Can't set a meta with empty name") - 1 -> { - replaceItem(name.first(), get(name), node) - } + 1 -> replaceItem(name.first(), get(name), node) else -> get(name.cutLast())?.attach(name.lastOrNull()!!.asName(), node) } } diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt index 534d907e..0d47db99 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt @@ -2,10 +2,11 @@ package space.kscience.dataforge.meta import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFails class MutableMetaTest{ @Test - fun testRemove(){ + fun remove(){ val meta = MutableMeta { "aNode" put { "innerNode" put { @@ -19,4 +20,13 @@ class MutableMetaTest{ meta.remove("aNode.c") assertEquals(meta["aNode.c"], null) } + + @Test + fun recursiveMeta(){ + val meta = MutableMeta { + "a" put 2 + } + + assertFails { meta["child.a"] = meta} + } } \ No newline at end of file From c480cd8e4de1afddfb467143a33e6ddd07e91641 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Tue, 30 Nov 2021 12:53:52 +0300 Subject: [PATCH 12/12] Fix native test --- .../kscience/dataforge/meta/MutableMetaTest.kt | 10 ---------- .../kscience/dataforge/meta/JvmMutableMetaTest.kt | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 dataforge-meta/src/jvmTest/kotlin/space/kscience/dataforge/meta/JvmMutableMetaTest.kt diff --git a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt index 0d47db99..da4474d8 100644 --- a/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt +++ b/dataforge-meta/src/commonTest/kotlin/space/kscience/dataforge/meta/MutableMetaTest.kt @@ -2,7 +2,6 @@ package space.kscience.dataforge.meta import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertFails class MutableMetaTest{ @Test @@ -20,13 +19,4 @@ class MutableMetaTest{ meta.remove("aNode.c") assertEquals(meta["aNode.c"], null) } - - @Test - fun recursiveMeta(){ - val meta = MutableMeta { - "a" put 2 - } - - assertFails { meta["child.a"] = meta} - } } \ No newline at end of file diff --git a/dataforge-meta/src/jvmTest/kotlin/space/kscience/dataforge/meta/JvmMutableMetaTest.kt b/dataforge-meta/src/jvmTest/kotlin/space/kscience/dataforge/meta/JvmMutableMetaTest.kt new file mode 100644 index 00000000..075a9e45 --- /dev/null +++ b/dataforge-meta/src/jvmTest/kotlin/space/kscience/dataforge/meta/JvmMutableMetaTest.kt @@ -0,0 +1,15 @@ +package space.kscience.dataforge.meta + +import org.junit.jupiter.api.Test +import kotlin.test.assertFails + +class JvmMutableMetaTest { + @Test + fun recursiveMeta(){ + val meta = MutableMeta { + "a" put 2 + } + + assertFails { meta["child.a"] = meta } + } +} \ No newline at end of file