Skip to content

Commit

Permalink
Fixes #25886: Migrate Environment variable from lift-json to zio-json
Browse files Browse the repository at this point in the history
  • Loading branch information
fanf committed Nov 18, 2024
1 parent 8591051 commit 2624c12
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ final case class Controller(
) extends PhysicalElement

final case class MemorySlot(
slotNumber: String, // string, because sometime it looks like: RAM slot #0
slotNumber: String, // string, because sometimes it looks like: RAM slot #0

name: Option[String] = None,
description: Option[String] = None,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ final case class Process(
description: Option[String] = None
) extends NodeElement

object Process {
implicit val codecProcess: JsonCodec[Process] = DeriveJsonCodec.gen
}

final case class VirtualMachine(
@jsonField("type") @jsonAliases("vmtype") vmtype: Option[String] = None,
subsystem: Option[String] = None,
Expand All @@ -114,10 +118,16 @@ final case class VirtualMachine(
) extends NodeElement

final case class EnvironmentVariable(
name: String,
value: Option[String] = None,
description: Option[String] = None
) extends NodeElement
name: String,
value: Option[String] = None
) extends NodeElement {
// description is never present in Environment Variable, and we don't store it if it was
val description: Option[String] = None
}

object EnvironmentVariable {
implicit val codecEnvironmentVariable: JsonCodec[EnvironmentVariable] = DeriveJsonCodec.gen
}

object InetAddressUtils {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ import com.normation.ldap.sdk.schema.LDAPObjectClass
import com.softwaremill.quicklens.*
import com.unboundid.ldap.sdk.{Version as _, *}
import java.net.InetAddress
import net.liftweb.json.*
import org.joda.time.DateTime
import zio.*
import zio.json.*
Expand Down Expand Up @@ -116,8 +115,6 @@ class InventoryMapper(
removedDit: InventoryDit
) {

implicit val formats: Formats = DefaultFormats

////////////////////////////////////////////////////////////
///////////////////////// Software /////////////////////////
////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -811,7 +808,7 @@ class InventoryMapper(
root.setOpt(server.receiveDate, A_RECEIVE_DATE, (x: DateTime) => GeneralizedTime(x).toString)
root.resetValuesTo(A_AGENTS_NAME, server.agents.map(x => x.toJson)*)
root.resetValuesTo(A_SOFTWARE_DN, server.softwareIds.map(x => dit.SOFTWARE.SOFT.dn(x).toString)*)
root.resetValuesTo(A_EV, server.environmentVariables.map(x => Serialization.write(x))*)
root.resetValuesTo(A_EV, server.environmentVariables.map(_.toJson)*)
root.resetValuesTo(A_LIST_OF_IP, server.serverIps.distinct*)
// we don't know their dit...
root.resetValuesTo(
Expand All @@ -828,7 +825,6 @@ class InventoryMapper(
}
server.customProperties.foreach(cp => root.addValues(A_CUSTOM_PROPERTY, cp.toJson))
server.softwareUpdates.foreach { s =>
import zio.json.*
import JsonSerializers.implicits.*
root.addValues(A_SOFTWARE_UPDATE, s.toJson)
}
Expand All @@ -844,7 +840,7 @@ class InventoryMapper(
// map process from node
def processesFromNode(node: NodeInventory): Seq[String] = {
// convert the processes
node.processes.map(x => Serialization.write(x))
node.processes.map(_.toJson)
}

// Create the entry with only processes
Expand Down Expand Up @@ -1030,8 +1026,8 @@ class InventoryMapper(

lastLoggedUser = entry(A_LAST_LOGGED_USER)
lastLoggedUserTime = entry.getAsGTime(A_LAST_LOGGED_USER_TIME).map(_.dateTime)
ev = entry.valuesFor(A_EV).toSeq.map(Serialization.read[EnvironmentVariable](_))
process = entry.valuesFor(A_PROCESS).toSeq.map(Serialization.read[Process](_))
ev = entry.valuesFor(A_EV).toSeq.map(_.fromJson[EnvironmentVariable]).collect { case Right(ev) => ev }
process = entry.valuesFor(A_PROCESS).toSeq.map(_.fromJson[Process]).collect { case Right(p) => p }
softwareIds = entry.valuesFor(A_SOFTWARE_DN).toSeq.flatMap(x => dit.SOFTWARE.SOFT.idFromDN(new DN(x)).toOption)
machineId <- mapSeqStringToMachineIdAndStatus(entry.valuesFor(A_CONTAINER_DN)).toList match {
case Nil => None.succeed
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,13 +155,55 @@ class TestNodeUnserialisation extends Specification {
|osArchitectureType: x86_64
|timezoneOffset: +0200
|timezoneName: Europe/Paris
|agentName: {"agentType":"cfengine-community","version":"6.1.0","securityToken":{"value":"-----BEGIN CERTIFICATE-----\nMIIFTjCCAzagAwIBAgIUfa0+S+CyJahRzuwNNOFLNQQjIH8wDQYJKoZIhvcNAQEL\nBQAwFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3MB4XDTI0MDMwMTExMDIxM1oXDTM0\nMDIyNzExMDIxM1owFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3MIICIjANBgkqhkiG\n9w0BAQEFAAOCAg8AMIICCgKCAgEAsmBUYI1A5vqkOTCW24m/mQhBaRu4WaAUXDAr\nArdIAAMN0KyHhEXP1/X32Flw90E0VjtUH4P+DYLBozXYWhJrUxdWLyn3TRSbv3Kr\npXoCWhMhMp8OK2s/mG+rfiMpTXIwaMhPnBaJFNV3c+bkijGAMHtFjl3+leXJvNZ7\nw3bIg4cA3e77kz7EWMyqxOUvvLMyY6wpd03ahe/By+iLgtOkgUwl9hMqMU8tJeaz\nNIeUporsHk5rrk8bSf6Mxxdknm43Sk6oflnueNCIUFdd4rS2JLieMugsTh8n/oH+\nk29ZyirE3ikhftmZ3vY8GQ3IcIzaXwiAOGnCKcze79zVx5jmOTGSZitGZaU/cZc6\nLjzEp+ZDmE8caVIksiA+hIlaZeBXNHB+YRv/gV1Rbt7kS1am+XUZJSfVFf+99YqE\nlZ5p0hqJsczkBb2RuMxWxxsWO3pesPUNCuL/yaeggTIp7eK06tQWi1EfXr40ctrf\noimbzMAKNBpRKWruL7652SlF75Usaq+PaPi1TtqYQjLRZmbpr+IR4uMeKnEMIZPw\n3dLDKBV6d71XkTAalCmcJU+fgYrRmgz1dEnaZDIXY+f+fYR15hsFpDrZ0avHgkzJ\nca/nT/rKeX7136BttxVSbZaTU9hnmjAvl+v0BF+JUWPQ3VPTcrmyUNAICt8xdVae\nuo1tsvkCAwEAAaOBkTCBjjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBSRG0uHQRIA\ngl91YqzSjaeaw+F7PDBSBgNVHSMESzBJgBSRG0uHQRIAgl91YqzSjaeaw+F7PKEb\npBkwFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3ghR9rT5L4LIlqFHO7A004Us1BCMg\nfzALBgNVHQ8EBAMCArwwDQYJKoZIhvcNAQELBQADggIBABR2vTH/6WlqjaTZ/hQc\nB+crqRlFimqCiVTRdX6qfkt0tLX2dK6scHNTBT04ORLjLTAn0KbcOUz3k6i0mIqB\nnG5UjqCFEQR+i2V4Hz7aK6+7LBQuwXWRhDhvO5xMn7MxjvP0LGgNf5iiA4r/N22L\nEMc1prDESIOGmWdKg9XlfLxd877R2d8/3hXyT82Y2uJHO25b53skj4pbUOWLsGSw\nd4FBNnWqM+7Hbg2v9xFvmtfs2G2Inqk4Xjtjnj8qkVk3ft6KzClUIMXGXQ8QqaRp\nYPkTJm5g2UBYDiguD/tlz3VmbsNYU6fkap7DKqhttbaseIx1zDwdAv4jtVOWDyjx\nWq4b7pljYiczfRa84X/9v1x05pT3raELy5udY+Pxmnz+hOXOM+jY5bzSKS24b8rS\n5Sklmutm0IdflMKd5vNrXd9yPFLu3QzN50ArzHHXczwBLgjaMlZsPAp1wITlqM7+\nWCjy3qM5/KgAjH3L24MPTq23o9PokBVh1NH7lesZqgJPgsj+OG7FMQDvKzg7ytrs\nQlIDF1c8Ko+9/RrnRVAS8C4GZOqbmMmfJjMp09GBz2d0ixlTusF6m6iwfIVMf/nI\nP/V0D7STRiV62cfnZ3e0w8kIeZwWAgXI7RMHJU3skLyurUu8yxkp635IQyzQsW2A\nYo7t7O7fxjqD9yVI2QfkERZ7\n-----END CERTIFICATE-----\n","type":"certificate"},"capabilities":["https"]}
|agentName: {"agentType":"cfengine-community","version":"6.1.0","securityToken":{"type":"certificate","value":"-----BEGIN CERTIFICATE-----\nMIIFTjCCAzagAwIBAgIUfa0+S+CyJahRzuwNNOFLNQQjIH8wDQYJKoZIhvcNAQEL\nBQAwFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3MB4XDTI0MDMwMTExMDIxM1oXDTM0\nMDIyNzExMDIxM1owFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3MIICIjANBgkqhkiG\n9w0BAQEFAAOCAg8AMIICCgKCAgEAsmBUYI1A5vqkOTCW24m/mQhBaRu4WaAUXDAr\nArdIAAMN0KyHhEXP1/X32Flw90E0VjtUH4P+DYLBozXYWhJrUxdWLyn3TRSbv3Kr\npXoCWhMhMp8OK2s/mG+rfiMpTXIwaMhPnBaJFNV3c+bkijGAMHtFjl3+leXJvNZ7\nw3bIg4cA3e77kz7EWMyqxOUvvLMyY6wpd03ahe/By+iLgtOkgUwl9hMqMU8tJeaz\nNIeUporsHk5rrk8bSf6Mxxdknm43Sk6oflnueNCIUFdd4rS2JLieMugsTh8n/oH+\nk29ZyirE3ikhftmZ3vY8GQ3IcIzaXwiAOGnCKcze79zVx5jmOTGSZitGZaU/cZc6\nLjzEp+ZDmE8caVIksiA+hIlaZeBXNHB+YRv/gV1Rbt7kS1am+XUZJSfVFf+99YqE\nlZ5p0hqJsczkBb2RuMxWxxsWO3pesPUNCuL/yaeggTIp7eK06tQWi1EfXr40ctrf\noimbzMAKNBpRKWruL7652SlF75Usaq+PaPi1TtqYQjLRZmbpr+IR4uMeKnEMIZPw\n3dLDKBV6d71XkTAalCmcJU+fgYrRmgz1dEnaZDIXY+f+fYR15hsFpDrZ0avHgkzJ\nca/nT/rKeX7136BttxVSbZaTU9hnmjAvl+v0BF+JUWPQ3VPTcrmyUNAICt8xdVae\nuo1tsvkCAwEAAaOBkTCBjjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBSRG0uHQRIA\ngl91YqzSjaeaw+F7PDBSBgNVHSMESzBJgBSRG0uHQRIAgl91YqzSjaeaw+F7PKEb\npBkwFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3ghR9rT5L4LIlqFHO7A004Us1BCMg\nfzALBgNVHQ8EBAMCArwwDQYJKoZIhvcNAQELBQADggIBABR2vTH/6WlqjaTZ/hQc\nB+crqRlFimqCiVTRdX6qfkt0tLX2dK6scHNTBT04ORLjLTAn0KbcOUz3k6i0mIqB\nnG5UjqCFEQR+i2V4Hz7aK6+7LBQuwXWRhDhvO5xMn7MxjvP0LGgNf5iiA4r/N22L\nEMc1prDESIOGmWdKg9XlfLxd877R2d8/3hXyT82Y2uJHO25b53skj4pbUOWLsGSw\nd4FBNnWqM+7Hbg2v9xFvmtfs2G2Inqk4Xjtjnj8qkVk3ft6KzClUIMXGXQ8QqaRp\nYPkTJm5g2UBYDiguD/tlz3VmbsNYU6fkap7DKqhttbaseIx1zDwdAv4jtVOWDyjx\nWq4b7pljYiczfRa84X/9v1x05pT3raELy5udY+Pxmnz+hOXOM+jY5bzSKS24b8rS\n5Sklmutm0IdflMKd5vNrXd9yPFLu3QzN50ArzHHXczwBLgjaMlZsPAp1wITlqM7+\nWCjy3qM5/KgAjH3L24MPTq23o9PokBVh1NH7lesZqgJPgsj+OG7FMQDvKzg7ytrs\nQlIDF1c8Ko+9/RrnRVAS8C4GZOqbmMmfJjMp09GBz2d0ixlTusF6m6iwfIVMf/nI\nP/V0D7STRiV62cfnZ3e0w8kIeZwWAgXI7RMHJU3skLyurUu8yxkp635IQyzQsW2A\nYo7t7O7fxjqD9yVI2QfkERZ7\n-----END CERTIFICATE-----\n"},"capabilities":["https"]}
|inventoryDate: 20180717000031.000Z
|receiveDate: 20180717000527.050Z
|lastLoggedUserTime: 20000714084300.000Z
|softwareUpdate: {"name":"rudder-agent","version":"7.0.0-realease","arch":"x86_64","from":"yum","kind":"none","description":"Local privilege escalation in pkexec","severity":"low","date":"2022-01-26T00:00:00Z","ids":["RHSA-2020-4566","CVE-2021-4034"]}
|customProperty: {"name":"simpleString","value":"a simple string"}
|customProperty: {"name":"someJson","value":{"foo":"bar","i":42,"b":true,"arr":[1,2]}}
|environmentVariable: {"name":"UPSTART_INSTANCE"}
|environmentVariable: {"name":"SHELL","value":"/bin/bash"}
|process: {"pid":193,"commandName":"/var/rudder/cfengine-community/bin/cf-serverd","cpuUsage":0.0,"memory":0.0,"started":"2014-09-25 09:45","tty":"?","user":"root","virtualMemory":36664}
|""".stripMargin
}

/*
* Mains changes from 7.0~8.2
* - process: commandName => name
*/
val linux83Ldif: String = {
"""dn: nodeId=root,ou=Nodes,ou=Accepted Inventories,ou=Inventories,cn=rudder-configuration
|objectClass: top
|objectClass: node
|objectClass: unixNode
|objectClass: linuxNode
|nodeId: root
|localAdministratorAccountName: root
|policyServerId: root
|osFullName: SUSE Linux Enterprise Server 11 (x86_64)
|osServicePack: 3
|ram: 1572864000
|swap: 781189120
|lastLoggedUser: root
|osKernelVersion: 3.0.76-0.11-default
|osName: Suse
|osVersion: 11
|keyStatus: certified
|nodeHostname: orchestrateur-3.labo.normation.com
|osArchitectureType: x86_64
|timezoneOffset: +0200
|timezoneName: Europe/Paris
|agentName: {"agentType":"cfengine-community","version":"6.1.0","securityToken":{"type":"certificate","value":"-----BEGIN CERTIFICATE-----\nMIIFTjCCAzagAwIBAgIUfa0+S+CyJahRzuwNNOFLNQQjIH8wDQYJKoZIhvcNAQEL\nBQAwFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3MB4XDTI0MDMwMTExMDIxM1oXDTM0\nMDIyNzExMDIxM1owFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3MIICIjANBgkqhkiG\n9w0BAQEFAAOCAg8AMIICCgKCAgEAsmBUYI1A5vqkOTCW24m/mQhBaRu4WaAUXDAr\nArdIAAMN0KyHhEXP1/X32Flw90E0VjtUH4P+DYLBozXYWhJrUxdWLyn3TRSbv3Kr\npXoCWhMhMp8OK2s/mG+rfiMpTXIwaMhPnBaJFNV3c+bkijGAMHtFjl3+leXJvNZ7\nw3bIg4cA3e77kz7EWMyqxOUvvLMyY6wpd03ahe/By+iLgtOkgUwl9hMqMU8tJeaz\nNIeUporsHk5rrk8bSf6Mxxdknm43Sk6oflnueNCIUFdd4rS2JLieMugsTh8n/oH+\nk29ZyirE3ikhftmZ3vY8GQ3IcIzaXwiAOGnCKcze79zVx5jmOTGSZitGZaU/cZc6\nLjzEp+ZDmE8caVIksiA+hIlaZeBXNHB+YRv/gV1Rbt7kS1am+XUZJSfVFf+99YqE\nlZ5p0hqJsczkBb2RuMxWxxsWO3pesPUNCuL/yaeggTIp7eK06tQWi1EfXr40ctrf\noimbzMAKNBpRKWruL7652SlF75Usaq+PaPi1TtqYQjLRZmbpr+IR4uMeKnEMIZPw\n3dLDKBV6d71XkTAalCmcJU+fgYrRmgz1dEnaZDIXY+f+fYR15hsFpDrZ0avHgkzJ\nca/nT/rKeX7136BttxVSbZaTU9hnmjAvl+v0BF+JUWPQ3VPTcrmyUNAICt8xdVae\nuo1tsvkCAwEAAaOBkTCBjjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBSRG0uHQRIA\ngl91YqzSjaeaw+F7PDBSBgNVHSMESzBJgBSRG0uHQRIAgl91YqzSjaeaw+F7PKEb\npBkwFzEVMBMGCgmSJomT8ixkAQEMBW5vZGU3ghR9rT5L4LIlqFHO7A004Us1BCMg\nfzALBgNVHQ8EBAMCArwwDQYJKoZIhvcNAQELBQADggIBABR2vTH/6WlqjaTZ/hQc\nB+crqRlFimqCiVTRdX6qfkt0tLX2dK6scHNTBT04ORLjLTAn0KbcOUz3k6i0mIqB\nnG5UjqCFEQR+i2V4Hz7aK6+7LBQuwXWRhDhvO5xMn7MxjvP0LGgNf5iiA4r/N22L\nEMc1prDESIOGmWdKg9XlfLxd877R2d8/3hXyT82Y2uJHO25b53skj4pbUOWLsGSw\nd4FBNnWqM+7Hbg2v9xFvmtfs2G2Inqk4Xjtjnj8qkVk3ft6KzClUIMXGXQ8QqaRp\nYPkTJm5g2UBYDiguD/tlz3VmbsNYU6fkap7DKqhttbaseIx1zDwdAv4jtVOWDyjx\nWq4b7pljYiczfRa84X/9v1x05pT3raELy5udY+Pxmnz+hOXOM+jY5bzSKS24b8rS\n5Sklmutm0IdflMKd5vNrXd9yPFLu3QzN50ArzHHXczwBLgjaMlZsPAp1wITlqM7+\nWCjy3qM5/KgAjH3L24MPTq23o9PokBVh1NH7lesZqgJPgsj+OG7FMQDvKzg7ytrs\nQlIDF1c8Ko+9/RrnRVAS8C4GZOqbmMmfJjMp09GBz2d0ixlTusF6m6iwfIVMf/nI\nP/V0D7STRiV62cfnZ3e0w8kIeZwWAgXI7RMHJU3skLyurUu8yxkp635IQyzQsW2A\nYo7t7O7fxjqD9yVI2QfkERZ7\n-----END CERTIFICATE-----\n"},"capabilities":["https"]}
|inventoryDate: 20180717000031.000Z
|receiveDate: 20180717000527.050Z
|lastLoggedUserTime: 20000714084300.000Z
|softwareUpdate: {"name":"rudder-agent","version":"7.0.0-realease","from":"yum","arch":"x86_64","kind":"none","description":"Local privilege escalation in pkexec","severity":"low","date":"2022-01-26T00:00:00Z","ids":["RHSA-2020-4566","CVE-2021-4034"]}
|softwareUpdate: {"name":"rudder-agent","version":"7.0.0-realease","arch":"x86_64","from":"yum","kind":"none","description":"Local privilege escalation in pkexec","severity":"low","date":"2022-01-26T00:00:00Z","ids":["RHSA-2020-4566","CVE-2021-4034"]}
|customProperty: {"name":"simpleString","value":"a simple string"}
|customProperty: {"name":"someJson","value":{"foo":"bar","i":42,"b":true,"arr":[1,2]}}
|environmentVariable: {"name":"UPSTART_INSTANCE"}
|environmentVariable: {"name":"SHELL","value":"/bin/bash"}
|process: {"pid":193,"name":"/var/rudder/cfengine-community/bin/cf-serverd","cpuUsage":0.0,"memory":0.0,"started":"2014-09-25 09:45","tty":"?","user":"root","virtualMemory":36664.0}
|""".stripMargin
}

Expand Down Expand Up @@ -253,5 +295,51 @@ class TestNodeUnserialisation extends Specification {
)
)
}

"correctly unserialize environment variable" in {
node(linux70Ldif).environmentVariables must containTheSameElementsAs(
List(
EnvironmentVariable("UPSTART_INSTANCE"),
EnvironmentVariable("SHELL", Some("/bin/bash"))
)
)
}

"correctly unserialize processes" in {
node(linux70Ldif).processes must containTheSameElementsAs(
List(
Process(
193,
Some("/var/rudder/cfengine-community/bin/cf-serverd"),
Some(0.0f),
Some(0.0f),
Some("2014-09-25 09:45"),
Some("?"),
Some("root"),
Some(36664)
)
)
)
}

"7.0 and 8.3 ldif lead to the same node" in {
val n1 = node(linux70Ldif)
val n2 = node(linux83Ldif)
n1 === n2
}

"idempotence of unser/ser for 8.3" in {
val nodeEntry = new LDAPEntry(new Entry(linux83Ldif.split("\n").toSeq*))
val prog = for {
n <- mapper.nodeFromEntry(nodeEntry)
e = mapper.treeFromNode(n).root
p = mapper.processesFromNode(n)
_ = e.resetValuesTo(LDAPConstants.A_PROCESS, p*)
} yield e

val entry = ZioRuntime.unsafeRun(prog)
val diff = com.unboundid.ldap.sdk.Entry.diff(nodeEntry.backed, entry.backed, false, true)
diff.toArray() must beEmpty
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -530,7 +530,7 @@ object NodeFact {
// really not sure about what to do here
softwareIds = List(),
node.accounts,
node.environmentVariables.map { case (a, b) => EnvironmentVariable(a, Some(b), None) },
node.environmentVariables.map { case (a, b) => EnvironmentVariable(a, Some(b)) },
node.processes,
node.vms,
node.networks,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -530,7 +530,7 @@ class TestCoreNodeFactInventory extends Specification with BeforeAfterAll {
.getAttributeValue("environmentVariable") must beEqualTo("""{"name":"envVAR","value":"envVALUE"}""")) and
(mockLdapFactStorage.testServer
.getEntry("nodeId=node7,ou=Nodes,ou=Accepted Inventories,ou=Inventories,cn=rudder-configuration")
.getAttributeValue("process") must beEqualTo("""{"pid":4242,"commandName":"process 4242 command line"}""")) and
.getAttributeValue("process") must beEqualTo("""{"pid":4242,"name":"process 4242 command line"}""")) and
(mockLdapFactStorage.testServer.entryExists(
"networkInterface=eth0,nodeId=node7,ou=Nodes,ou=Accepted Inventories,ou=Inventories,cn=rudder-configuration"
) must beTrue) and
Expand Down

0 comments on commit 2624c12

Please sign in to comment.