🧹 remove deprecated v7 structs (#916)

* 🧹 remove deprecated v7 structs
* 🧹 update yac
* fix tests

Makefile

@@ -77,6 +77,7 @@ policy/generate:
 	go generate ./policy
 	go generate ./policy/scan
 	go generate ./policy/upstream
+	go generate ./internal/bundle/yacit
 
 upstream/generate:
 	go generate ./upstream

internal/bundle/bundle_test.go

@@ -29,26 +29,6 @@ func TestParser(t *testing.T) {
 	}, baseline.Queries[0].Impact)
 }
 
-func TestParser_DeprecatedV7(t *testing.T) {
-	raw, err := os.ReadFile("../../policy/deprecated_v7.mql.yaml")
-	require.NoError(t, err)
-	require.NotEmpty(t, raw)
-
-	v8raw, err := DeprecatedV7_ToV8(raw)
-	require.NoError(t, err)
-
-	baseline, err := ParseYaml(v8raw)
-	require.NoError(t, err)
-	assert.NotNil(t, baseline)
-	assert.Equal(t, 5, len(baseline.Queries))
-	assert.Equal(t, &Impact{
-		Value: &ImpactValue{
-			Value: 30,
-		},
-		FileContext: FileContext{27, 13},
-	}, baseline.Queries[0].Impact)
-}
-
 func TestRemediationDecoding(t *testing.T) {
 	t.Run("simple remediation text", func(t *testing.T) {
 		desc := "remediation text"

internal/bundle/fmt.go

@@ -5,7 +5,6 @@ package bundle
 
 import (
 	"bytes"
-	"encoding/json"
 	"os"
 	"strings"
 	"unicode"
@@ -14,7 +13,6 @@ import (
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnspec/v9/policy"
 	"gopkg.in/yaml.v3"
-	k8s_yaml "sigs.k8s.io/yaml"
 )
 
 // Formats the given bundle to a yaml string
@@ -60,38 +58,6 @@ func ParseYaml(data []byte) (*Bundle, error) {
 	return &baseline, err
 }
 
-func DeprecatedV7_ToV8(data []byte) ([]byte, error) {
-	// In the case of deprecated V7, we are only going to focus on the
-	// conversion, throwing away everything else, including comments.
-	// The focus is to get it to v8, none of the other formatting matters in this
-	// step.
-	v7baseline := policy.DeprecatedV7_Bundle{}
-	if err := k8s_yaml.Unmarshal([]byte(data), &v7baseline); err != nil {
-		return nil, err
-	}
-
-	v8 := v7baseline.ToV8()
-
-	// this step will unfortunately not produce well-formatted YAML at all
-	// because the proto structures don't have the yaml tags (only the
-	// yac-it structures do) ...
-	// for the same reason we convert between the 2 types using JSON since both
-	// structs contain the JSON tags.
-	interim, err := json.Marshal(v8)
-	if err != nil {
-		return nil, err
-	}
-
-	// ... so we have to ping pong convert it a bit ...
-	v8yaci, err := ParseYaml(interim)
-	if err != nil {
-		return nil, err
-	}
-
-	// ... until we have it where we want it
-	return Format(v8yaci)
-}
-
 // sanitizeStringForYaml is here to help generating literal style yaml strings
 // if a string has a trailing space in a line, it is automatically converted into quoted style
 func sanitizeStringForYaml(s string) string {
@@ -102,15 +68,32 @@ func sanitizeStringForYaml(s string) string {
 	return strings.Join(lines, "\n")
 }
 
-// Format formats the .mql.yaml bundle
 func FormatFile(filename string) error {
 	log.Info().Str("file", filename).Msg("format file")
 	data, err := os.ReadFile(filename)
 	if err != nil {
 		return err
 	}
 
+	data, err = FormatBundleData(data)
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(filename, data, 0o644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Format formats the .mql.yaml bundle
+func FormatBundleData(data []byte) ([]byte, error) {
 	b, err := ParseYaml(data)
+	if err != nil {
+		return nil, err
+	}
 
 	// to improve the formatting we need to remove the whitespace at the end of the lines
 	for i := range b.Queries {
@@ -142,31 +125,5 @@ func FormatFile(filename string) error {
 		}
 	}
 
-	// we have v7 structs in v8 bundle, so it can happen that v7 parses properly
-	// for that case we need to make sure all the structs are properly converted
-	if err != nil || hasV7Structs(b) {
-		data, err = DeprecatedV7_ToV8(data)
-	} else {
-		data, err = Format(b)
-	}
-	if err != nil {
-		return err
-	}
-
-	err = os.WriteFile(filename, data, 0o644)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func hasV7Structs(b *Bundle) bool {
-	for i := range b.Policies {
-		p := b.Policies[i]
-		if len(p.Specs) > 0 {
-			return true
-		}
-	}
-	return false
+	return Format(b)
 }

internal/bundle/fmt_test.go

@@ -21,11 +21,10 @@ policies:
       key: value
       another-key: another-value
     name: SSH Server Policy
-    specs:
-      - asset_filter:
-          query: asset.family.contains('unix')
-        scoring_queries:
-          query1:
+    groups:
+      - filters: asset.family.contains('unix')
+        checks:
+          - uid: query1
     version: "1.0.0"
     scoring_system: 2
 queries:
@@ -37,20 +36,19 @@ queries:
         Run the "mokutil --sb-state" command and check whether it prints "SecureBoot enabled"
       remediation: |
         Enable Secure Boot in your computer's firmware and use a Linux distribution supporting Secure Boot
-    query: |
+    mql: |
       command('mokutil --sb-state').stdout.downcase.contains('secureboot enabled')
-    severity: 100
+    impact: 100
     title: Ensure Secure Boot is enabled
 `
 
-	formatted, err := DeprecatedV7_ToV8([]byte(data))
+	formatted, err := FormatBundleData([]byte(data))
 	require.NoError(t, err)
 
 	expected := `policies:
   - uid: sshd-server-policy
     name: SSH Server Policy
     version: 1.0.0
-    license: unspecified
     tags:
       another-key: another-value
       key: value

internal/bundle/yacit/main.go

@@ -48,9 +48,10 @@ func main() {
 
 			"Authors": 51,
 
-			"Docs":   50,
-			"Refs":   49,
-			"Groups": 40,
+			"Docs":          50,
+			"Refs":          49,
+			"Groups":        40,
+			"ScoringSystem": 39,
 
 			// frameworks
 			"FrameworkOwner":        90,
@@ -62,7 +63,6 @@ func main() {
 	})
 
 	res.Add(&policy.Bundle{})
-	res.Add(&policy.DeprecatedV7_Bundle{})
 
 	code := res.String()
 	formatted, err := format.Source([]byte(code))

policy/bundle.go

@@ -166,45 +166,7 @@ func aggregateBundles(a *Bundle, b *Bundle) *Bundle {
 func BundleFromYAML(data []byte) (*Bundle, error) {
 	var res Bundle
 	err := yaml.Unmarshal(data, &res)
-
-	// FIXME: DEPRECATED, remove in v9.0 vv
-	// first we want to see if this looks like a new Bundle. If it does, just
-	// return it and we are done. But if it doesn't, then we will try to
-	// parse it as a v7 bundle instead and see if that works.
-	if err == nil {
-		// Only new policies and bundles support logic where you don't have
-		// any policy in the bundle at all.
-		if len(res.Policies) == 0 {
-			return &res, nil
-		}
-
-		// If the policy as the groups field, then we know it's a new one
-		for i := range res.Policies {
-			cur := res.Policies[i]
-			if cur.Groups != nil {
-				return &res, nil
-			}
-		}
-	}
-
-	// We either got here because there is an error, or because it may also
-	// be an old bundle. So let's try to parse it as an old bundle.
-	var altRes DeprecatedV7_Bundle
-	altErr := yaml.Unmarshal(data, &altRes)
-	if altErr == nil && len(altRes.Policies) != 0 {
-		// we still want to do a sanity check that this is a valid v7 policy
-		for i := range altRes.Policies {
-			cur := altRes.Policies[i]
-			if cur.Specs != nil {
-				return altRes.ToV8(), nil
-			}
-		}
-	}
-
-	// This is the final fallthrough, where we either have an error or
-	// it's not a valid v7 policy
 	return &res, err
-	// ^^
 }
 
 // ToYAML returns the policy bundle as yaml
@@ -612,10 +574,6 @@ func (p *Bundle) Compile(ctx context.Context, schema llx.Schema, library Library
 		ownerMrn = "//local.cnspec.io/run/local-execution"
 	}
 
-	// FIXME: DEPRECATED, remove in v9.0 vv
-	p.DeprecatedV7Conversions()
-	// ^^
-
 	cache := &bundleCache{
 		ownerMrn:    ownerMrn,
 		bundle:      p,