Skip to content

Commit

Permalink
Add compact umbilical format (#31)
Browse files Browse the repository at this point in the history 
Add compact umbilical format

This is only available for CAs that require umbilical evidence.

Instead of storing the certificates directly in the evidence, we store
the sha256 fingerprint.

In each batch, the CA publishes an extra file called umbilical-certificates.
This is just a list of X509 certificates together with an index to look
them up by hash. The certificate referred to in the evidence doesn't
have to appear in the same batch, but could appear in any batch within
the validity window.

This deduplicates intermediates, and deduplicates when multiple MTCs are
issued using the same umbilical leaf certificate.

Co-authored-by: Luke Valenta <[email protected]>
  • Loading branch information
@bwesterb @lukevalenta
bwesterb and lukevalenta authored Mar 7, 2025
1 parent df6521e commit 973b433
Show file tree
Hide file tree
Showing 4 changed files with 658 additions and 18 deletions.
185 changes: 167 additions & 18 deletions ca/ca.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (

"github.com/bwesterb/mtc"
"github.com/bwesterb/mtc/umbilical"
"github.com/bwesterb/mtc/umbilical/frozencas"
"github.com/bwesterb/mtc/umbilical/revocation"

"github.com/nightlyone/lockfile"
Expand Down Expand Up @@ -62,11 +63,12 @@ type Handle struct {
// Caches. Access requires either write lock on mux, or a read lock on mux
// and a lock on cacheMux.
cacheMux sync.Mutex
indices map[uint32]*Index
aas map[uint32]*os.File
evs map[uint32]*os.File
trees map[uint32]*Tree
batchNumbersCache []uint32 // cache for existing batches
indices map[uint32]*Index // index files
aas map[uint32]*os.File // abridged-assertions files
evs map[uint32]*os.File // evidence files
trees map[uint32]*Tree // tree files
ucs map[uint32]*frozencas.Handle // umbilical-certificates
batchNumbersCache []uint32 // cache for existing batches
}

func (ca *Handle) Params() mtc.CAParams {
Expand Down Expand Up @@ -101,6 +103,10 @@ func (ca *Handle) Close() error {
t.Close()
}

for _, r := range ca.ucs {
r.Close()
}

ca.closed = true
return ca.flock.Unlock()
}
Expand Down Expand Up @@ -347,6 +353,7 @@ func Open(path string) (*Handle, error) {
indices: make(map[uint32]*Index),
aas: make(map[uint32]*os.File),
evs: make(map[uint32]*os.File),
ucs: make(map[uint32]*frozencas.Handle),
trees: make(map[uint32]*Tree),
}
if err := h.lockFolder(); err != nil {
Expand Down Expand Up @@ -411,6 +418,10 @@ func (h *Handle) indexPath(number uint32) string {
return gopath.Join(h.batchPath(number), "index")
}

func (h *Handle) ucPath(number uint32) string {
return gopath.Join(h.batchPath(number), "umbilical-certificates")
}

func (h *Handle) aaPath(number uint32) string {
return gopath.Join(h.batchPath(number), "abridged-assertions")
}
Expand Down Expand Up @@ -648,6 +659,14 @@ func (h *Handle) closeBatch(batch uint32) error {
delete(h.evs, batch)
}

if r, ok := h.ucs[batch]; ok {
err := r.Close()
if err != nil {
return fmt.Errorf("closing umbilical-certificates for %d: %w", batch, err)
}
delete(h.ucs, batch)
}

if r, ok := h.trees[batch]; ok {
err := r.Close()
if err != nil {
Expand Down Expand Up @@ -734,6 +753,25 @@ func (ca *Handle) evFileFor(batch uint32) (*os.File, error) {
return r, nil
}

// Returns the umbilical certificates file for the given batch.
func (ca *Handle) ucFor(batch uint32) (*frozencas.Handle, error) {
ca.cacheMux.Lock()
defer ca.cacheMux.Unlock()

if r, ok := ca.ucs[batch]; ok {
return r, nil
}

r, err := frozencas.Open(ca.ucPath(batch))
if err != nil {
return nil, err
}

ca.ucs[batch] = r

return r, nil
}

type keySearchResult struct {
Batch uint32
SequenceNumber uint64
Expand Down Expand Up @@ -1018,17 +1056,17 @@ func (h *Handle) issueBatch(number uint32, empty bool) error {
}

// Ok, let's compare
err = assertFilesEqual(
dir1,
dir2,
[]string{
"tree",
"signed-validity-window",
"abridged-assertions",
"evidence",
"index",
},
)
toCheck := []string{
"tree",
"signed-validity-window",
"abridged-assertions",
"evidence",
"index",
}
if h.params.EvidencePolicy == mtc.UmbilicalEvidencePolicyType {
toCheck = append(toCheck, "umbilical-certificates")
}
err = assertFilesEqual(dir1, dir2, toCheck)
if err != nil {
return err
}
Expand Down Expand Up @@ -1129,6 +1167,75 @@ func sha256File(path string) ([]byte, error) {
return h.Sum(nil), nil
}

func (h *Handle) compressEvidence(ev mtc.Evidence, batch mtc.Batch,
ucBuilder *frozencas.Builder) (mtc.Evidence, error) {
uev, ok := ev.(mtc.UmbilicalEvidence)
if !ok {
return ev, nil
}

chain, err := uev.RawChain()
if err != nil {
return nil, err
}

// Oldest batch to inspect for umbilical certificate
end := int64(batch.Number) - int64(h.params.ValidityWindowSize)
if end < 0 {
end = 0
}

hasher := sha256.New()
hashes := make([][32]byte, len(chain))
for i, cert := range chain {
_, _ = hasher.Write(cert)
hasher.Sum(hashes[i][:0])
hash := hashes[i]
hasher.Reset()

ok := false
for bn := int64(batch.Number) - 1; bn >= end; bn-- {
uc, err := h.ucFor(uint32(bn))
if err != nil {
return nil, fmt.Errorf(
"opening umbilical certificates for batch %d: %w",
bn,
err,
)
}

blob, err := uc.Get(hash[:])
if err != nil {
return nil, fmt.Errorf(
"Looking up umbilical certificate hash in batch %d: %w",
bn,
err,
)
}

if blob != nil {
ok = true // found!
break
}
}

if ok {
continue
}

// Umbilical certificate not logged yet.
err = ucBuilder.Add(cert)
if err != nil {
return nil, fmt.Errorf(
"Writing umbilical certificate to frozencas: %w",
err,
)
}
}

return mtc.NewCompressedUmbilicalEvidence(hashes)
}

// Like issueBatch, but don't write out to the correct directory yet.
// Instead, write to dir. Also, don't empty the queue.
func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error {
Expand Down Expand Up @@ -1167,6 +1274,23 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error {
defer evW.Close()
evBW := bufio.NewWriter(evW)

ucPath := gopath.Join(dir, "umbilical-certificates")
var (
ucBuilder *frozencas.Builder
ucW *os.File
)
if h.params.EvidencePolicy == mtc.UmbilicalEvidencePolicyType {
ucW, err = os.Create(ucPath)
if err != nil {
return fmt.Errorf("creating %s: %w", ucPath, err)
}
defer ucW.Close()
ucBuilder, err = frozencas.NewBuilder(ucW)
if err != nil {
return fmt.Errorf("creating %s: %w", ucPath, err)
}
}

if !empty {
err = h.walkQueue(func(ar mtc.AssertionRequest) error {
// Skip assertions that are already expired.
Expand All @@ -1192,8 +1316,22 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error {
)
}

ev := ar.Evidence
buf, err = ev.MarshalBinary()
evs := ar.Evidence
if ucBuilder != nil {
for i := range len(evs) {
evs[i], err = h.compressEvidence(evs[i], batch, ucBuilder)
if err != nil {
return fmt.Errorf(
"Compressing evidence #%d for %x: %w",
i,
ar.Checksum,
err,
)
}
}
}

buf, err = evs.MarshalBinary()
if err != nil {
return fmt.Errorf("Marshalling evidence %x: %w", ar.Checksum, err)
}
Expand Down Expand Up @@ -1245,6 +1383,17 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error {
}
defer evR.Close()

if ucBuilder != nil {
err = ucBuilder.Finish()
if err != nil {
return fmt.Errorf("finishing %s: %w", ucPath, err)
}
err = ucW.Close()
if err != nil {
return fmt.Errorf("closing %s: %w", ucPath, err)
}
}

// Compute tree
tree, err := batch.ComputeTree(bufio.NewReader(aasR))
if err != nil {
Expand Down
76 changes: 76 additions & 0 deletions cmd/mtc/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/bwesterb/mtc"
"github.com/bwesterb/mtc/ca"
"github.com/bwesterb/mtc/umbilical"
"github.com/bwesterb/mtc/umbilical/frozencas"
"github.com/urfave/cli/v2"
"golang.org/x/crypto/cryptobyte"

Expand All @@ -27,6 +28,7 @@ import (
var (
errNoCaParams = errors.New("missing ca-params flag")
errArgs = errors.New("Wrong number of arguments")
errNotFound = errors.New("not found")
fCpuProfile *os.File
evPolicyMap = map[string]mtc.EvidencePolicyType{
"empty": mtc.EmptyEvidencePolicyType,
Expand Down Expand Up @@ -650,6 +652,60 @@ func handleInspectSignedValidityWindow(cc *cli.Context) error {
return nil
}

func handleInspectUC(cc *cli.Context) error {
if cc.Args().Len() != 1 {
return errArgs
}

uc, err := frozencas.Open(cc.Args().Get(0))
if err != nil {
return err
}
defer uc.Close()

if cc.IsSet("key") {
key, err := hex.DecodeString(cc.String("key"))
if err != nil {
return err
}
blob, err := uc.Get(key)
if err != nil {
return err
}

if blob == nil {
return errNotFound
}

cert, err := x509.ParseCertificate(blob)
if err != nil {
return err
}

fmt.Printf("subject\t%s\n", cert.Subject.String())
fmt.Printf("issuer\t%s\n", cert.Issuer.String())
fmt.Printf("serial_no\t%x\n", cert.SerialNumber)
fmt.Printf("not_before\t%s\n", cert.NotBefore)
fmt.Printf("not_after\t%s\n", cert.NotAfter)

return nil
}

total := 0
entries, err := uc.Entries()
if err != nil {
return err
}
fmt.Printf("%64s %7s %7s\n", "key", "offset", "length")
for _, entry := range entries {
fmt.Printf("%x %7d %7d\n", entry.Key, entry.Offset, entry.Length)
total++
}

fmt.Printf("\ntotal number of entries: %d\n", total)
return nil
}

func handleInspectIndex(cc *cli.Context) error {
buf, err := inspectGetBuf(cc)
if err != nil {
Expand Down Expand Up @@ -745,6 +801,13 @@ func writeEvidenceList(w *tabwriter.Writer, el mtc.EvidenceList) error {
fmt.Fprintf(w, "evidence-list (%d entries)\n", len(el))
for _, ev := range el {
switch ev.Type() {
case mtc.CompressedUmbilicalEvidenceType:
fmt.Fprintf(w, "compressed umbilical\n")
chain := ev.(mtc.CompressedUmbilicalEvidence).Chain()
for _, cert := range chain {
fmt.Fprintf(w, " %x\n", cert)
}

case mtc.UmbilicalEvidenceType:
fmt.Fprintf(w, "umbilical\n")
chain, err := ev.(mtc.UmbilicalEvidence).Chain()
Expand Down Expand Up @@ -1138,6 +1201,19 @@ func main() {
Action: handleInspectCert,
ArgsUsage: "[path]",
},
{
Name: "umbilical-certificates",
Usage: "parses batch's umbilical-certificates file",
Action: handleInspectUC,
ArgsUsage: "path",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "key",
Usage: "key to look up",
Aliases: []string{"k"},
},
},
},
},
Flags: []cli.Flag{
&cli.StringFlag{
Expand Down
Loading

0 comments on commit 973b433

Please sign in to comment.