From 2983757d24da95f7e63ee36fd26ddc7963282606 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Thu, 9 May 2024 16:55:27 +0200 Subject: [PATCH 1/8] wip --- internal/layout/flex.go | 333 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 internal/layout/flex.go diff --git a/internal/layout/flex.go b/internal/layout/flex.go new file mode 100644 index 0000000..65e2838 --- /dev/null +++ b/internal/layout/flex.go @@ -0,0 +1,333 @@ +package layout + +import ( + // . "photofield/internal" + + "fmt" + "math" + "photofield/internal/image" + "photofield/internal/render" + "time" +) + +type FlexPhoto struct { + Id image.ImageId + AspectRatio float64 +} + +type FlexNode struct { + Index int + Cost float64 + ImageHeight float64 + TotalAspect float64 + Links []*FlexNode + Shortest *FlexNode +} + +// func (n *FlexNode) Dot() string { +// // dot := "" +// dot := fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\"];\n", n.Index, n.Index, n.Cost, n.ImageHeight) +// for _, link := range n.Links { +// dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) +// // dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) +// dot += link.Dot() +// } +// return dot +// } + +func (n *FlexNode) Dot() string { + // dot := "" + + stack := []*FlexNode{n} + visited := make(map[int]bool) + dot := "" + for len(stack) > 0 { + node := stack[0] + stack = stack[1:] + if visited[node.Index] { + continue + } + visited[node.Index] = true + dot += fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\\nTotalAspect: %.2f\"];\n", node.Index, node.Index, node.Cost, node.ImageHeight, node.TotalAspect) + for _, link := range node.Links { + attr := "" + if link.Shortest == node { + attr = " [penwidth=3]" + } + dot += fmt.Sprintf("\t%d -> %d%s;\n", node.Index, link.Index, attr) + stack = append(stack, link) + } + } + + // dot := fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\"];\n", n.Index, n.Index, n.Cost, n.ImageHeight) + // for _, link := range n.Links { + // dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) + // // dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) + // dot += link.Dot() + // } + return dot +} + +func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Scene, source *image.Source) { + + layout.ImageSpacing = 0.02 * layout.ImageHeight + layout.LineSpacing = 0.02 * layout.ImageHeight + + sceneMargin := 10. + + scene.Bounds.W = layout.ViewportWidth + + rect := render.Rect{ + X: sceneMargin, + Y: sceneMargin + 64, + W: scene.Bounds.W - sceneMargin*2, + H: 0, + } + + // section := Section{} + + scene.Solids = make([]render.Solid, 0) + scene.Texts = make([]render.Text, 0) + + // layoutPlaced := metrics.Elapsed("layout placing") + // layoutCounter := metrics.Counter{ + // Name: "layout", + // Interval: 1 * time.Second, + // } + + // row := make([]SectionPhoto, 0) + + // x := 0. + // y := 0. + + idealHeight := layout.ImageHeight + minHeight := 0.8 * idealHeight + maxHeight := 1.2 * idealHeight + + // baseWidth := layout.ViewportWidth * 0.29 + + photos := make([]FlexPhoto, 0) + + for info := range infos { + photo := FlexPhoto{ + Id: info.Id, + AspectRatio: float64(info.Width) / float64(info.Height), + } + photos = append(photos, photo) + } + + for startTime := time.Now(); time.Since(startTime) < 10*time.Second; { + + scene.Photos = scene.Photos[:0] + root := &FlexNode{ + Index: -1, + Cost: 0, + Links: nil, + TotalAspect: 0, + } + stack := []*FlexNode{root} + indexToNode := make(map[int]*FlexNode) + + maxLineWidth := rect.W + + for len(stack) > 0 { + node := stack[0] + stack = stack[1:] + + totalAspect := 0. + + // fmt.Printf("stack %d\n", node.Index) + + for i := node.Index + 1; i < len(photos); i++ { + photo := photos[i] + totalAspect += photo.AspectRatio + totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) + photoHeight := (maxLineWidth - totalSpacing) / totalAspect + valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 + badness := math.Abs(photoHeight - idealHeight) + cost := badness*badness + 10 + // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) + // Handle edge case where there is no other option + // but to accept a photo that would otherwise break outside of the desired size + if photoHeight < minHeight && len(stack) == 0 { + valid = true + } + if valid { + n, ok := indexToNode[i] + totalCost := node.Cost + cost + if ok { + if n.Cost > totalCost { + n.Cost = totalCost + n.TotalAspect = totalAspect + n.Shortest = node + // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) + } else { + // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) + } + } else { + n = &FlexNode{ + Index: i, + Cost: totalCost, + ImageHeight: photoHeight, + Links: nil, + TotalAspect: totalAspect, + Shortest: node, + } + indexToNode[i] = n + if i < len(photos)-1 { + stack = append(stack, n) + } + // fmt.Printf(" node %d added with cost %f total aspect %f\n", i, n.Cost, n.TotalAspect) + } + node.Links = append(node.Links, n) + } + if photoHeight < minHeight { + break + } else if photoHeight > maxHeight { + continue + } + } + } + + // dot := "digraph NodeGraph {\n" + // dot += root.Dot() + // dot += "}" + // fmt.Println(dot) + + // fmt.Printf("photos %d\n", len(photos)) + + shortestPath := make([]*FlexNode, 0) + for node := indexToNode[len(photos)-1]; node != nil; { + // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) + shortestPath = append(shortestPath, node) + node = node.Shortest + } + + // fmt.Printf("max line width %f\n", maxLineWidth) + x := 0. + y := 0. + idx := 0 + for i := len(shortestPath) - 2; i >= 0; i-- { + node := shortestPath[i] + prev := shortestPath[i+1] + totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) + imageHeight := (maxLineWidth - totalSpacing) / node.TotalAspect + // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) + for ; idx <= node.Index; idx++ { + photo := photos[idx] + imageWidth := imageHeight * photo.AspectRatio + scene.Photos = append(scene.Photos, render.Photo{ + Id: photo.Id, + Sprite: render.Sprite{ + Rect: render.Rect{ + X: rect.X + x, + Y: rect.Y + y, + W: imageWidth, + H: imageHeight, + }, + }, + }) + x += imageWidth + layout.ImageSpacing + // fmt.Printf("photo %d aspect %f\n", idx, photo.AspectRatio) + } + x = 0 + y += imageHeight + layout.LineSpacing + } + + // idx := 0 + // for node := root; node != nil; { + + // bestCost := math.MaxFloat64 + // var bestNode *FlexNode + // for _, link := range node.Links { + // if link.Cost < bestCost { + // bestCost = link.Cost + // bestNode = link + // } + // } + // node = bestNode + // if node == nil { + // break + // } + // imageHeight := maxLineWidth / node.TotalAspect + // fmt.Printf("node %d cost %f total aspect %f height %f\n", node.Index, node.Cost, node.TotalAspect, imageHeight) + // for ; idx <= node.Index; idx++ { + // photo := photos[idx] + // imageWidth := imageHeight * photo.AspectRatio + // scene.Photos = append(scene.Photos, render.Photo{ + // Id: photo.Id, + // Sprite: render.Sprite{ + // Rect: render.Rect{ + // X: rect.X + x, + // Y: rect.Y + y, + // W: imageWidth, + // H: imageHeight, + // }, + // }, + // }) + // x += imageWidth + // fmt.Printf("photo %d aspect %f\n", idx, photo.AspectRatio) + // } + // x = 0 + // y += imageHeight + // } + + // index := 0 + // for info := range infos { + // photo := SectionPhoto{ + // Photo: render.Photo{ + // Id: info.Id, + // Sprite: render.Sprite{}, + // }, + // Size: image.Size{ + // X: info.Width, + // Y: info.Height, + // }, + // } + + // imageWidth := baseWidth + // // section.infos = append(section.infos, info.SourcedInfo) + + // if x+imageWidth > rect.W { + // for _, p := range row { + // scene.Photos = append(scene.Photos, p.Photo) + // } + // row = nil + // x = 0 + // y += layout.ImageHeight + layout.LineSpacing + // } + + // photo.Photo.Sprite.PlaceFitWidth( + // rect.X+x, + // rect.Y+y, + // imageWidth, + // float64(photo.Size.X), + // float64(photo.Size.Y), + // ) + + // row = append(row, photo) + + // x += imageWidth + layout.ImageSpacing + + // layoutCounter.Set(index) + // index++ + // scene.FileCount = index + // } + // for _, p := range row { + // scene.Photos = append(scene.Photos, p.Photo) + // } + // x = 0 + // y += layout.ImageHeight + layout.LineSpacing + + // rect.Y = y + + // newBounds := addSectionToScene(§ion, scene, rect, layout, source) + // layoutPlaced() + + scene.Bounds.H = rect.Y + y + sceneMargin + } + + scene.RegionSource = PhotoRegionSource{ + Source: source, + } +} From f2384e269331d2376425debb5a2e8087dd83b651 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Fri, 10 May 2024 20:21:07 +0200 Subject: [PATCH 2/8] Add first working versions of flex and highlights layouts --- go.mod | 1 + go.sum | 2 + internal/clip/clip.go | 13 + internal/geo/geo.go | 3 + internal/image/database.go | 133 ++++++- internal/image/source.go | 22 ++ internal/layout/common.go | 16 +- internal/layout/flex.go | 413 ++++++++++------------ internal/layout/highlights.go | 479 ++++++++++++++++++++++++++ internal/render/sprite.go | 20 ++ internal/render/text.go | 9 +- internal/scene/sceneSource.go | 11 +- main.go | 2 +- ui/src/components/DisplaySettings.vue | 2 + 14 files changed, 882 insertions(+), 244 deletions(-) create mode 100644 internal/layout/highlights.go diff --git a/go.mod b/go.mod index 03ed1e1..168a0da 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.9.0 // indirect github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 // indirect + github.com/gammazero/deque v0.2.1 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/go.sum b/go.sum index 81ab621..c8629b6 100644 --- a/go.sum +++ b/go.sum @@ -174,6 +174,8 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= +github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= +github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= diff --git a/internal/clip/clip.go b/internal/clip/clip.go index 985a6db..9904e14 100644 --- a/internal/clip/clip.go +++ b/internal/clip/clip.go @@ -29,6 +29,19 @@ func DotProductFloat32Float(a []float32, b []Float) (float32, error) { return dot, nil } +func DotProductFloat32Float32(a []float32, b []float32) (float32, error) { + l := len(a) + if l != len(b) { + return 0, fmt.Errorf("slice lengths do not match, a %d b %d", l, len(b)) + } + + dot := float32(0) + for i := 0; i < l; i++ { + dot += a[i] * b[i] + } + return dot, nil +} + // Most real world inverse vector norms of embeddings fall // within ~500 of 11843, so it's more efficient to store // the inverse vector norm as an offset of this number. diff --git a/internal/geo/geo.go b/internal/geo/geo.go index 2970531..513060e 100644 --- a/internal/geo/geo.go +++ b/internal/geo/geo.go @@ -101,6 +101,9 @@ func (g *Geo) String() string { if g == nil || !g.config.ReverseGeocode { return "geo reverse geocoding disabled" } + if g.gp == nil { + return "geo geopackage not loaded" + } return "geo using " + g.uri } diff --git a/internal/image/database.go b/internal/image/database.go index b448883..8c0ad21 100644 --- a/internal/image/database.go +++ b/internal/image/database.go @@ -1,6 +1,7 @@ package image import ( + "context" "embed" "errors" "fmt" @@ -125,6 +126,17 @@ type TagIdRange struct { type tagSet map[tag.Id]struct{} +func readEmbedding(stmt *sqlite.Stmt, invnormIndex int, embeddingIndex int) (clip.Embedding, error) { + invnorm := uint16(clip.InvNormMean + stmt.ColumnInt64(invnormIndex)) + size := stmt.ColumnLen(embeddingIndex) + bytes := make([]byte, size) + read := stmt.ColumnBytes(embeddingIndex, bytes) + if read != size { + return nil, fmt.Errorf("unable to read embedding bytes, expected %d, got %d", size, read) + } + return clip.FromRaw(bytes, invnorm), nil +} + func (tags *tagSet) Add(id tag.Id) { (*tags)[id] = struct{}{} } @@ -1467,6 +1479,115 @@ func (source *Database) List(dirs []string, options ListOptions) <-chan InfoList return out } +func (source *Database) ListWithEmbeddings(dirs []string, options ListOptions) <-chan InfoEmb { + out := make(chan InfoEmb, 1000) + go func() { + defer metrics.Elapsed("list infos sqlite")() + + conn := source.pool.Get(context.TODO()) + defer source.pool.Put(conn) + + sql := "" + + sql += ` + SELECT infos.id, width, height, orientation, color, created_at_unix, created_at_tz_offset, latitude, longitude, inv_norm, embedding + FROM infos + INNER JOIN clip_emb ON clip_emb.file_id = id + ` + + sql += ` + WHERE path_prefix_id IN ( + SELECT id + FROM prefix + WHERE ` + + for i := range dirs { + sql += `str LIKE ? ` + if i < len(dirs)-1 { + sql += "OR " + } + } + + sql += ` + ) + ` + + switch options.OrderBy { + case None: + case DateAsc: + sql += ` + ORDER BY created_at_unix ASC + ` + case DateDesc: + sql += ` + ORDER BY created_at_unix DESC + ` + default: + panic("Unsupported listing order") + } + + if options.Limit > 0 { + sql += ` + LIMIT ? + ` + } + + sql += ";" + + stmt := conn.Prep(sql) + defer stmt.Reset() + + bindIndex := 1 + + for _, dir := range dirs { + stmt.BindText(bindIndex, dir+"%") + bindIndex++ + } + + if options.Limit > 0 { + stmt.BindInt64(bindIndex, (int64)(options.Limit)) + } + + for { + if exists, err := stmt.Step(); err != nil { + log.Printf("Error listing files: %s\n", err.Error()) + } else if !exists { + break + } + + var info InfoEmb + info.Id = (ImageId)(stmt.ColumnInt64(0)) + + info.Width = stmt.ColumnInt(1) + info.Height = stmt.ColumnInt(2) + info.Orientation = Orientation(stmt.ColumnInt(3)) + info.Color = (uint32)(stmt.ColumnInt64(4)) + + unix := stmt.ColumnInt64(5) + timezoneOffset := stmt.ColumnInt(6) + + info.DateTime = time.Unix(unix, 0).In(time.FixedZone("", timezoneOffset*60)) + + if stmt.ColumnType(7) == sqlite.TypeNull || stmt.ColumnType(8) == sqlite.TypeNull { + info.LatLng = NaNLatLng() + } else { + info.LatLng = s2.LatLngFromDegrees(stmt.ColumnFloat(7), stmt.ColumnFloat(8)) + } + + emb, err := readEmbedding(stmt, 9, 10) + if err != nil { + log.Printf("Error reading embedding: %s\n", err.Error()) + } + info.Embedding = emb + + out <- info + } + + close(out) + }() + return out +} + func (source *Database) GetImageEmbedding(id ImageId) (clip.Embedding, error) { conn := source.pool.Get(nil) defer source.pool.Put(conn) @@ -1553,19 +1674,15 @@ func (source *Database) ListEmbeddings(dirs []string, options ListOptions) <-cha } id := (ImageId)(stmt.ColumnInt64(0)) - invnorm := uint16(clip.InvNormMean + stmt.ColumnInt64(1)) - - size := stmt.ColumnLen(2) - bytes := make([]byte, size) - read := stmt.ColumnBytes(2, bytes) - if read != size { - log.Printf("Error reading embedding: buffer underrun, expected %d actual %d bytes\n", size, read) + emb, err := readEmbedding(stmt, 1, 2) + if err != nil { + log.Printf("Error reading embedding: %s\n", err.Error()) continue } out <- EmbeddingsResult{ Id: id, - Embedding: clip.FromRaw(bytes, invnorm), + Embedding: emb, } } diff --git a/internal/image/source.go b/internal/image/source.go index 8666a44..e63fc5f 100644 --- a/internal/image/source.go +++ b/internal/image/source.go @@ -80,6 +80,11 @@ type SimilarityInfo struct { Similarity float32 } +type InfoEmb struct { + SourcedInfo + Embedding clip.Embedding +} + func SimilarityInfosToSourcedInfos(sinfos <-chan SimilarityInfo) <-chan SourcedInfo { out := make(chan SourcedInfo) go func() { @@ -398,6 +403,23 @@ func (source *Source) ListInfos(dirs []string, options ListOptions) <-chan Sourc return out } +func (source *Source) ListInfosEmb(dirs []string, options ListOptions) <-chan InfoEmb { + for i := range dirs { + dirs[i] = filepath.FromSlash(dirs[i]) + } + out := make(chan InfoEmb, 1000) + go func() { + defer metrics.Elapsed("list infos embedded")() + + infos := source.database.ListWithEmbeddings(dirs, options) + for info := range infos { + out <- info + } + close(out) + }() + return out +} + func (source *Source) ListInfosWithExistence(dirs []string, options ListOptions) <-chan SourcedInfo { for i := range dirs { dirs[i] = filepath.FromSlash(dirs[i]) diff --git a/internal/layout/common.go b/internal/layout/common.go index fdff5cb..a16482a 100644 --- a/internal/layout/common.go +++ b/internal/layout/common.go @@ -17,13 +17,15 @@ import ( type Type string const ( - Album Type = "ALBUM" - Timeline Type = "TIMELINE" - Square Type = "SQUARE" - Wall Type = "WALL" - Map Type = "MAP" - Search Type = "SEARCH" - Strip Type = "STRIP" + Album Type = "ALBUM" + Timeline Type = "TIMELINE" + Square Type = "SQUARE" + Wall Type = "WALL" + Map Type = "MAP" + Search Type = "SEARCH" + Strip Type = "STRIP" + Highlights Type = "HIGHLIGHTS" + Flex Type = "FLEX" ) type Order int diff --git a/internal/layout/flex.go b/internal/layout/flex.go index 65e2838..60c35a1 100644 --- a/internal/layout/flex.go +++ b/internal/layout/flex.go @@ -3,38 +3,35 @@ package layout import ( // . "photofield/internal" - "fmt" + "context" "math" "photofield/internal/image" + "photofield/internal/metrics" "photofield/internal/render" "time" + + "github.com/gammazero/deque" + "github.com/golang/geo/s2" + "github.com/tdewolff/canvas" ) type FlexPhoto struct { Id image.ImageId - AspectRatio float64 + AspectRatio float32 + Aux bool +} + +type FlexAux struct { + Text string } type FlexNode struct { Index int - Cost float64 - ImageHeight float64 - TotalAspect float64 - Links []*FlexNode + Cost float32 + TotalAspect float32 Shortest *FlexNode } -// func (n *FlexNode) Dot() string { -// // dot := "" -// dot := fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\"];\n", n.Index, n.Index, n.Cost, n.ImageHeight) -// for _, link := range n.Links { -// dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) -// // dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) -// dot += link.Dot() -// } -// return dot -// } - func (n *FlexNode) Dot() string { // dot := "" @@ -48,23 +45,16 @@ func (n *FlexNode) Dot() string { continue } visited[node.Index] = true - dot += fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\\nTotalAspect: %.2f\"];\n", node.Index, node.Index, node.Cost, node.ImageHeight, node.TotalAspect) - for _, link := range node.Links { - attr := "" - if link.Shortest == node { - attr = " [penwidth=3]" - } - dot += fmt.Sprintf("\t%d -> %d%s;\n", node.Index, link.Index, attr) - stack = append(stack, link) - } + // dot += fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\\nTotalAspect: %.2f\"];\n", node.Index, node.Index, node.Cost, node.ImageHeight, node.TotalAspect) + // for _, link := range node.Links { + // attr := "" + // if link.Shortest == node { + // attr = " [penwidth=3]" + // } + // dot += fmt.Sprintf("\t%d -> %d%s;\n", node.Index, link.Index, attr) + // stack = append(stack, link) + // } } - - // dot := fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\"];\n", n.Index, n.Index, n.Cost, n.ImageHeight) - // for _, link := range n.Links { - // dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) - // // dot += fmt.Sprintf("\t%d -> %d;\n", n.Index, link.Index) - // dot += link.Dot() - // } return dot } @@ -84,16 +74,10 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce H: 0, } - // section := Section{} - scene.Solids = make([]render.Solid, 0) scene.Texts = make([]render.Text, 0) - // layoutPlaced := metrics.Elapsed("layout placing") - // layoutCounter := metrics.Counter{ - // Name: "layout", - // Interval: 1 * time.Second, - // } + layoutPlaced := metrics.Elapsed("layout placing") // row := make([]SectionPhoto, 0) @@ -106,116 +90,180 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce // baseWidth := layout.ViewportWidth * 0.29 + scene.Photos = scene.Photos[:0] photos := make([]FlexPhoto, 0) + layoutCounter := metrics.Counter{ + Name: "layout", + Interval: 1 * time.Second, + } + + auxs := make([]FlexAux, 0) + + // Fetch all photos + var lastLoc s2.LatLng + var lastLocTime time.Time + var lastLocation string for info := range infos { + if source.Geo.Available() { + photoTime := info.DateTime + lastLocCheck := lastLocTime.Sub(photoTime) + if lastLocCheck < 0 { + lastLocCheck = -lastLocCheck + } + queryLocation := lastLocTime.IsZero() || lastLocCheck > 15*time.Minute + // fmt.Printf("lastLocTime %v photoTime %v lastLocCheck %v queryLocation %v\n", lastLocTime, photoTime, lastLocCheck, queryLocation) + if queryLocation && image.IsValidLatLng(info.LatLng) { + lastLocTime = photoTime + dist := image.AngleToKm(lastLoc.Distance(info.LatLng)) + if dist > 1 { + location, err := source.Geo.ReverseGeocode(context.TODO(), info.LatLng) + if err == nil && location != lastLocation { + lastLocation = location + aux := FlexAux{ + Text: location, + } + auxs = append(auxs, aux) + photos = append(photos, FlexPhoto{ + Id: image.ImageId(len(auxs) - 1), + AspectRatio: float32(len(location)) / 5, + Aux: true, + }) + } + lastLoc = info.LatLng + } + } + } photo := FlexPhoto{ Id: info.Id, - AspectRatio: float64(info.Width) / float64(info.Height), + AspectRatio: float32(info.Width) / float32(info.Height), } photos = append(photos, photo) + layoutCounter.Set(len(photos)) } - for startTime := time.Now(); time.Since(startTime) < 10*time.Second; { - - scene.Photos = scene.Photos[:0] - root := &FlexNode{ - Index: -1, - Cost: 0, - Links: nil, - TotalAspect: 0, - } - stack := []*FlexNode{root} - indexToNode := make(map[int]*FlexNode) - - maxLineWidth := rect.W - - for len(stack) > 0 { - node := stack[0] - stack = stack[1:] - - totalAspect := 0. + root := &FlexNode{ + Index: -1, + Cost: 0, + TotalAspect: 0, + } - // fmt.Printf("stack %d\n", node.Index) + q := deque.New[*FlexNode](len(photos) / 4) + q.PushBack(root) + indexToNode := make(map[int]*FlexNode, len(photos)) + + maxLineWidth := rect.W + + for q.Len() > 0 { + node := q.PopFront() + totalAspect := 0. + fallback := false + + // fmt.Printf("queue %d\n", node.Index) + for i := node.Index + 1; i < len(photos); i++ { + photo := photos[i] + totalAspect += float64(photo.AspectRatio) + totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) + photoHeight := (maxLineWidth - totalSpacing) / totalAspect + valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback + badness := math.Abs(photoHeight - idealHeight) + cost := badness*badness + 10 + if i < len(photos)-1 && photos[i+1].Aux { + cost *= 0.1 + } - for i := node.Index + 1; i < len(photos); i++ { - photo := photos[i] - totalAspect += photo.AspectRatio - totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) - photoHeight := (maxLineWidth - totalSpacing) / totalAspect - valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 - badness := math.Abs(photoHeight - idealHeight) - cost := badness*badness + 10 - // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) + // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) + + // Handle edge case where there is no other option + // but to accept a photo that would otherwise break outside of the desired size + // if i != len(photos)-1 && q.Len() == 0 { + // valid = true + // } + if valid { + n, ok := indexToNode[i] + totalCost := node.Cost + float32(cost) + if ok { + if n.Cost > totalCost { + n.Cost = totalCost + n.TotalAspect = float32(totalAspect) + n.Shortest = node + // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) + } + // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) + // } + } else { + n = &FlexNode{ + Index: i, + Cost: totalCost, + TotalAspect: float32(totalAspect), + Shortest: node, + } + indexToNode[i] = n + if i < len(photos)-1 { + q.PushBack(n) + } + // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) + } + // fmt.Printf(" node %d %v cost %f\n", i, ok, n.Cost) + } + if photoHeight < minHeight { // Handle edge case where there is no other option // but to accept a photo that would otherwise break outside of the desired size - if photoHeight < minHeight && len(stack) == 0 { - valid = true - } - if valid { - n, ok := indexToNode[i] - totalCost := node.Cost + cost - if ok { - if n.Cost > totalCost { - n.Cost = totalCost - n.TotalAspect = totalAspect - n.Shortest = node - // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) - } else { - // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) - } - } else { - n = &FlexNode{ - Index: i, - Cost: totalCost, - ImageHeight: photoHeight, - Links: nil, - TotalAspect: totalAspect, - Shortest: node, - } - indexToNode[i] = n - if i < len(photos)-1 { - stack = append(stack, n) - } - // fmt.Printf(" node %d added with cost %f total aspect %f\n", i, n.Cost, n.TotalAspect) + if !fallback && i != len(photos)-1 && q.Len() == 0 { + fallback = true + for j := 0; j < 2 && i > node.Index; j++ { + // fmt.Printf(" fallback %d\n", i) + totalAspect -= float64(photos[i].AspectRatio) + i-- } - node.Links = append(node.Links, n) - } - if photoHeight < minHeight { - break - } else if photoHeight > maxHeight { continue } + break } } + } - // dot := "digraph NodeGraph {\n" - // dot += root.Dot() - // dot += "}" - // fmt.Println(dot) - - // fmt.Printf("photos %d\n", len(photos)) - - shortestPath := make([]*FlexNode, 0) - for node := indexToNode[len(photos)-1]; node != nil; { - // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) - shortestPath = append(shortestPath, node) - node = node.Shortest - } + // dot := "digraph NodeGraph {\n" + // dot += root.Dot() + // dot += "}" + // fmt.Println(dot) + + // Trace back the shortest path + shortestPath := make([]*FlexNode, 0) + for node := indexToNode[len(photos)-1]; node != nil; { + // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) + shortestPath = append(shortestPath, node) + node = node.Shortest + } - // fmt.Printf("max line width %f\n", maxLineWidth) - x := 0. - y := 0. - idx := 0 - for i := len(shortestPath) - 2; i >= 0; i-- { - node := shortestPath[i] - prev := shortestPath[i+1] - totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) - imageHeight := (maxLineWidth - totalSpacing) / node.TotalAspect - // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) - for ; idx <= node.Index; idx++ { - photo := photos[idx] - imageWidth := imageHeight * photo.AspectRatio + // Finally, place the photos based on the shortest path breaks + x := 0. + y := 0. + idx := 0 + for i := len(shortestPath) - 2; i >= 0; i-- { + node := shortestPath[i] + prev := shortestPath[i+1] + totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) + imageHeight := (maxLineWidth - totalSpacing) / float64(node.TotalAspect) + // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) + for ; idx <= node.Index; idx++ { + photo := photos[idx] + imageWidth := imageHeight * float64(photo.AspectRatio) + if photo.Aux { + aux := auxs[photo.Id] + font := scene.Fonts.Main.Face(imageHeight*0.6, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) + text := render.NewTextFromRect( + render.Rect{ + X: rect.X + x + imageWidth*0.01, + Y: rect.Y + y - imageHeight*0.1, + W: imageWidth, + H: imageHeight, + }, + &font, + aux.Text, + ) + scene.Texts = append(scene.Texts, text) + } else { scene.Photos = append(scene.Photos, render.Photo{ Id: photo.Id, Sprite: render.Sprite{ @@ -227,105 +275,20 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce }, }, }) - x += imageWidth + layout.ImageSpacing - // fmt.Printf("photo %d aspect %f\n", idx, photo.AspectRatio) } - x = 0 - y += imageHeight + layout.LineSpacing + x += imageWidth + layout.ImageSpacing + // fmt.Printf("photo %d x %4.0f y %4.0f aspect %f height %f\n", idx, rect.X+x, rect.Y+y, photo.AspectRatio, imageHeight) } + x = 0 + y += imageHeight + layout.LineSpacing + } - // idx := 0 - // for node := root; node != nil; { - - // bestCost := math.MaxFloat64 - // var bestNode *FlexNode - // for _, link := range node.Links { - // if link.Cost < bestCost { - // bestCost = link.Cost - // bestNode = link - // } - // } - // node = bestNode - // if node == nil { - // break - // } - // imageHeight := maxLineWidth / node.TotalAspect - // fmt.Printf("node %d cost %f total aspect %f height %f\n", node.Index, node.Cost, node.TotalAspect, imageHeight) - // for ; idx <= node.Index; idx++ { - // photo := photos[idx] - // imageWidth := imageHeight * photo.AspectRatio - // scene.Photos = append(scene.Photos, render.Photo{ - // Id: photo.Id, - // Sprite: render.Sprite{ - // Rect: render.Rect{ - // X: rect.X + x, - // Y: rect.Y + y, - // W: imageWidth, - // H: imageHeight, - // }, - // }, - // }) - // x += imageWidth - // fmt.Printf("photo %d aspect %f\n", idx, photo.AspectRatio) - // } - // x = 0 - // y += imageHeight - // } - - // index := 0 - // for info := range infos { - // photo := SectionPhoto{ - // Photo: render.Photo{ - // Id: info.Id, - // Sprite: render.Sprite{}, - // }, - // Size: image.Size{ - // X: info.Width, - // Y: info.Height, - // }, - // } - - // imageWidth := baseWidth - // // section.infos = append(section.infos, info.SourcedInfo) - - // if x+imageWidth > rect.W { - // for _, p := range row { - // scene.Photos = append(scene.Photos, p.Photo) - // } - // row = nil - // x = 0 - // y += layout.ImageHeight + layout.LineSpacing - // } - - // photo.Photo.Sprite.PlaceFitWidth( - // rect.X+x, - // rect.Y+y, - // imageWidth, - // float64(photo.Size.X), - // float64(photo.Size.Y), - // ) - - // row = append(row, photo) - - // x += imageWidth + layout.ImageSpacing - - // layoutCounter.Set(index) - // index++ - // scene.FileCount = index - // } - // for _, p := range row { - // scene.Photos = append(scene.Photos, p.Photo) - // } - // x = 0 - // y += layout.ImageHeight + layout.LineSpacing - - // rect.Y = y - - // newBounds := addSectionToScene(§ion, scene, rect, layout, source) - // layoutPlaced() + // fmt.Printf("photos %d indextonode %d stack %d\n", len(photos), len(indexToNode), q.Len()) + // fmt.Printf("photos %d stack %d\n", cap(photos), q.Cap()) - scene.Bounds.H = rect.Y + y + sceneMargin - } + rect.H = rect.Y + y + sceneMargin - layout.LineSpacing + scene.Bounds.H = rect.H + layoutPlaced() scene.RegionSource = PhotoRegionSource{ Source: source, diff --git a/internal/layout/highlights.go b/internal/layout/highlights.go new file mode 100644 index 0000000..946da31 --- /dev/null +++ b/internal/layout/highlights.go @@ -0,0 +1,479 @@ +package layout + +import ( + // . "photofield/internal" + + "context" + "log" + "math" + "photofield/internal/clip" + "photofield/internal/image" + "photofield/internal/metrics" + "photofield/internal/render" + "strings" + + "time" + + "github.com/gammazero/deque" + "github.com/golang/geo/s2" + "github.com/tdewolff/canvas" +) + +type HighlightPhoto struct { + FlexPhoto + Height float32 +} + +func longestLine(s string) int { + lines := strings.Split(s, "\r") + longest := 0 + for _, line := range lines { + if len(line) > longest { + longest = len(line) + } + } + return longest +} + +func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.Scene, source *image.Source) { + + layout.ImageSpacing = math.Min(2, 0.02*layout.ImageHeight) + layout.LineSpacing = layout.ImageSpacing + + sceneMargin := 10. + + scene.Bounds.W = layout.ViewportWidth + + rect := render.Rect{ + X: sceneMargin, + Y: sceneMargin + 64, + W: scene.Bounds.W - sceneMargin*2, + H: 0, + } + + scene.Solids = make([]render.Solid, 0) + scene.Texts = make([]render.Text, 0) + + layoutPlaced := metrics.Elapsed("layout placing") + + // row := make([]SectionPhoto, 0) + + // x := 0. + // y := 0. + + idealHeight := math.Min(layout.ImageHeight, layout.ViewportHeight*0.9) + minHeightFrac := 0.05 + auxHeight := math.Max(80, idealHeight) + minAuxHeight := auxHeight * 0.8 + simMin := 0.6 + // simPow := 3.3 + // simPow := 0.7 + // simPow := 0.3 + // simPow := 1.8 + simPow := 1.5 + // simPow := 0.5 + + // baseWidth := layout.ViewportWidth * 0.29 + + // Gather all photos + scene.Photos = scene.Photos[:0] + photos := make([]HighlightPhoto, 0) + var prevEmb []float32 + var prevInvNorm float32 + var prevLoc s2.LatLng + var prevLocTime time.Time + var prevLocation string + var prevAuxTime time.Time + layoutCounter := metrics.Counter{ + Name: "layout", + Interval: 1 * time.Second, + } + auxs := make([]FlexAux, 0) + for info := range infos { + + if source.Geo.Available() { + photoTime := info.DateTime + lastLocCheck := prevLocTime.Sub(photoTime) + if lastLocCheck < 0 { + lastLocCheck = -lastLocCheck + } + queryLocation := prevLocTime.IsZero() || lastLocCheck > 15*time.Minute + // fmt.Printf("lastLocTime %v photoTime %v lastLocCheck %v queryLocation %v\n", lastLocTime, photoTime, lastLocCheck, queryLocation) + if queryLocation && image.IsValidLatLng(info.LatLng) { + prevLocTime = photoTime + dist := image.AngleToKm(prevLoc.Distance(info.LatLng)) + if dist > 1 { + location, err := source.Geo.ReverseGeocode(context.TODO(), info.LatLng) + if err == nil && location != prevLocation { + prevLocation = location + text := "" + if prevAuxTime.Year() != photoTime.Year() { + text += photoTime.Format("2006\r") + } + if prevAuxTime.YearDay() != photoTime.YearDay() { + text += photoTime.Format("Jan 2\rMonday\r") + } + prevAuxTime = photoTime + text += location + aux := FlexAux{ + Text: text, + } + auxs = append(auxs, aux) + photos = append(photos, HighlightPhoto{ + FlexPhoto: FlexPhoto{ + Id: image.ImageId(len(auxs) - 1), + AspectRatio: 0.2 + float32(longestLine(text))/10, + Aux: true, + }, + Height: float32(auxHeight), + }) + } + prevLoc = info.LatLng + } + } + } + + similarity := float32(0.) + emb := info.Embedding.Float32() + invnorm := info.Embedding.InvNormFloat32() + simHeight := idealHeight + if prevEmb != nil { + dot, err := clip.DotProductFloat32Float32( + prevEmb, + emb, + ) + if err != nil { + log.Printf("dot product error: %v", err) + } + similarity = dot * prevInvNorm * invnorm + simHeight = idealHeight * math.Min(1, minHeightFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minHeightFrac)) + } + prevEmb = emb + prevInvNorm = invnorm + + photo := HighlightPhoto{ + FlexPhoto: FlexPhoto{ + Id: info.Id, + AspectRatio: float32(info.Width) / float32(info.Height), + }, + Height: float32(simHeight), + } + photos = append(photos, photo) + layoutCounter.Set(len(photos)) + } + + root := &FlexNode{ + Index: -1, + Cost: 0, + TotalAspect: 0, + } + + q := deque.New[*FlexNode](len(photos) / 4) + q.PushBack(root) + indexToNode := make(map[int]*FlexNode, len(photos)) + + maxLineWidth := rect.W + for q.Len() > 0 { + node := q.PopFront() + totalAspect := 0. + fallback := false + hasAux := false + + // fmt.Printf("queue %d\n", node.Index) + + prevHeight := photos[0].Height + + for i := node.Index + 1; i < len(photos); i++ { + photo := photos[i] + totalAspect += float64(photo.AspectRatio) + totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) + photoHeight := (maxLineWidth - totalSpacing) / totalAspect + minHeight := 0.3 * float64(photo.Height) + maxHeight := 1.7 * float64(photo.Height) + valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback + // badness := math.Abs(photoHeight - idealHeight) + badness := math.Abs(photoHeight - float64(photo.Height)) + prevDiff := 0.1 * math.Abs(float64(prevHeight-photo.Height)) + prevHeight = photo.Height + // viewportDiff := 1000. * float64(photoHeight) + viewportDiff := 1000. * math.Max(0, float64(photoHeight)-layout.ViewportHeight) + cost := badness*badness + prevDiff*prevDiff + viewportDiff*viewportDiff + 10 + // Incentivise aux items to be placed at the beginning + if i < len(photos)-1 && photos[i+1].Aux { + cost -= 1000000 + } + if hasAux && photoHeight < minAuxHeight { + auxDiff := (minAuxHeight - photoHeight) * 4 + cost += auxDiff * auxDiff + } + if photo.Aux { + hasAux = true + } + // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) + if valid { + n, ok := indexToNode[i] + totalCost := node.Cost + float32(cost) + if ok { + if n.Cost > totalCost { + n.Cost = totalCost + n.TotalAspect = float32(totalAspect) + n.Shortest = node + // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) + } + // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) + // } + } else { + n = &FlexNode{ + Index: i, + Cost: totalCost, + TotalAspect: float32(totalAspect), + Shortest: node, + } + indexToNode[i] = n + if i < len(photos)-1 { + q.PushBack(n) + } + // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) + } + // fmt.Printf(" node %d %v cost %f\n", i, ok, n.Cost) + } + if photoHeight < minHeight { + // Handle edge case where there is no other option + // but to accept a photo that would otherwise break outside of the desired size + if !fallback && i != len(photos)-1 && q.Len() == 0 { + fallback = true + for j := 0; j < 2 && i-j > node.Index; j++ { + totalAspect -= float64(photos[i-j].AspectRatio) + } + i = i - 2 + if i < node.Index+1 { + i = node.Index + 1 + } + continue + } + break + } + } + } + + // dot := "digraph NodeGraph {\n" + // dot += root.Dot() + // dot += "}" + // fmt.Println(dot) + + // Trace back the shortest path + shortestPath := make([]*FlexNode, 0) + for node := indexToNode[len(photos)-1]; node != nil; { + // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) + shortestPath = append(shortestPath, node) + node = node.Shortest + } + + // Finally, place the photos based on the shortest path breaks + x := 0. + y := 0. + idx := 0 + for i := len(shortestPath) - 2; i >= 0; i-- { + node := shortestPath[i] + prev := shortestPath[i+1] + totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) + imageHeight := (maxLineWidth - totalSpacing) / float64(node.TotalAspect) + // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) + for ; idx <= node.Index; idx++ { + photo := photos[idx] + imageWidth := imageHeight * float64(photo.AspectRatio) + + if photo.Aux { + aux := auxs[photo.Id] + size := imageHeight * 0.5 + // lines := strings.Count(aux.Text, "\r") + 1 + font := scene.Fonts.Main.Face(size, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) + // lineOffset := float64(lines-1) * size * 0.4 + padding := 2. + text := render.Text{ + Sprite: render.Sprite{ + Rect: render.Rect{ + X: rect.X + x + padding, + Y: rect.Y + y + padding, + W: imageWidth - 2*padding, + H: imageHeight - 2*padding, + }, + }, + Font: &font, + Text: aux.Text, + HAlign: canvas.Left, + VAlign: canvas.Bottom, + } + scene.Texts = append(scene.Texts, text) + } else { + scene.Photos = append(scene.Photos, render.Photo{ + Id: photo.Id, + Sprite: render.Sprite{ + Rect: render.Rect{ + X: rect.X + x, + Y: rect.Y + y, + W: imageWidth, + H: imageHeight, + }, + }, + }) + } + x += imageWidth + layout.ImageSpacing + // fmt.Printf("photo %d aspect %f height %f\n", idx, photo.AspectRatio, photo.Height) + } + x = 0 + y += imageHeight + layout.LineSpacing + } + + // fmt.Printf("photos %d indextonode %d stack %d\n", len(photos), len(indexToNode), q.Len()) + // fmt.Printf("photos %d stack %d\n", cap(photos), q.Cap()) + + rect.H = rect.Y + y + sceneMargin - layout.LineSpacing + scene.Bounds.H = rect.H + layoutPlaced() + + scene.RegionSource = PhotoRegionSource{ + Source: source, + } +} + +func LayoutHighlightsBasic(infos <-chan image.InfoEmb, layout Layout, scene *render.Scene, source *image.Source) { + + layout.ImageSpacing = 0.02 * layout.ImageHeight + layout.LineSpacing = 0.02 * layout.ImageHeight + + sceneMargin := 10. + + scene.Bounds.W = layout.ViewportWidth + + rect := render.Rect{ + X: sceneMargin, + Y: sceneMargin + 64, + W: scene.Bounds.W - sceneMargin*2, + H: 0, + } + + // section := Section{} + + scene.Solids = make([]render.Solid, 0) + scene.Texts = make([]render.Text, 0) + + layoutPlaced := metrics.Elapsed("layout placing") + layoutCounter := metrics.Counter{ + Name: "layout", + Interval: 1 * time.Second, + } + + row := make([]SectionPhoto, 0) + + x := 0. + y := 0. + + simMin := 0.5 + simPow := 3.3 + // simPow := 0.7 + maxWidthFrac := 0.49 + minWidthFrac := 0.05 + baseWidth := layout.ViewportWidth * maxWidthFrac + + var prevEmb []float32 + var prevInvNorm float32 + + scene.Photos = scene.Photos[:0] + index := 0 + for info := range infos { + photo := SectionPhoto{ + Photo: render.Photo{ + Id: info.Id, + Sprite: render.Sprite{}, + }, + Size: image.Size{ + X: info.Width, + Y: info.Height, + }, + } + // section.infos = append(section.infos, info.SourcedInfo) + + similarity := float32(0.) + emb := info.Embedding.Float32() + invnorm := info.Embedding.InvNormFloat32() + if prevEmb != nil { + dot, err := clip.DotProductFloat32Float32( + prevEmb, + emb, + ) + if err != nil { + log.Printf("dot product error: %v", err) + } + similarity = dot * prevInvNorm * invnorm + } + prevEmb = emb + prevInvNorm = invnorm + + // simWidth := baseWidth * math.Pow(math.Min(1., 1-(float64(similarity)-simMin)), simPow) + simWidth := baseWidth * math.Min(1, minWidthFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minWidthFrac)) + + // fmt.Printf("id: %6d, similarity: %f, width: %f / %f\n", info.Id, similarity, simWidth, baseWidth) + + // aspectRatio := float64(photo.Size.X) / float64(photo.Size.Y) + imageWidth := simWidth + // imageHeight := imageWidth / aspectRatio + + if x+imageWidth > rect.W { + + x = 0 + for i := range row { + photo := &row[i] + photo.Photo.Sprite.PlaceFitHeight( + rect.X+x, + rect.Y+y, + layout.ImageHeight, + float64(photo.Size.X), + float64(photo.Size.Y), + ) + x += photo.Sprite.Rect.W + layout.ImageSpacing + } + x -= layout.ImageSpacing + + scale := layoutFitRow(row, rect, layout.ImageSpacing) + + for _, p := range row { + scene.Photos = append(scene.Photos, p.Photo) + } + row = nil + x = 0 + y += layout.ImageHeight*scale + layout.LineSpacing + } + + photo.Photo.Sprite.PlaceFitWidth( + rect.X+x, + rect.Y+y, + imageWidth, + float64(photo.Size.X), + float64(photo.Size.Y), + ) + + row = append(row, photo) + + x += imageWidth + layout.ImageSpacing + + layoutCounter.Set(index) + index++ + scene.FileCount = index + } + for _, p := range row { + scene.Photos = append(scene.Photos, p.Photo) + } + x = 0 + y += layout.ImageHeight + layout.LineSpacing + + rect.Y = y + + // newBounds := addSectionToScene(§ion, scene, rect, layout, source) + layoutPlaced() + + scene.Bounds.H = rect.Y + sceneMargin + scene.RegionSource = PhotoRegionSource{ + Source: source, + } +} diff --git a/internal/render/sprite.go b/internal/render/sprite.go index 165ee62..712d72d 100644 --- a/internal/render/sprite.go +++ b/internal/render/sprite.go @@ -30,6 +30,26 @@ func (sprite *Sprite) PlaceFitHeight( } } +func (sprite *Sprite) PlaceFitWidth( + x float64, + y float64, + fitWidth float64, + contentWidth float64, + contentHeight float64, +) { + scale := fitWidth / contentWidth + if math.IsNaN(scale) || math.IsInf(scale, 0) { + scale = 1 + } + + sprite.Rect = Rect{ + X: x, + Y: y, + W: contentWidth * scale, + H: contentHeight * scale, + } +} + func (sprite *Sprite) PlaceFit( x float64, y float64, diff --git a/internal/render/text.go b/internal/render/text.go index b461a99..0714366 100644 --- a/internal/render/text.go +++ b/internal/render/text.go @@ -10,6 +10,8 @@ type Text struct { Sprite Sprite Font *canvas.FontFace Text string + HAlign canvas.TextAlign + VAlign canvas.TextAlign } func NewTextFromRect(rect Rect, font *canvas.FontFace, txt string) Text { @@ -28,7 +30,10 @@ func (text *Text) Draw(config *Render, c *canvas.Context, scales Scales) { return } - textLine := canvas.NewTextLine(*text.Font, text.Text, canvas.Left) - c.RenderText(textLine, c.View().Mul(text.Sprite.Rect.GetMatrix())) + // textLine := canvas.NewTextLine(*text.Font, text.Text, canvas.Left) + textLine := canvas.NewTextBox(*text.Font, text.Text, text.Sprite.Rect.W, text.Sprite.Rect.H, text.HAlign, text.VAlign, 0, 0) + rect := text.Sprite.Rect + rect.Y -= rect.H + c.RenderText(textLine, c.View().Mul(rect.GetMatrix())) } } diff --git a/internal/scene/sceneSource.go b/internal/scene/sceneSource.go index fd975d3..ddcfac6 100644 --- a/internal/scene/sceneSource.go +++ b/internal/scene/sceneSource.go @@ -111,7 +111,14 @@ func (source *SceneSource) loadScene(config SceneConfig, imageSource *image.Sour searchDone() } - if scene.SearchEmbedding != nil { + if config.Layout.Type == layout.Highlights { + infos := imageSource.ListInfosEmb(config.Collection.Dirs, image.ListOptions{ + OrderBy: image.ListOrder(config.Layout.Order), + Limit: config.Collection.Limit, + }) + + layout.LayoutHighlights(infos, config.Layout, &scene, imageSource) + } else if scene.SearchEmbedding != nil { // Similarity order infos := config.Collection.GetSimilar(imageSource, scene.SearchEmbedding, image.ListOptions{ Limit: config.Collection.Limit, @@ -144,6 +151,8 @@ func (source *SceneSource) loadScene(config SceneConfig, imageSource *image.Sour layout.LayoutMap(infos, config.Layout, &scene, imageSource) case layout.Strip: layout.LayoutStrip(infos, config.Layout, &scene, imageSource) + case layout.Flex: + layout.LayoutFlex(infos, config.Layout, &scene, imageSource) default: layout.LayoutAlbum(infos, config.Layout, &scene, imageSource) } diff --git a/main.go b/main.go index 7bfd3dd..6d1b925 100644 --- a/main.go +++ b/main.go @@ -1237,7 +1237,7 @@ func applyConfig(appConfig *AppConfig) { log.Printf("%v", globalGeo.String()) } - imageSource = image.NewSource(appConfig.Media, migrations, migrationsThumbs, nil) + imageSource = image.NewSource(appConfig.Media, migrations, migrationsThumbs, globalGeo) imageSource.HandleDirUpdates(invalidateDirs) if tileRequestConfig.Concurrency > 0 { log.Printf("request concurrency %v", tileRequestConfig.Concurrency) diff --git a/ui/src/components/DisplaySettings.vue b/ui/src/components/DisplaySettings.vue index 675c504..40cbc29 100644 --- a/ui/src/components/DisplaySettings.vue +++ b/ui/src/components/DisplaySettings.vue @@ -66,6 +66,8 @@ const layoutOptions = ref([ { label: "Timeline", value: "TIMELINE" }, { label: "Wall", value: "WALL" }, { label: "Map", value: "MAP" }, + { label: "Highlights", value: "HIGHLIGHTS" }, + { label: "Flex", value: "FLEX" }, ]); const extra = ref(false); From 7ea675fc8000f5d72f461a5c3a7ac6fb0d5a2d64 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Fri, 10 May 2024 21:48:14 +0200 Subject: [PATCH 3/8] wip --- internal/layout/dag/dag.go | 77 +++++ internal/layout/flex.go | 128 +++++--- internal/layout/highlights.go | 589 +++++++++++++++++----------------- 3 files changed, 449 insertions(+), 345 deletions(-) create mode 100644 internal/layout/dag/dag.go diff --git a/internal/layout/dag/dag.go b/internal/layout/dag/dag.go new file mode 100644 index 0000000..5284d87 --- /dev/null +++ b/internal/layout/dag/dag.go @@ -0,0 +1,77 @@ +package dag + +import ( + "photofield/internal/image" +) + +type Id = image.ImageId + +type Photo struct { + Id Id + AspectRatio float32 + Aux bool +} + +type Aux struct { + Text string +} + +type Index int + +type Node struct { + Index Index + ShortestParent Index + Cost float32 + TotalAspect float32 +} + +// type Graph[T Item] struct { +// items []Item +// q *deque.Deque[Index] +// nodes map[Index]Node +// } + +// func New[T Item](items []Item) *Graph[T] { +// g := &Graph[T]{ +// items: items, +// q: deque.New[Index](len(items) / 4), +// nodes: make(map[Index]Node, len(items)), +// } +// g.nodes[-1] = Node{ +// ItemIndex: -1, +// Cost: 0, +// TotalAspect: 0, +// } +// g.q.PushBack(0) +// return g +// } + +// func (g *Graph[T]) Next() Node { +// idx := g.q.PopFront() +// return g.nodes[idx] +// } + +// func (g *Graph[T]) Add(i Index, cost float32) { +// n, ok := g.nodes[i] +// if ok { +// if n.Cost > cost { +// n.Cost = cost +// n.ShortestParent = i +// // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) +// } +// // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) +// // } +// } else { +// n = &FlexNode{ +// Index: i, +// Cost: totalCost, +// TotalAspect: float32(totalAspect), +// Shortest: node, +// } +// indexToNode[i] = n +// if i < len(photos)-1 { +// q.PushBack(n) +// } +// // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) +// } +// } diff --git a/internal/layout/flex.go b/internal/layout/flex.go index 60c35a1..d5102de 100644 --- a/internal/layout/flex.go +++ b/internal/layout/flex.go @@ -26,37 +26,37 @@ type FlexAux struct { } type FlexNode struct { - Index int - Cost float32 - TotalAspect float32 - Shortest *FlexNode + Index int + Cost float32 + TotalAspect float32 + ShortestParent int } -func (n *FlexNode) Dot() string { - // dot := "" - - stack := []*FlexNode{n} - visited := make(map[int]bool) - dot := "" - for len(stack) > 0 { - node := stack[0] - stack = stack[1:] - if visited[node.Index] { - continue - } - visited[node.Index] = true - // dot += fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\\nTotalAspect: %.2f\"];\n", node.Index, node.Index, node.Cost, node.ImageHeight, node.TotalAspect) - // for _, link := range node.Links { - // attr := "" - // if link.Shortest == node { - // attr = " [penwidth=3]" - // } - // dot += fmt.Sprintf("\t%d -> %d%s;\n", node.Index, link.Index, attr) - // stack = append(stack, link) - // } - } - return dot -} +// func (n *FlexNode) Dot() string { +// // dot := "" + +// stack := []*FlexNode{n} +// visited := make(map[int]bool) +// dot := "" +// for len(stack) > 0 { +// node := stack[0] +// stack = stack[1:] +// if visited[node.Index] { +// continue +// } +// visited[node.Index] = true +// // dot += fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\\nTotalAspect: %.2f\"];\n", node.Index, node.Index, node.Cost, node.ImageHeight, node.TotalAspect) +// // for _, link := range node.Links { +// // attr := "" +// // if link.Shortest == node { +// // attr = " [penwidth=3]" +// // } +// // dot += fmt.Sprintf("\t%d -> %d%s;\n", node.Index, link.Index, attr) +// // stack = append(stack, link) +// // } +// } +// return dot +// } func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Scene, source *image.Source) { @@ -92,6 +92,7 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce scene.Photos = scene.Photos[:0] photos := make([]FlexPhoto, 0) + // photos2 := make([]dag.Item, 0) layoutCounter := metrics.Counter{ Name: "layout", @@ -139,23 +140,51 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce AspectRatio: float32(info.Width) / float32(info.Height), } photos = append(photos, photo) + // photos2 = append(photos2, dag.Item{ + // Id: info.Id, + // AspectRatio: float32(info.Width) / float32(info.Height), + // }) layoutCounter.Set(len(photos)) } - root := &FlexNode{ - Index: -1, - Cost: 0, - TotalAspect: 0, + // root := & + + q := deque.New[int](len(photos) / 4) + q.PushBack(-1) + indexToNode := make(map[int]FlexNode, len(photos)) + indexToNode[-1] = FlexNode{ + Index: -1, + Cost: 0, + TotalAspect: 0, + ShortestParent: -1, } - q := deque.New[*FlexNode](len(photos) / 4) - q.PushBack(root) - indexToNode := make(map[int]*FlexNode, len(photos)) + // g := dag.New(photos2) maxLineWidth := rect.W + // for node := g.Next(); node != nil; node = g.Next() { + // totalAspect := 0. + // fallback := false + + // for i := node.ItemIndex + 1; i < len(photos2); i++ { + // photo := photos2[i] + // totalAspect += float64(photo.AspectRatio) + // totalSpacing := layout.ImageSpacing * float64(i-1-node.ItemIndex) + // photoHeight := (maxLineWidth - totalSpacing) / totalAspect + // valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback + // badness := math.Abs(photoHeight - idealHeight) + // cost := badness*badness + 10 + // if i < len(photos2)-1 && photos2[i+1].Aux { + // cost *= 0.1 + // } + + // if valid { + // n := g.Add(i, node.Cost + float32(cost)) + for q.Len() > 0 { - node := q.PopFront() + // node.Index := q.PopFront() + node := indexToNode[q.PopFront()] totalAspect := 0. fallback := false @@ -186,21 +215,22 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce if n.Cost > totalCost { n.Cost = totalCost n.TotalAspect = float32(totalAspect) - n.Shortest = node + n.ShortestParent = node.Index // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) } + indexToNode[i] = n // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) // } } else { - n = &FlexNode{ - Index: i, - Cost: totalCost, - TotalAspect: float32(totalAspect), - Shortest: node, + n = FlexNode{ + Index: i, + Cost: totalCost, + TotalAspect: float32(totalAspect), + ShortestParent: node.Index, } indexToNode[i] = n if i < len(photos)-1 { - q.PushBack(n) + q.PushBack(i) } // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) } @@ -229,11 +259,13 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce // fmt.Println(dot) // Trace back the shortest path - shortestPath := make([]*FlexNode, 0) - for node := indexToNode[len(photos)-1]; node != nil; { - // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) + shortestPath := make([]FlexNode, 0) + + for parent := len(photos) - 1; parent > 0; { + node := indexToNode[parent] + // fmt.Printf("%d node %d cost %f\n", parent, node.Index, node.Cost) shortestPath = append(shortestPath, node) - node = node.Shortest + parent = node.ShortestParent } // Finally, place the photos based on the shortest path breaks diff --git a/internal/layout/highlights.go b/internal/layout/highlights.go index 946da31..7fd3d0c 100644 --- a/internal/layout/highlights.go +++ b/internal/layout/highlights.go @@ -3,7 +3,6 @@ package layout import ( // . "photofield/internal" - "context" "log" "math" "photofield/internal/clip" @@ -13,10 +12,6 @@ import ( "strings" "time" - - "github.com/gammazero/deque" - "github.com/golang/geo/s2" - "github.com/tdewolff/canvas" ) type HighlightPhoto struct { @@ -37,300 +32,300 @@ func longestLine(s string) int { func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.Scene, source *image.Source) { - layout.ImageSpacing = math.Min(2, 0.02*layout.ImageHeight) - layout.LineSpacing = layout.ImageSpacing - - sceneMargin := 10. - - scene.Bounds.W = layout.ViewportWidth - - rect := render.Rect{ - X: sceneMargin, - Y: sceneMargin + 64, - W: scene.Bounds.W - sceneMargin*2, - H: 0, - } - - scene.Solids = make([]render.Solid, 0) - scene.Texts = make([]render.Text, 0) - - layoutPlaced := metrics.Elapsed("layout placing") - - // row := make([]SectionPhoto, 0) - + // layout.ImageSpacing = math.Min(2, 0.02*layout.ImageHeight) + // layout.LineSpacing = layout.ImageSpacing + + // sceneMargin := 10. + + // scene.Bounds.W = layout.ViewportWidth + + // rect := render.Rect{ + // X: sceneMargin, + // Y: sceneMargin + 64, + // W: scene.Bounds.W - sceneMargin*2, + // H: 0, + // } + + // scene.Solids = make([]render.Solid, 0) + // scene.Texts = make([]render.Text, 0) + + // layoutPlaced := metrics.Elapsed("layout placing") + + // // row := make([]SectionPhoto, 0) + + // // x := 0. + // // y := 0. + + // idealHeight := math.Min(layout.ImageHeight, layout.ViewportHeight*0.9) + // minHeightFrac := 0.05 + // auxHeight := math.Max(80, idealHeight) + // minAuxHeight := auxHeight * 0.8 + // simMin := 0.6 + // // simPow := 3.3 + // // simPow := 0.7 + // // simPow := 0.3 + // // simPow := 1.8 + // simPow := 1.5 + // // simPow := 0.5 + + // // baseWidth := layout.ViewportWidth * 0.29 + + // // Gather all photos + // scene.Photos = scene.Photos[:0] + // photos := make([]HighlightPhoto, 0) + // var prevEmb []float32 + // var prevInvNorm float32 + // var prevLoc s2.LatLng + // var prevLocTime time.Time + // var prevLocation string + // var prevAuxTime time.Time + // layoutCounter := metrics.Counter{ + // Name: "layout", + // Interval: 1 * time.Second, + // } + // auxs := make([]FlexAux, 0) + // for info := range infos { + + // if source.Geo.Available() { + // photoTime := info.DateTime + // lastLocCheck := prevLocTime.Sub(photoTime) + // if lastLocCheck < 0 { + // lastLocCheck = -lastLocCheck + // } + // queryLocation := prevLocTime.IsZero() || lastLocCheck > 15*time.Minute + // // fmt.Printf("lastLocTime %v photoTime %v lastLocCheck %v queryLocation %v\n", lastLocTime, photoTime, lastLocCheck, queryLocation) + // if queryLocation && image.IsValidLatLng(info.LatLng) { + // prevLocTime = photoTime + // dist := image.AngleToKm(prevLoc.Distance(info.LatLng)) + // if dist > 1 { + // location, err := source.Geo.ReverseGeocode(context.TODO(), info.LatLng) + // if err == nil && location != prevLocation { + // prevLocation = location + // text := "" + // if prevAuxTime.Year() != photoTime.Year() { + // text += photoTime.Format("2006\r") + // } + // if prevAuxTime.YearDay() != photoTime.YearDay() { + // text += photoTime.Format("Jan 2\rMonday\r") + // } + // prevAuxTime = photoTime + // text += location + // aux := FlexAux{ + // Text: text, + // } + // auxs = append(auxs, aux) + // photos = append(photos, HighlightPhoto{ + // FlexPhoto: FlexPhoto{ + // Id: image.ImageId(len(auxs) - 1), + // AspectRatio: 0.2 + float32(longestLine(text))/10, + // Aux: true, + // }, + // Height: float32(auxHeight), + // }) + // } + // prevLoc = info.LatLng + // } + // } + // } + + // similarity := float32(0.) + // emb := info.Embedding.Float32() + // invnorm := info.Embedding.InvNormFloat32() + // simHeight := idealHeight + // if prevEmb != nil { + // dot, err := clip.DotProductFloat32Float32( + // prevEmb, + // emb, + // ) + // if err != nil { + // log.Printf("dot product error: %v", err) + // } + // similarity = dot * prevInvNorm * invnorm + // simHeight = idealHeight * math.Min(1, minHeightFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minHeightFrac)) + // } + // prevEmb = emb + // prevInvNorm = invnorm + + // photo := HighlightPhoto{ + // FlexPhoto: FlexPhoto{ + // Id: info.Id, + // AspectRatio: float32(info.Width) / float32(info.Height), + // }, + // Height: float32(simHeight), + // } + // photos = append(photos, photo) + // layoutCounter.Set(len(photos)) + // } + + // root := &FlexNode{ + // Index: -1, + // Cost: 0, + // TotalAspect: 0, + // } + + // q := deque.New[*FlexNode](len(photos) / 4) + // q.PushBack(root) + // indexToNode := make(map[int]*FlexNode, len(photos)) + + // maxLineWidth := rect.W + // for q.Len() > 0 { + // node := q.PopFront() + // totalAspect := 0. + // fallback := false + // hasAux := false + + // // fmt.Printf("queue %d\n", node.Index) + + // prevHeight := photos[0].Height + + // for i := node.Index + 1; i < len(photos); i++ { + // photo := photos[i] + // totalAspect += float64(photo.AspectRatio) + // totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) + // photoHeight := (maxLineWidth - totalSpacing) / totalAspect + // minHeight := 0.3 * float64(photo.Height) + // maxHeight := 1.7 * float64(photo.Height) + // valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback + // // badness := math.Abs(photoHeight - idealHeight) + // badness := math.Abs(photoHeight - float64(photo.Height)) + // prevDiff := 0.1 * math.Abs(float64(prevHeight-photo.Height)) + // prevHeight = photo.Height + // // viewportDiff := 1000. * float64(photoHeight) + // viewportDiff := 1000. * math.Max(0, float64(photoHeight)-layout.ViewportHeight) + // cost := badness*badness + prevDiff*prevDiff + viewportDiff*viewportDiff + 10 + // // Incentivise aux items to be placed at the beginning + // if i < len(photos)-1 && photos[i+1].Aux { + // cost -= 1000000 + // } + // if hasAux && photoHeight < minAuxHeight { + // auxDiff := (minAuxHeight - photoHeight) * 4 + // cost += auxDiff * auxDiff + // } + // if photo.Aux { + // hasAux = true + // } + // // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) + // if valid { + // n, ok := indexToNode[i] + // totalCost := node.Cost + float32(cost) + // if ok { + // if n.Cost > totalCost { + // n.Cost = totalCost + // n.TotalAspect = float32(totalAspect) + // n.ShortestParent = node + // // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) + // } + // // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) + // // } + // } else { + // n = &FlexNode{ + // Index: i, + // Cost: totalCost, + // TotalAspect: float32(totalAspect), + // ShortestParent: node, + // } + // indexToNode[i] = n + // if i < len(photos)-1 { + // q.PushBack(n) + // } + // // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) + // } + // // fmt.Printf(" node %d %v cost %f\n", i, ok, n.Cost) + // } + // if photoHeight < minHeight { + // // Handle edge case where there is no other option + // // but to accept a photo that would otherwise break outside of the desired size + // if !fallback && i != len(photos)-1 && q.Len() == 0 { + // fallback = true + // for j := 0; j < 2 && i-j > node.Index; j++ { + // totalAspect -= float64(photos[i-j].AspectRatio) + // } + // i = i - 2 + // if i < node.Index+1 { + // i = node.Index + 1 + // } + // continue + // } + // break + // } + // } + // } + + // // dot := "digraph NodeGraph {\n" + // // dot += root.Dot() + // // dot += "}" + // // fmt.Println(dot) + + // // Trace back the shortest path + // shortestPath := make([]*FlexNode, 0) + // for node := indexToNode[len(photos)-1]; node != nil; { + // // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) + // shortestPath = append(shortestPath, node) + // node = node.ShortestParent + // } + + // // Finally, place the photos based on the shortest path breaks // x := 0. // y := 0. - - idealHeight := math.Min(layout.ImageHeight, layout.ViewportHeight*0.9) - minHeightFrac := 0.05 - auxHeight := math.Max(80, idealHeight) - minAuxHeight := auxHeight * 0.8 - simMin := 0.6 - // simPow := 3.3 - // simPow := 0.7 - // simPow := 0.3 - // simPow := 1.8 - simPow := 1.5 - // simPow := 0.5 - - // baseWidth := layout.ViewportWidth * 0.29 - - // Gather all photos - scene.Photos = scene.Photos[:0] - photos := make([]HighlightPhoto, 0) - var prevEmb []float32 - var prevInvNorm float32 - var prevLoc s2.LatLng - var prevLocTime time.Time - var prevLocation string - var prevAuxTime time.Time - layoutCounter := metrics.Counter{ - Name: "layout", - Interval: 1 * time.Second, - } - auxs := make([]FlexAux, 0) - for info := range infos { - - if source.Geo.Available() { - photoTime := info.DateTime - lastLocCheck := prevLocTime.Sub(photoTime) - if lastLocCheck < 0 { - lastLocCheck = -lastLocCheck - } - queryLocation := prevLocTime.IsZero() || lastLocCheck > 15*time.Minute - // fmt.Printf("lastLocTime %v photoTime %v lastLocCheck %v queryLocation %v\n", lastLocTime, photoTime, lastLocCheck, queryLocation) - if queryLocation && image.IsValidLatLng(info.LatLng) { - prevLocTime = photoTime - dist := image.AngleToKm(prevLoc.Distance(info.LatLng)) - if dist > 1 { - location, err := source.Geo.ReverseGeocode(context.TODO(), info.LatLng) - if err == nil && location != prevLocation { - prevLocation = location - text := "" - if prevAuxTime.Year() != photoTime.Year() { - text += photoTime.Format("2006\r") - } - if prevAuxTime.YearDay() != photoTime.YearDay() { - text += photoTime.Format("Jan 2\rMonday\r") - } - prevAuxTime = photoTime - text += location - aux := FlexAux{ - Text: text, - } - auxs = append(auxs, aux) - photos = append(photos, HighlightPhoto{ - FlexPhoto: FlexPhoto{ - Id: image.ImageId(len(auxs) - 1), - AspectRatio: 0.2 + float32(longestLine(text))/10, - Aux: true, - }, - Height: float32(auxHeight), - }) - } - prevLoc = info.LatLng - } - } - } - - similarity := float32(0.) - emb := info.Embedding.Float32() - invnorm := info.Embedding.InvNormFloat32() - simHeight := idealHeight - if prevEmb != nil { - dot, err := clip.DotProductFloat32Float32( - prevEmb, - emb, - ) - if err != nil { - log.Printf("dot product error: %v", err) - } - similarity = dot * prevInvNorm * invnorm - simHeight = idealHeight * math.Min(1, minHeightFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minHeightFrac)) - } - prevEmb = emb - prevInvNorm = invnorm - - photo := HighlightPhoto{ - FlexPhoto: FlexPhoto{ - Id: info.Id, - AspectRatio: float32(info.Width) / float32(info.Height), - }, - Height: float32(simHeight), - } - photos = append(photos, photo) - layoutCounter.Set(len(photos)) - } - - root := &FlexNode{ - Index: -1, - Cost: 0, - TotalAspect: 0, - } - - q := deque.New[*FlexNode](len(photos) / 4) - q.PushBack(root) - indexToNode := make(map[int]*FlexNode, len(photos)) - - maxLineWidth := rect.W - for q.Len() > 0 { - node := q.PopFront() - totalAspect := 0. - fallback := false - hasAux := false - - // fmt.Printf("queue %d\n", node.Index) - - prevHeight := photos[0].Height - - for i := node.Index + 1; i < len(photos); i++ { - photo := photos[i] - totalAspect += float64(photo.AspectRatio) - totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) - photoHeight := (maxLineWidth - totalSpacing) / totalAspect - minHeight := 0.3 * float64(photo.Height) - maxHeight := 1.7 * float64(photo.Height) - valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback - // badness := math.Abs(photoHeight - idealHeight) - badness := math.Abs(photoHeight - float64(photo.Height)) - prevDiff := 0.1 * math.Abs(float64(prevHeight-photo.Height)) - prevHeight = photo.Height - // viewportDiff := 1000. * float64(photoHeight) - viewportDiff := 1000. * math.Max(0, float64(photoHeight)-layout.ViewportHeight) - cost := badness*badness + prevDiff*prevDiff + viewportDiff*viewportDiff + 10 - // Incentivise aux items to be placed at the beginning - if i < len(photos)-1 && photos[i+1].Aux { - cost -= 1000000 - } - if hasAux && photoHeight < minAuxHeight { - auxDiff := (minAuxHeight - photoHeight) * 4 - cost += auxDiff * auxDiff - } - if photo.Aux { - hasAux = true - } - // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) - if valid { - n, ok := indexToNode[i] - totalCost := node.Cost + float32(cost) - if ok { - if n.Cost > totalCost { - n.Cost = totalCost - n.TotalAspect = float32(totalAspect) - n.Shortest = node - // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) - } - // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) - // } - } else { - n = &FlexNode{ - Index: i, - Cost: totalCost, - TotalAspect: float32(totalAspect), - Shortest: node, - } - indexToNode[i] = n - if i < len(photos)-1 { - q.PushBack(n) - } - // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) - } - // fmt.Printf(" node %d %v cost %f\n", i, ok, n.Cost) - } - if photoHeight < minHeight { - // Handle edge case where there is no other option - // but to accept a photo that would otherwise break outside of the desired size - if !fallback && i != len(photos)-1 && q.Len() == 0 { - fallback = true - for j := 0; j < 2 && i-j > node.Index; j++ { - totalAspect -= float64(photos[i-j].AspectRatio) - } - i = i - 2 - if i < node.Index+1 { - i = node.Index + 1 - } - continue - } - break - } - } - } - - // dot := "digraph NodeGraph {\n" - // dot += root.Dot() - // dot += "}" - // fmt.Println(dot) - - // Trace back the shortest path - shortestPath := make([]*FlexNode, 0) - for node := indexToNode[len(photos)-1]; node != nil; { - // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) - shortestPath = append(shortestPath, node) - node = node.Shortest - } - - // Finally, place the photos based on the shortest path breaks - x := 0. - y := 0. - idx := 0 - for i := len(shortestPath) - 2; i >= 0; i-- { - node := shortestPath[i] - prev := shortestPath[i+1] - totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) - imageHeight := (maxLineWidth - totalSpacing) / float64(node.TotalAspect) - // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) - for ; idx <= node.Index; idx++ { - photo := photos[idx] - imageWidth := imageHeight * float64(photo.AspectRatio) - - if photo.Aux { - aux := auxs[photo.Id] - size := imageHeight * 0.5 - // lines := strings.Count(aux.Text, "\r") + 1 - font := scene.Fonts.Main.Face(size, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) - // lineOffset := float64(lines-1) * size * 0.4 - padding := 2. - text := render.Text{ - Sprite: render.Sprite{ - Rect: render.Rect{ - X: rect.X + x + padding, - Y: rect.Y + y + padding, - W: imageWidth - 2*padding, - H: imageHeight - 2*padding, - }, - }, - Font: &font, - Text: aux.Text, - HAlign: canvas.Left, - VAlign: canvas.Bottom, - } - scene.Texts = append(scene.Texts, text) - } else { - scene.Photos = append(scene.Photos, render.Photo{ - Id: photo.Id, - Sprite: render.Sprite{ - Rect: render.Rect{ - X: rect.X + x, - Y: rect.Y + y, - W: imageWidth, - H: imageHeight, - }, - }, - }) - } - x += imageWidth + layout.ImageSpacing - // fmt.Printf("photo %d aspect %f height %f\n", idx, photo.AspectRatio, photo.Height) - } - x = 0 - y += imageHeight + layout.LineSpacing - } - - // fmt.Printf("photos %d indextonode %d stack %d\n", len(photos), len(indexToNode), q.Len()) - // fmt.Printf("photos %d stack %d\n", cap(photos), q.Cap()) - - rect.H = rect.Y + y + sceneMargin - layout.LineSpacing - scene.Bounds.H = rect.H - layoutPlaced() + // idx := 0 + // for i := len(shortestPath) - 2; i >= 0; i-- { + // node := shortestPath[i] + // prev := shortestPath[i+1] + // totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) + // imageHeight := (maxLineWidth - totalSpacing) / float64(node.TotalAspect) + // // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) + // for ; idx <= node.Index; idx++ { + // photo := photos[idx] + // imageWidth := imageHeight * float64(photo.AspectRatio) + + // if photo.Aux { + // aux := auxs[photo.Id] + // size := imageHeight * 0.5 + // // lines := strings.Count(aux.Text, "\r") + 1 + // font := scene.Fonts.Main.Face(size, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) + // // lineOffset := float64(lines-1) * size * 0.4 + // padding := 2. + // text := render.Text{ + // Sprite: render.Sprite{ + // Rect: render.Rect{ + // X: rect.X + x + padding, + // Y: rect.Y + y + padding, + // W: imageWidth - 2*padding, + // H: imageHeight - 2*padding, + // }, + // }, + // Font: &font, + // Text: aux.Text, + // HAlign: canvas.Left, + // VAlign: canvas.Bottom, + // } + // scene.Texts = append(scene.Texts, text) + // } else { + // scene.Photos = append(scene.Photos, render.Photo{ + // Id: photo.Id, + // Sprite: render.Sprite{ + // Rect: render.Rect{ + // X: rect.X + x, + // Y: rect.Y + y, + // W: imageWidth, + // H: imageHeight, + // }, + // }, + // }) + // } + // x += imageWidth + layout.ImageSpacing + // // fmt.Printf("photo %d aspect %f height %f\n", idx, photo.AspectRatio, photo.Height) + // } + // x = 0 + // y += imageHeight + layout.LineSpacing + // } + + // // fmt.Printf("photos %d indextonode %d stack %d\n", len(photos), len(indexToNode), q.Len()) + // // fmt.Printf("photos %d stack %d\n", cap(photos), q.Cap()) + + // rect.H = rect.Y + y + sceneMargin - layout.LineSpacing + // scene.Bounds.H = rect.H + // layoutPlaced() scene.RegionSource = PhotoRegionSource{ Source: source, From d0f673527172786ef4c7a4682c8f0cec8eddfc06 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Fri, 10 May 2024 22:42:33 +0200 Subject: [PATCH 4/8] Simplify and cleanup flex & highlights --- internal/layout/common.go | 11 + internal/layout/dag/dag.go | 55 +--- internal/layout/flex.go | 260 ++++++---------- internal/layout/highlights.go | 572 ++++++++++++++++------------------ 4 files changed, 366 insertions(+), 532 deletions(-) diff --git a/internal/layout/common.go b/internal/layout/common.go index a16482a..2d7846f 100644 --- a/internal/layout/common.go +++ b/internal/layout/common.go @@ -103,6 +103,17 @@ type PhotoRegionData struct { // SmallestThumbnail string `json:"smallest_thumbnail"` } +func longestLine(s string) int { + lines := strings.Split(s, "\r") + longest := 0 + for _, line := range lines { + if len(line) > longest { + longest = len(line) + } + } + return longest +} + func (regionSource PhotoRegionSource) getRegionFromPhoto(id int, photo *render.Photo, scene *render.Scene, regionConfig render.RegionConfig) render.Region { source := regionSource.Source diff --git a/internal/layout/dag/dag.go b/internal/layout/dag/dag.go index 5284d87..a3922a2 100644 --- a/internal/layout/dag/dag.go +++ b/internal/layout/dag/dag.go @@ -5,6 +5,7 @@ import ( ) type Id = image.ImageId +type Index = int type Photo struct { Id Id @@ -16,62 +17,8 @@ type Aux struct { Text string } -type Index int - type Node struct { - Index Index ShortestParent Index Cost float32 TotalAspect float32 } - -// type Graph[T Item] struct { -// items []Item -// q *deque.Deque[Index] -// nodes map[Index]Node -// } - -// func New[T Item](items []Item) *Graph[T] { -// g := &Graph[T]{ -// items: items, -// q: deque.New[Index](len(items) / 4), -// nodes: make(map[Index]Node, len(items)), -// } -// g.nodes[-1] = Node{ -// ItemIndex: -1, -// Cost: 0, -// TotalAspect: 0, -// } -// g.q.PushBack(0) -// return g -// } - -// func (g *Graph[T]) Next() Node { -// idx := g.q.PopFront() -// return g.nodes[idx] -// } - -// func (g *Graph[T]) Add(i Index, cost float32) { -// n, ok := g.nodes[i] -// if ok { -// if n.Cost > cost { -// n.Cost = cost -// n.ShortestParent = i -// // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) -// } -// // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) -// // } -// } else { -// n = &FlexNode{ -// Index: i, -// Cost: totalCost, -// TotalAspect: float32(totalAspect), -// Shortest: node, -// } -// indexToNode[i] = n -// if i < len(photos)-1 { -// q.PushBack(n) -// } -// // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) -// } -// } diff --git a/internal/layout/flex.go b/internal/layout/flex.go index d5102de..2a5b453 100644 --- a/internal/layout/flex.go +++ b/internal/layout/flex.go @@ -6,6 +6,7 @@ import ( "context" "math" "photofield/internal/image" + "photofield/internal/layout/dag" "photofield/internal/metrics" "photofield/internal/render" "time" @@ -15,49 +16,6 @@ import ( "github.com/tdewolff/canvas" ) -type FlexPhoto struct { - Id image.ImageId - AspectRatio float32 - Aux bool -} - -type FlexAux struct { - Text string -} - -type FlexNode struct { - Index int - Cost float32 - TotalAspect float32 - ShortestParent int -} - -// func (n *FlexNode) Dot() string { -// // dot := "" - -// stack := []*FlexNode{n} -// visited := make(map[int]bool) -// dot := "" -// for len(stack) > 0 { -// node := stack[0] -// stack = stack[1:] -// if visited[node.Index] { -// continue -// } -// visited[node.Index] = true -// // dot += fmt.Sprintf("%d [label=\"%d\\nCost: %.0f\\nHeight: %.0f\\nTotalAspect: %.2f\"];\n", node.Index, node.Index, node.Cost, node.ImageHeight, node.TotalAspect) -// // for _, link := range node.Links { -// // attr := "" -// // if link.Shortest == node { -// // attr = " [penwidth=3]" -// // } -// // dot += fmt.Sprintf("\t%d -> %d%s;\n", node.Index, link.Index, attr) -// // stack = append(stack, link) -// // } -// } -// return dot -// } - func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Scene, source *image.Source) { layout.ImageSpacing = 0.02 * layout.ImageHeight @@ -79,170 +37,132 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce layoutPlaced := metrics.Elapsed("layout placing") - // row := make([]SectionPhoto, 0) - - // x := 0. - // y := 0. - - idealHeight := layout.ImageHeight + idealHeight := math.Min(layout.ImageHeight, layout.ViewportHeight*0.9) + auxHeight := math.Max(80, idealHeight) + minAuxHeight := auxHeight * 0.8 minHeight := 0.8 * idealHeight maxHeight := 1.2 * idealHeight - // baseWidth := layout.ViewportWidth * 0.29 - scene.Photos = scene.Photos[:0] - photos := make([]FlexPhoto, 0) - // photos2 := make([]dag.Item, 0) + photos := make([]dag.Photo, 0) layoutCounter := metrics.Counter{ Name: "layout", Interval: 1 * time.Second, } - auxs := make([]FlexAux, 0) + auxs := make([]dag.Aux, 0) // Fetch all photos - var lastLoc s2.LatLng - var lastLocTime time.Time - var lastLocation string + var prevLoc s2.LatLng + var prevLocTime time.Time + var prevLocation string + var prevAuxTime time.Time for info := range infos { if source.Geo.Available() { photoTime := info.DateTime - lastLocCheck := lastLocTime.Sub(photoTime) + lastLocCheck := prevLocTime.Sub(photoTime) if lastLocCheck < 0 { lastLocCheck = -lastLocCheck } - queryLocation := lastLocTime.IsZero() || lastLocCheck > 15*time.Minute - // fmt.Printf("lastLocTime %v photoTime %v lastLocCheck %v queryLocation %v\n", lastLocTime, photoTime, lastLocCheck, queryLocation) + queryLocation := prevLocTime.IsZero() || lastLocCheck > 15*time.Minute if queryLocation && image.IsValidLatLng(info.LatLng) { - lastLocTime = photoTime - dist := image.AngleToKm(lastLoc.Distance(info.LatLng)) + prevLocTime = photoTime + dist := image.AngleToKm(prevLoc.Distance(info.LatLng)) if dist > 1 { location, err := source.Geo.ReverseGeocode(context.TODO(), info.LatLng) - if err == nil && location != lastLocation { - lastLocation = location - aux := FlexAux{ - Text: location, + if err == nil && location != prevLocation { + prevLocation = location + text := "" + if prevAuxTime.Year() != photoTime.Year() { + text += photoTime.Format("2006\r") + } + if prevAuxTime.YearDay() != photoTime.YearDay() { + text += photoTime.Format("Jan 2\rMonday\r") + } + prevAuxTime = photoTime + text += location + aux := dag.Aux{ + Text: text, } auxs = append(auxs, aux) - photos = append(photos, FlexPhoto{ + photos = append(photos, dag.Photo{ Id: image.ImageId(len(auxs) - 1), - AspectRatio: float32(len(location)) / 5, + AspectRatio: 0.2 + float32(longestLine(text))/10, Aux: true, }) } - lastLoc = info.LatLng + prevLoc = info.LatLng } } } - photo := FlexPhoto{ + photo := dag.Photo{ Id: info.Id, AspectRatio: float32(info.Width) / float32(info.Height), } photos = append(photos, photo) - // photos2 = append(photos2, dag.Item{ - // Id: info.Id, - // AspectRatio: float32(info.Width) / float32(info.Height), - // }) layoutCounter.Set(len(photos)) } - // root := & - - q := deque.New[int](len(photos) / 4) - q.PushBack(-1) - indexToNode := make(map[int]FlexNode, len(photos)) - indexToNode[-1] = FlexNode{ - Index: -1, + // Create a directed acyclic graph to find the optimal layout + root := dag.Node{ Cost: 0, TotalAspect: 0, - ShortestParent: -1, + ShortestParent: -2, } - // g := dag.New(photos2) + q := deque.New[dag.Index](len(photos) / 4) + q.PushBack(-1) + indexToNode := make(map[dag.Index]dag.Node, len(photos)) + indexToNode[-1] = root maxLineWidth := rect.W - // for node := g.Next(); node != nil; node = g.Next() { - // totalAspect := 0. - // fallback := false - - // for i := node.ItemIndex + 1; i < len(photos2); i++ { - // photo := photos2[i] - // totalAspect += float64(photo.AspectRatio) - // totalSpacing := layout.ImageSpacing * float64(i-1-node.ItemIndex) - // photoHeight := (maxLineWidth - totalSpacing) / totalAspect - // valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback - // badness := math.Abs(photoHeight - idealHeight) - // cost := badness*badness + 10 - // if i < len(photos2)-1 && photos2[i+1].Aux { - // cost *= 0.1 - // } - - // if valid { - // n := g.Add(i, node.Cost + float32(cost)) - for q.Len() > 0 { - // node.Index := q.PopFront() - node := indexToNode[q.PopFront()] + nodeIndex := q.PopFront() + node := indexToNode[nodeIndex] totalAspect := 0. fallback := false + hasAux := false - // fmt.Printf("queue %d\n", node.Index) - for i := node.Index + 1; i < len(photos); i++ { + for i := nodeIndex + 1; i < len(photos); i++ { photo := photos[i] totalAspect += float64(photo.AspectRatio) - totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) + totalSpacing := layout.ImageSpacing * float64(i-1-nodeIndex) photoHeight := (maxLineWidth - totalSpacing) / totalAspect valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback + badness := math.Abs(photoHeight - idealHeight) cost := badness*badness + 10 if i < len(photos)-1 && photos[i+1].Aux { - cost *= 0.1 + cost -= 1000000 + } + if hasAux && photoHeight < minAuxHeight { + auxDiff := (minAuxHeight - photoHeight) * 4 + cost += auxDiff * auxDiff + } + if photo.Aux { + hasAux = true } - - // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) - - // Handle edge case where there is no other option - // but to accept a photo that would otherwise break outside of the desired size - // if i != len(photos)-1 && q.Len() == 0 { - // valid = true - // } if valid { - n, ok := indexToNode[i] totalCost := node.Cost + float32(cost) - if ok { - if n.Cost > totalCost { - n.Cost = totalCost - n.TotalAspect = float32(totalAspect) - n.ShortestParent = node.Index - // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) - } - indexToNode[i] = n - // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) - // } - } else { - n = FlexNode{ - Index: i, - Cost: totalCost, - TotalAspect: float32(totalAspect), - ShortestParent: node.Index, - } - indexToNode[i] = n - if i < len(photos)-1 { - q.PushBack(i) - } - // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) + n, ok := indexToNode[i] + if !ok || (ok && n.Cost > totalCost) { + n.Cost = totalCost + n.TotalAspect = float32(totalAspect) + n.ShortestParent = nodeIndex + } + if !ok && i < len(photos)-1 { + q.PushBack(i) } - // fmt.Printf(" node %d %v cost %f\n", i, ok, n.Cost) + indexToNode[i] = n } if photoHeight < minHeight { // Handle edge case where there is no other option // but to accept a photo that would otherwise break outside of the desired size if !fallback && i != len(photos)-1 && q.Len() == 0 { fallback = true - for j := 0; j < 2 && i > node.Index; j++ { - // fmt.Printf(" fallback %d\n", i) + for j := 0; j < 2 && i > nodeIndex; j++ { totalAspect -= float64(photos[i].AspectRatio) i-- } @@ -253,19 +173,11 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce } } - // dot := "digraph NodeGraph {\n" - // dot += root.Dot() - // dot += "}" - // fmt.Println(dot) - // Trace back the shortest path - shortestPath := make([]FlexNode, 0) - - for parent := len(photos) - 1; parent > 0; { - node := indexToNode[parent] - // fmt.Printf("%d node %d cost %f\n", parent, node.Index, node.Cost) - shortestPath = append(shortestPath, node) - parent = node.ShortestParent + shortestPath := make([]int, 0) + for nodeIndex := len(photos) - 1; nodeIndex != -2; { + shortestPath = append(shortestPath, nodeIndex) + nodeIndex = indexToNode[nodeIndex].ShortestParent } // Finally, place the photos based on the shortest path breaks @@ -273,27 +185,33 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce y := 0. idx := 0 for i := len(shortestPath) - 2; i >= 0; i-- { - node := shortestPath[i] - prev := shortestPath[i+1] - totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) + nodeIdx := shortestPath[i] + prevIdx := shortestPath[i+1] + node := indexToNode[nodeIdx] + totalSpacing := layout.ImageSpacing * float64(nodeIdx-1-prevIdx) imageHeight := (maxLineWidth - totalSpacing) / float64(node.TotalAspect) - // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) - for ; idx <= node.Index; idx++ { + for ; idx <= nodeIdx; idx++ { photo := photos[idx] imageWidth := imageHeight * float64(photo.AspectRatio) if photo.Aux { aux := auxs[photo.Id] - font := scene.Fonts.Main.Face(imageHeight*0.6, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) - text := render.NewTextFromRect( - render.Rect{ - X: rect.X + x + imageWidth*0.01, - Y: rect.Y + y - imageHeight*0.1, - W: imageWidth, - H: imageHeight, + size := imageHeight * 0.5 + font := scene.Fonts.Main.Face(size, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) + padding := 2. + text := render.Text{ + Sprite: render.Sprite{ + Rect: render.Rect{ + X: rect.X + x + padding, + Y: rect.Y + y + padding, + W: imageWidth - 2*padding, + H: imageHeight - 2*padding, + }, }, - &font, - aux.Text, - ) + Font: &font, + Text: aux.Text, + HAlign: canvas.Left, + VAlign: canvas.Bottom, + } scene.Texts = append(scene.Texts, text) } else { scene.Photos = append(scene.Photos, render.Photo{ @@ -309,15 +227,11 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce }) } x += imageWidth + layout.ImageSpacing - // fmt.Printf("photo %d x %4.0f y %4.0f aspect %f height %f\n", idx, rect.X+x, rect.Y+y, photo.AspectRatio, imageHeight) } x = 0 y += imageHeight + layout.LineSpacing } - // fmt.Printf("photos %d indextonode %d stack %d\n", len(photos), len(indexToNode), q.Len()) - // fmt.Printf("photos %d stack %d\n", cap(photos), q.Cap()) - rect.H = rect.Y + y + sceneMargin - layout.LineSpacing scene.Bounds.H = rect.H layoutPlaced() diff --git a/internal/layout/highlights.go b/internal/layout/highlights.go index 7fd3d0c..3b4db75 100644 --- a/internal/layout/highlights.go +++ b/internal/layout/highlights.go @@ -3,329 +3,291 @@ package layout import ( // . "photofield/internal" + "context" "log" "math" "photofield/internal/clip" "photofield/internal/image" + "photofield/internal/layout/dag" "photofield/internal/metrics" "photofield/internal/render" - "strings" "time" + + "github.com/gammazero/deque" + "github.com/golang/geo/s2" + "github.com/tdewolff/canvas" ) type HighlightPhoto struct { - FlexPhoto + dag.Photo Height float32 } -func longestLine(s string) int { - lines := strings.Split(s, "\r") - longest := 0 - for _, line := range lines { - if len(line) > longest { - longest = len(line) +func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.Scene, source *image.Source) { + + layout.ImageSpacing = math.Min(2, 0.02*layout.ImageHeight) + layout.LineSpacing = layout.ImageSpacing + + sceneMargin := 10. + + scene.Bounds.W = layout.ViewportWidth + + rect := render.Rect{ + X: sceneMargin, + Y: sceneMargin + 64, + W: scene.Bounds.W - sceneMargin*2, + H: 0, + } + + scene.Solids = make([]render.Solid, 0) + scene.Texts = make([]render.Text, 0) + + layoutPlaced := metrics.Elapsed("layout placing") + + idealHeight := math.Min(layout.ImageHeight, layout.ViewportHeight*0.9) + auxHeight := math.Max(80, idealHeight) + minAuxHeight := auxHeight * 0.8 + minHeightFrac := 0.05 + simMin := 0.6 + // simPow := 3.3 + // simPow := 0.7 + // simPow := 0.3 + // simPow := 1.8 + simPow := 1.5 + // simPow := 0.5 + + scene.Photos = scene.Photos[:0] + photos := make([]HighlightPhoto, 0) + + layoutCounter := metrics.Counter{ + Name: "layout", + Interval: 1 * time.Second, + } + + auxs := make([]dag.Aux, 0) + + // Fetch all photos + var prevLoc s2.LatLng + var prevLocTime time.Time + var prevLocation string + var prevAuxTime time.Time + var prevEmb []float32 + var prevInvNorm float32 + + for info := range infos { + if source.Geo.Available() { + photoTime := info.DateTime + lastLocCheck := prevLocTime.Sub(photoTime) + if lastLocCheck < 0 { + lastLocCheck = -lastLocCheck + } + queryLocation := prevLocTime.IsZero() || lastLocCheck > 15*time.Minute + if queryLocation && image.IsValidLatLng(info.LatLng) { + prevLocTime = photoTime + dist := image.AngleToKm(prevLoc.Distance(info.LatLng)) + if dist > 1 { + location, err := source.Geo.ReverseGeocode(context.TODO(), info.LatLng) + if err == nil && location != prevLocation { + prevLocation = location + text := "" + if prevAuxTime.Year() != photoTime.Year() { + text += photoTime.Format("2006\r") + } + if prevAuxTime.YearDay() != photoTime.YearDay() { + text += photoTime.Format("Jan 2\rMonday\r") + } + prevAuxTime = photoTime + text += location + aux := dag.Aux{ + Text: text, + } + auxs = append(auxs, aux) + photos = append(photos, HighlightPhoto{ + Photo: dag.Photo{ + Id: image.ImageId(len(auxs) - 1), + AspectRatio: 0.2 + float32(longestLine(text))/10, + Aux: true, + }, + Height: float32(auxHeight), + }) + } + prevLoc = info.LatLng + } + } } + + similarity := float32(0.) + emb := info.Embedding.Float32() + invnorm := info.Embedding.InvNormFloat32() + simHeight := idealHeight + if prevEmb != nil { + dot, err := clip.DotProductFloat32Float32( + prevEmb, + emb, + ) + if err != nil { + log.Printf("dot product error: %v", err) + } + similarity = dot * prevInvNorm * invnorm + simHeight = idealHeight * math.Min(1, minHeightFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minHeightFrac)) + } + prevEmb = emb + prevInvNorm = invnorm + + photo := HighlightPhoto{ + Photo: dag.Photo{ + Id: info.Id, + AspectRatio: float32(info.Width) / float32(info.Height), + }, + Height: float32(simHeight), + } + photos = append(photos, photo) + layoutCounter.Set(len(photos)) } - return longest -} -func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.Scene, source *image.Source) { + // Create a directed acyclic graph to find the optimal layout + root := dag.Node{ + Cost: 0, + TotalAspect: 0, + ShortestParent: -2, + } + + q := deque.New[dag.Index](len(photos) / 4) + q.PushBack(-1) + indexToNode := make(map[dag.Index]dag.Node, len(photos)) + indexToNode[-1] = root + + maxLineWidth := rect.W + + for q.Len() > 0 { + nodeIndex := q.PopFront() + node := indexToNode[nodeIndex] + totalAspect := 0. + fallback := false + hasAux := false + + // fmt.Printf("queue %d\n", node.Index) + + prevHeight := photos[0].Height + + for i := nodeIndex + 1; i < len(photos); i++ { + photo := photos[i] + totalAspect += float64(photo.AspectRatio) + totalSpacing := layout.ImageSpacing * float64(i-1-nodeIndex) + photoHeight := (maxLineWidth - totalSpacing) / totalAspect + minHeight := 0.3 * float64(photo.Height) + maxHeight := 1.7 * float64(photo.Height) + valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback + // badness := math.Abs(photoHeight - idealHeight) + badness := math.Abs(photoHeight - float64(photo.Height)) + prevDiff := 0.1 * math.Abs(float64(prevHeight-photo.Height)) + prevHeight = photo.Height + // viewportDiff := 1000. * float64(photoHeight) + viewportDiff := 1000. * math.Max(0, float64(photoHeight)-layout.ViewportHeight) + cost := badness*badness + prevDiff*prevDiff + viewportDiff*viewportDiff + 10 + // Incentivise aux items to be placed at the beginning + if i < len(photos)-1 && photos[i+1].Aux { + cost -= 1000000 + } + if hasAux && photoHeight < minAuxHeight { + auxDiff := (minAuxHeight - photoHeight) * 4 + cost += auxDiff * auxDiff + } + if photo.Aux { + hasAux = true + } + if valid { + totalCost := node.Cost + float32(cost) + n, ok := indexToNode[i] + if !ok || (ok && n.Cost > totalCost) { + n.Cost = totalCost + n.TotalAspect = float32(totalAspect) + n.ShortestParent = nodeIndex + } + if !ok && i < len(photos)-1 { + q.PushBack(i) + } + indexToNode[i] = n + } + if photoHeight < minHeight { + // Handle edge case where there is no other option + // but to accept a photo that would otherwise break outside of the desired size + if !fallback && i != len(photos)-1 && q.Len() == 0 { + fallback = true + for j := 0; j < 2 && i > nodeIndex; j++ { + totalAspect -= float64(photos[i].AspectRatio) + i-- + } + continue + } + break + } + } + } - // layout.ImageSpacing = math.Min(2, 0.02*layout.ImageHeight) - // layout.LineSpacing = layout.ImageSpacing - - // sceneMargin := 10. - - // scene.Bounds.W = layout.ViewportWidth - - // rect := render.Rect{ - // X: sceneMargin, - // Y: sceneMargin + 64, - // W: scene.Bounds.W - sceneMargin*2, - // H: 0, - // } - - // scene.Solids = make([]render.Solid, 0) - // scene.Texts = make([]render.Text, 0) - - // layoutPlaced := metrics.Elapsed("layout placing") - - // // row := make([]SectionPhoto, 0) - - // // x := 0. - // // y := 0. - - // idealHeight := math.Min(layout.ImageHeight, layout.ViewportHeight*0.9) - // minHeightFrac := 0.05 - // auxHeight := math.Max(80, idealHeight) - // minAuxHeight := auxHeight * 0.8 - // simMin := 0.6 - // // simPow := 3.3 - // // simPow := 0.7 - // // simPow := 0.3 - // // simPow := 1.8 - // simPow := 1.5 - // // simPow := 0.5 - - // // baseWidth := layout.ViewportWidth * 0.29 - - // // Gather all photos - // scene.Photos = scene.Photos[:0] - // photos := make([]HighlightPhoto, 0) - // var prevEmb []float32 - // var prevInvNorm float32 - // var prevLoc s2.LatLng - // var prevLocTime time.Time - // var prevLocation string - // var prevAuxTime time.Time - // layoutCounter := metrics.Counter{ - // Name: "layout", - // Interval: 1 * time.Second, - // } - // auxs := make([]FlexAux, 0) - // for info := range infos { - - // if source.Geo.Available() { - // photoTime := info.DateTime - // lastLocCheck := prevLocTime.Sub(photoTime) - // if lastLocCheck < 0 { - // lastLocCheck = -lastLocCheck - // } - // queryLocation := prevLocTime.IsZero() || lastLocCheck > 15*time.Minute - // // fmt.Printf("lastLocTime %v photoTime %v lastLocCheck %v queryLocation %v\n", lastLocTime, photoTime, lastLocCheck, queryLocation) - // if queryLocation && image.IsValidLatLng(info.LatLng) { - // prevLocTime = photoTime - // dist := image.AngleToKm(prevLoc.Distance(info.LatLng)) - // if dist > 1 { - // location, err := source.Geo.ReverseGeocode(context.TODO(), info.LatLng) - // if err == nil && location != prevLocation { - // prevLocation = location - // text := "" - // if prevAuxTime.Year() != photoTime.Year() { - // text += photoTime.Format("2006\r") - // } - // if prevAuxTime.YearDay() != photoTime.YearDay() { - // text += photoTime.Format("Jan 2\rMonday\r") - // } - // prevAuxTime = photoTime - // text += location - // aux := FlexAux{ - // Text: text, - // } - // auxs = append(auxs, aux) - // photos = append(photos, HighlightPhoto{ - // FlexPhoto: FlexPhoto{ - // Id: image.ImageId(len(auxs) - 1), - // AspectRatio: 0.2 + float32(longestLine(text))/10, - // Aux: true, - // }, - // Height: float32(auxHeight), - // }) - // } - // prevLoc = info.LatLng - // } - // } - // } - - // similarity := float32(0.) - // emb := info.Embedding.Float32() - // invnorm := info.Embedding.InvNormFloat32() - // simHeight := idealHeight - // if prevEmb != nil { - // dot, err := clip.DotProductFloat32Float32( - // prevEmb, - // emb, - // ) - // if err != nil { - // log.Printf("dot product error: %v", err) - // } - // similarity = dot * prevInvNorm * invnorm - // simHeight = idealHeight * math.Min(1, minHeightFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minHeightFrac)) - // } - // prevEmb = emb - // prevInvNorm = invnorm - - // photo := HighlightPhoto{ - // FlexPhoto: FlexPhoto{ - // Id: info.Id, - // AspectRatio: float32(info.Width) / float32(info.Height), - // }, - // Height: float32(simHeight), - // } - // photos = append(photos, photo) - // layoutCounter.Set(len(photos)) - // } - - // root := &FlexNode{ - // Index: -1, - // Cost: 0, - // TotalAspect: 0, - // } - - // q := deque.New[*FlexNode](len(photos) / 4) - // q.PushBack(root) - // indexToNode := make(map[int]*FlexNode, len(photos)) - - // maxLineWidth := rect.W - // for q.Len() > 0 { - // node := q.PopFront() - // totalAspect := 0. - // fallback := false - // hasAux := false - - // // fmt.Printf("queue %d\n", node.Index) - - // prevHeight := photos[0].Height - - // for i := node.Index + 1; i < len(photos); i++ { - // photo := photos[i] - // totalAspect += float64(photo.AspectRatio) - // totalSpacing := layout.ImageSpacing * float64(i-1-node.Index) - // photoHeight := (maxLineWidth - totalSpacing) / totalAspect - // minHeight := 0.3 * float64(photo.Height) - // maxHeight := 1.7 * float64(photo.Height) - // valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback - // // badness := math.Abs(photoHeight - idealHeight) - // badness := math.Abs(photoHeight - float64(photo.Height)) - // prevDiff := 0.1 * math.Abs(float64(prevHeight-photo.Height)) - // prevHeight = photo.Height - // // viewportDiff := 1000. * float64(photoHeight) - // viewportDiff := 1000. * math.Max(0, float64(photoHeight)-layout.ViewportHeight) - // cost := badness*badness + prevDiff*prevDiff + viewportDiff*viewportDiff + 10 - // // Incentivise aux items to be placed at the beginning - // if i < len(photos)-1 && photos[i+1].Aux { - // cost -= 1000000 - // } - // if hasAux && photoHeight < minAuxHeight { - // auxDiff := (minAuxHeight - photoHeight) * 4 - // cost += auxDiff * auxDiff - // } - // if photo.Aux { - // hasAux = true - // } - // // fmt.Printf(" photo %d aspect %f total %f width %f height %f valid %v badness %f cost %f\n", i, photo.AspectRatio, totalAspect, maxLineWidth, photoHeight, valid, badness, cost) - // if valid { - // n, ok := indexToNode[i] - // totalCost := node.Cost + float32(cost) - // if ok { - // if n.Cost > totalCost { - // n.Cost = totalCost - // n.TotalAspect = float32(totalAspect) - // n.ShortestParent = node - // // fmt.Printf(" node %d exists, lower cost %f\n", i, n.Cost) - // } - // // fmt.Printf(" node %d exists, keep cost %f\n", i, n.Cost) - // // } - // } else { - // n = &FlexNode{ - // Index: i, - // Cost: totalCost, - // TotalAspect: float32(totalAspect), - // ShortestParent: node, - // } - // indexToNode[i] = n - // if i < len(photos)-1 { - // q.PushBack(n) - // } - // // fmt.Printf(" node %d added with cost %f\n", i, n.Cost) - // } - // // fmt.Printf(" node %d %v cost %f\n", i, ok, n.Cost) - // } - // if photoHeight < minHeight { - // // Handle edge case where there is no other option - // // but to accept a photo that would otherwise break outside of the desired size - // if !fallback && i != len(photos)-1 && q.Len() == 0 { - // fallback = true - // for j := 0; j < 2 && i-j > node.Index; j++ { - // totalAspect -= float64(photos[i-j].AspectRatio) - // } - // i = i - 2 - // if i < node.Index+1 { - // i = node.Index + 1 - // } - // continue - // } - // break - // } - // } - // } - - // // dot := "digraph NodeGraph {\n" - // // dot += root.Dot() - // // dot += "}" - // // fmt.Println(dot) - - // // Trace back the shortest path - // shortestPath := make([]*FlexNode, 0) - // for node := indexToNode[len(photos)-1]; node != nil; { - // // fmt.Printf("node %d cost %f\n", node.Index, node.Cost) - // shortestPath = append(shortestPath, node) - // node = node.ShortestParent - // } - - // // Finally, place the photos based on the shortest path breaks - // x := 0. - // y := 0. - // idx := 0 - // for i := len(shortestPath) - 2; i >= 0; i-- { - // node := shortestPath[i] - // prev := shortestPath[i+1] - // totalSpacing := layout.ImageSpacing * float64(node.Index-1-prev.Index) - // imageHeight := (maxLineWidth - totalSpacing) / float64(node.TotalAspect) - // // fmt.Printf("node %d (%d) cost %f total aspect %f height %f\n", node.Index, prev.Index, node.Cost, node.TotalAspect, imageHeight) - // for ; idx <= node.Index; idx++ { - // photo := photos[idx] - // imageWidth := imageHeight * float64(photo.AspectRatio) - - // if photo.Aux { - // aux := auxs[photo.Id] - // size := imageHeight * 0.5 - // // lines := strings.Count(aux.Text, "\r") + 1 - // font := scene.Fonts.Main.Face(size, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) - // // lineOffset := float64(lines-1) * size * 0.4 - // padding := 2. - // text := render.Text{ - // Sprite: render.Sprite{ - // Rect: render.Rect{ - // X: rect.X + x + padding, - // Y: rect.Y + y + padding, - // W: imageWidth - 2*padding, - // H: imageHeight - 2*padding, - // }, - // }, - // Font: &font, - // Text: aux.Text, - // HAlign: canvas.Left, - // VAlign: canvas.Bottom, - // } - // scene.Texts = append(scene.Texts, text) - // } else { - // scene.Photos = append(scene.Photos, render.Photo{ - // Id: photo.Id, - // Sprite: render.Sprite{ - // Rect: render.Rect{ - // X: rect.X + x, - // Y: rect.Y + y, - // W: imageWidth, - // H: imageHeight, - // }, - // }, - // }) - // } - // x += imageWidth + layout.ImageSpacing - // // fmt.Printf("photo %d aspect %f height %f\n", idx, photo.AspectRatio, photo.Height) - // } - // x = 0 - // y += imageHeight + layout.LineSpacing - // } - - // // fmt.Printf("photos %d indextonode %d stack %d\n", len(photos), len(indexToNode), q.Len()) - // // fmt.Printf("photos %d stack %d\n", cap(photos), q.Cap()) - - // rect.H = rect.Y + y + sceneMargin - layout.LineSpacing - // scene.Bounds.H = rect.H - // layoutPlaced() + // Trace back the shortest path + shortestPath := make([]int, 0) + for nodeIndex := len(photos) - 1; nodeIndex != -2; { + shortestPath = append(shortestPath, nodeIndex) + nodeIndex = indexToNode[nodeIndex].ShortestParent + } + + // Finally, place the photos based on the shortest path breaks + x := 0. + y := 0. + idx := 0 + for i := len(shortestPath) - 2; i >= 0; i-- { + nodeIdx := shortestPath[i] + prevIdx := shortestPath[i+1] + node := indexToNode[nodeIdx] + totalSpacing := layout.ImageSpacing * float64(nodeIdx-1-prevIdx) + imageHeight := (maxLineWidth - totalSpacing) / float64(node.TotalAspect) + for ; idx <= nodeIdx; idx++ { + photo := photos[idx] + imageWidth := imageHeight * float64(photo.AspectRatio) + if photo.Aux { + aux := auxs[photo.Id] + size := imageHeight * 0.5 + font := scene.Fonts.Main.Face(size, canvas.Dimgray, canvas.FontRegular, canvas.FontNormal) + padding := 2. + text := render.Text{ + Sprite: render.Sprite{ + Rect: render.Rect{ + X: rect.X + x + padding, + Y: rect.Y + y + padding, + W: imageWidth - 2*padding, + H: imageHeight - 2*padding, + }, + }, + Font: &font, + Text: aux.Text, + HAlign: canvas.Left, + VAlign: canvas.Bottom, + } + scene.Texts = append(scene.Texts, text) + } else { + scene.Photos = append(scene.Photos, render.Photo{ + Id: photo.Id, + Sprite: render.Sprite{ + Rect: render.Rect{ + X: rect.X + x, + Y: rect.Y + y, + W: imageWidth, + H: imageHeight, + }, + }, + }) + } + x += imageWidth + layout.ImageSpacing + } + x = 0 + y += imageHeight + layout.LineSpacing + } + + rect.H = rect.Y + y + sceneMargin - layout.LineSpacing + scene.Bounds.H = rect.H + layoutPlaced() scene.RegionSource = PhotoRegionSource{ Source: source, From f991565ecbe3440360ba05b31cf8cd9ef27b6063 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Fri, 10 May 2024 22:59:41 +0200 Subject: [PATCH 5/8] Fix default width and height for photos --- internal/layout/flex.go | 4 ++++ internal/layout/highlights.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/internal/layout/flex.go b/internal/layout/flex.go index 2a5b453..986fd05 100644 --- a/internal/layout/flex.go +++ b/internal/layout/flex.go @@ -96,6 +96,10 @@ func LayoutFlex(infos <-chan image.SourcedInfo, layout Layout, scene *render.Sce } } } + if info.Width == 0 || info.Height == 0 { + info.Width = 3 + info.Height = 2 + } photo := dag.Photo{ Id: info.Id, AspectRatio: float32(info.Width) / float32(info.Height), diff --git a/internal/layout/highlights.go b/internal/layout/highlights.go index 3b4db75..a47f23a 100644 --- a/internal/layout/highlights.go +++ b/internal/layout/highlights.go @@ -135,6 +135,10 @@ func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.S prevEmb = emb prevInvNorm = invnorm + if info.Width == 0 || info.Height == 0 { + info.Width = 3 + info.Height = 2 + } photo := HighlightPhoto{ Photo: dag.Photo{ Id: info.Id, From 297f29bf6c434ebc4f8e474af19ffc46e94c7638 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Fri, 10 May 2024 23:11:47 +0200 Subject: [PATCH 6/8] Text alignment fixes --- internal/layout/album.go | 14 +++++++------- internal/layout/timeline.go | 14 +++++++------- internal/render/text.go | 1 - 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/internal/layout/album.go b/internal/layout/album.go index b4d5804..2fa1f2c 100644 --- a/internal/layout/album.go +++ b/internal/layout/album.go @@ -37,13 +37,14 @@ func LayoutAlbumEvent(layout Layout, rect render.Rect, event *AlbumEvent, scene X: rect.X, Y: rect.Y, W: rect.W, - H: 30, + H: 40, }, &font, event.StartTime.Format(dateFormat), ) + text.VAlign = canvas.Bottom scene.Texts = append(scene.Texts, text) - rect.Y += text.Sprite.Rect.H + 15 + rect.Y += text.Sprite.Rect.H } font := scene.Fonts.Main.Face(50, canvas.Black, canvas.FontRegular, canvas.FontNormal) @@ -53,21 +54,20 @@ func LayoutAlbumEvent(layout Layout, rect render.Rect, event *AlbumEvent, scene X: rect.X, Y: rect.Y, W: rect.W, - H: 30, + H: 40, }, &font, time, ) + text.VAlign = canvas.Bottom scene.Texts = append(scene.Texts, text) - rect.Y += text.Sprite.Rect.H + 10 + rect.Y += text.Sprite.Rect.H newBounds := addSectionToScene(&event.Section, scene, rect, layout, source) rect.Y = newBounds.Y + newBounds.H if event.LastOnDay { - rect.Y += 40 - } else { - rect.Y += 6 + rect.Y += 32 } return rect } diff --git a/internal/layout/timeline.go b/internal/layout/timeline.go index dbfd964..ad79f68 100644 --- a/internal/layout/timeline.go +++ b/internal/layout/timeline.go @@ -53,14 +53,14 @@ func LayoutTimelineEvent(layout Layout, rect render.Rect, event *TimelineEvent, font := scene.Fonts.Main.Face(40, canvas.Black, canvas.FontRegular, canvas.FontNormal) - scene.Texts = append(scene.Texts, - render.NewTextFromRect( - textBounds, - &font, - headerText, - ), + text := render.NewTextFromRect( + textBounds, + &font, + headerText, ) - rect.Y += textHeight + 15 + text.VAlign = canvas.Bottom + scene.Texts = append(scene.Texts, text) + rect.Y += textHeight + 4 newBounds := addSectionToScene(&event.Section, scene, rect, layout, source) diff --git a/internal/render/text.go b/internal/render/text.go index 0714366..ffb709c 100644 --- a/internal/render/text.go +++ b/internal/render/text.go @@ -30,7 +30,6 @@ func (text *Text) Draw(config *Render, c *canvas.Context, scales Scales) { return } - // textLine := canvas.NewTextLine(*text.Font, text.Text, canvas.Left) textLine := canvas.NewTextBox(*text.Font, text.Text, text.Sprite.Rect.W, text.Sprite.Rect.H, text.HAlign, text.VAlign, 0, 0) rect := text.Sprite.Rect rect.Y -= rect.H From 0017568b64935f89fc08dc274cc2c707696fb307 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Sun, 12 May 2024 22:59:18 +0200 Subject: [PATCH 7/8] Fix index out of range in LayoutHighlights --- internal/layout/highlights.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/layout/highlights.go b/internal/layout/highlights.go index a47f23a..6f6e661 100644 --- a/internal/layout/highlights.go +++ b/internal/layout/highlights.go @@ -173,7 +173,10 @@ func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.S // fmt.Printf("queue %d\n", node.Index) - prevHeight := photos[0].Height + prevHeight := float32(0.) + if nodeIndex+1 < len(photos) { + prevHeight = photos[nodeIndex+1].Height + } for i := nodeIndex + 1; i < len(photos); i++ { photo := photos[i] @@ -185,7 +188,7 @@ func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.S valid := photoHeight >= minHeight && photoHeight <= maxHeight || i == len(photos)-1 || fallback // badness := math.Abs(photoHeight - idealHeight) badness := math.Abs(photoHeight - float64(photo.Height)) - prevDiff := 0.1 * math.Abs(float64(prevHeight-photo.Height)) + prevDiff := 10 * math.Abs(float64(prevHeight-photo.Height)) prevHeight = photo.Height // viewportDiff := 1000. * float64(photoHeight) viewportDiff := 1000. * math.Max(0, float64(photoHeight)-layout.ViewportHeight) From f5bedf50e3f1f842b8a3991fdf5953960c4aa1dc Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Mon, 13 May 2024 23:42:28 +0200 Subject: [PATCH 8/8] Graceful degradation for non-available embeddings --- go.mod | 2 +- internal/clip/ai.go | 2 +- internal/image/database.go | 11 +++++++---- internal/layout/highlights.go | 7 +++---- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 168a0da..83c4ecb 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/dgraph-io/ristretto v0.1.2-0.20230929213430-5239be55a219 github.com/docker/go-units v0.4.0 github.com/felixge/fgprof v0.9.1 + github.com/gammazero/deque v0.2.1 github.com/go-chi/chi/v5 v5.0.4 github.com/go-chi/cors v1.2.0 github.com/go-chi/render v1.0.1 @@ -55,7 +56,6 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.9.0 // indirect github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 // indirect - github.com/gammazero/deque v0.2.1 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/internal/clip/ai.go b/internal/clip/ai.go index 8f168dc..2750ed6 100644 --- a/internal/clip/ai.go +++ b/internal/clip/ai.go @@ -30,7 +30,7 @@ func (e embedding) Byte() []byte { } func (e embedding) Float() []Float { - if e.bytes == nil { + if e.bytes == nil || len(e.bytes) == 0 { return nil } p := unsafe.Pointer(&e.bytes[0]) diff --git a/internal/image/database.go b/internal/image/database.go index 8c0ad21..1548d08 100644 --- a/internal/image/database.go +++ b/internal/image/database.go @@ -127,12 +127,15 @@ type TagIdRange struct { type tagSet map[tag.Id]struct{} func readEmbedding(stmt *sqlite.Stmt, invnormIndex int, embeddingIndex int) (clip.Embedding, error) { + if stmt.ColumnType(invnormIndex) == sqlite.TypeNull || stmt.ColumnType(embeddingIndex) == sqlite.TypeNull { + return clip.FromRaw(nil, 0), ErrNotFound + } invnorm := uint16(clip.InvNormMean + stmt.ColumnInt64(invnormIndex)) size := stmt.ColumnLen(embeddingIndex) bytes := make([]byte, size) read := stmt.ColumnBytes(embeddingIndex, bytes) if read != size { - return nil, fmt.Errorf("unable to read embedding bytes, expected %d, got %d", size, read) + return clip.FromRaw(nil, 0), fmt.Errorf("unable to read embedding bytes, expected %d, got %d", size, read) } return clip.FromRaw(bytes, invnorm), nil } @@ -741,7 +744,7 @@ func (source *Database) GetPathFromId(id ImageId) (string, bool) { func (source *Database) Get(id ImageId) (InfoResult, bool) { - conn := source.pool.Get(nil) + conn := source.pool.Get(context.TODO()) defer source.pool.Put(conn) stmt := conn.Prep(` @@ -1492,7 +1495,7 @@ func (source *Database) ListWithEmbeddings(dirs []string, options ListOptions) < sql += ` SELECT infos.id, width, height, orientation, color, created_at_unix, created_at_tz_offset, latitude, longitude, inv_norm, embedding FROM infos - INNER JOIN clip_emb ON clip_emb.file_id = id + LEFT JOIN clip_emb ON clip_emb.file_id = id ` sql += ` @@ -1576,7 +1579,7 @@ func (source *Database) ListWithEmbeddings(dirs []string, options ListOptions) < emb, err := readEmbedding(stmt, 9, 10) if err != nil { - log.Printf("Error reading embedding: %s\n", err.Error()) + log.Printf("Error reading embedding for %d: %s\n", info.Id, err.Error()) } info.Embedding = emb diff --git a/internal/layout/highlights.go b/internal/layout/highlights.go index 6f6e661..fec03a5 100644 --- a/internal/layout/highlights.go +++ b/internal/layout/highlights.go @@ -126,11 +126,10 @@ func LayoutHighlights(infos <-chan image.InfoEmb, layout Layout, scene *render.S prevEmb, emb, ) - if err != nil { - log.Printf("dot product error: %v", err) + if err == nil { + similarity = dot * prevInvNorm * invnorm + simHeight = idealHeight * math.Min(1, minHeightFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minHeightFrac)) } - similarity = dot * prevInvNorm * invnorm - simHeight = idealHeight * math.Min(1, minHeightFrac+math.Pow(1-(float64(similarity)-simMin)/(1-simMin), simPow)*(1-minHeightFrac)) } prevEmb = emb prevInvNorm = invnorm