diff --git a/defaults.yaml b/defaults.yaml index 8aa0dbb..75568b6 100644 --- a/defaults.yaml +++ b/defaults.yaml @@ -169,7 +169,7 @@ media: sqlite: path: photofield.thumbs.db cost: - time: 5ms + time: 6ms goexif: extensions: [".jpg", ".jpeg"] @@ -177,17 +177,17 @@ media: height: 256 fit: "INSIDE" cost: - time: 30ms # 15ms + extra cost + time_per_original_megapixel: 670us image: extensions: [".jpg", ".jpeg", ".png"] cost: - time_per_original_megapixel: 90ms + time_per_original_megapixel: 71ms thumb: fit: "INSIDE" cost: - time_per_resized_megapixel: 140ms + time_per_resized_megapixel: 70ms ffmpeg: fit: "INSIDE" @@ -197,17 +197,16 @@ media: sources: - # Internal thumbnail database + # # Internal thumbnail database - type: sqlite - # Embedded JPEG thumbnails + # # # Embedded JPEG thumbnails - type: goexif # Native image decoding - type: image - # # Synology Moments / Photo Station thumbnails # @@ -218,6 +217,8 @@ media: fit: "INSIDE" width: 120 height: 120 + cost: + time_per_resized_megapixel: 89ms - name: SM type: thumb @@ -226,6 +227,8 @@ media: fit: OUTSIDE width: 240 height: 240 + cost: + time_per_resized_megapixel: 70ms - name: M type: thumb @@ -234,6 +237,8 @@ media: fit: OUTSIDE width: 320 height: 320 + cost: + time_per_resized_megapixel: 66ms - name: B type: thumb @@ -242,6 +247,8 @@ media: fit: INSIDE width: 640 height: 640 + cost: + time_per_resized_megapixel: 42ms - name: XL type: thumb @@ -250,6 +257,8 @@ media: fit: OUTSIDE width: 1280 height: 1280 + cost: + time_per_resized_megapixel: 35ms # # Synology Moments / Photo Station video variants @@ -275,16 +284,22 @@ media: width: 256 height: 256 fit: INSIDE + cost: + time_per_original_megapixel: 120ms - type: ffmpeg width: 1280 height: 1280 fit: INSIDE + cost: + time_per_original_megapixel: 160ms - type: ffmpeg width: 4096 height: 4096 fit: INSIDE + cost: + time_per_original_megapixel: 180ms # These sources are used for handling small thumbnails specifically for diff --git a/docker/grafana/dashboards/photofield.json b/docker/grafana/dashboards/photofield.json index ca3b1eb..2f71504 100644 --- a/docker/grafana/dashboards/photofield.json +++ b/docker/grafana/dashboards/photofield.json @@ -217,7 +217,7 @@ }, "gridPos": { "h": 8, - "w": 8, + "w": 6, "x": 0, "y": 9 }, @@ -242,14 +242,14 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(pf_source_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", + "expr": "histogram_quantile($percentile/100, sum(rate(pf_source_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", "interval": "", "legendFormat": "{{source}}", "range": true, "refId": "A" } ], - "title": "Image Load Latency (p90)", + "title": "Load Latency (p$percentile)", "type": "timeseries" }, { @@ -314,8 +314,8 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 8, + "w": 6, + "x": 6, "y": 9 }, "id": 230, @@ -339,14 +339,14 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(pf_source_per_original_megapixel_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", + "expr": "histogram_quantile($percentile/100, sum(rate(pf_source_per_original_megapixel_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", "interval": "", "legendFormat": "{{source}}", "range": true, "refId": "A" } ], - "title": "Image Load Latency per Original Megapixel (p90)", + "title": "Load Latency / Original Megapixel (p$percentile)", "type": "timeseries" }, { @@ -354,6 +354,7 @@ "type": "prometheus", "uid": "RmeUbDMnz" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -411,8 +412,8 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 16, + "w": 6, + "x": 12, "y": 9 }, "id": 231, @@ -436,14 +437,111 @@ }, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(pf_source_per_resized_megapixel_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", + "expr": "histogram_quantile($percentile/100, sum(rate(pf_source_per_resized_megapixel_latency_bucket{source=~\"$source\"}[1m])) by (le, source))", "interval": "", "legendFormat": "{{source}}", "range": true, "refId": "A" } ], - "title": "Image Load Latency per Resized Megapixel (p90)", + "title": "Load Latency / Resized Megapixel (p$percentile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 284, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile($percentile/100, sum(rate(pf_source_latency_abs_diff_bucket{source=~\"$source\"}[1m])) by (le, source))", + "interval": "", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Latency Diff vs. Predicted (p$percentile)", "type": "timeseries" }, { @@ -461,7 +559,8 @@ "mode": "absolute", "steps": [ { - "color": "text" + "color": "text", + "value": null } ] }, @@ -471,7 +570,7 @@ }, "gridPos": { "h": 8, - "w": 8, + "w": 6, "x": 0, "y": 17 }, @@ -499,7 +598,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sort(histogram_quantile(0.90, sum(rate(pf_source_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", + "expr": "sort(histogram_quantile($percentile/100, sum(rate(pf_source_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", "format": "time_series", "instant": true, "interval": "", @@ -508,7 +607,7 @@ "refId": "A" } ], - "title": "Image Load Latency (p90)", + "title": "Load Latency (p$percentile)", "transformations": [], "type": "bargauge" }, @@ -538,8 +637,8 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 8, + "w": 6, + "x": 6, "y": 17 }, "id": 250, @@ -566,7 +665,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sort(histogram_quantile(0.90, sum(rate(pf_source_per_original_megapixel_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", + "expr": "sort(histogram_quantile($percentile/100, sum(rate(pf_source_per_original_megapixel_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", "format": "time_series", "instant": true, "interval": "", @@ -575,7 +674,7 @@ "refId": "A" } ], - "title": "Image Load Latency per Original Megapixel (p90)", + "title": "Load Latency / Original Megapixel (p$percentile)", "transformations": [], "type": "bargauge" }, @@ -605,8 +704,8 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 16, + "w": 6, + "x": 12, "y": 17 }, "id": 251, @@ -633,7 +732,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sort(histogram_quantile(0.90, sum(rate(pf_source_per_resized_megapixel_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", + "expr": "sort(histogram_quantile($percentile/100, sum(rate(pf_source_per_resized_megapixel_latency_bucket{source=~\"$source\"}[$__range])) by (le, source)))", "format": "time_series", "instant": true, "interval": "", @@ -642,7 +741,7 @@ "refId": "A" } ], - "title": "Image Load Latency per Resized Megapixel (p90)", + "title": "Load Latency / Resized Megapixel (p$percentile)", "transformations": [], "type": "bargauge" }, @@ -651,12 +750,478 @@ "type": "prometheus", "uid": "RmeUbDMnz" }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 17 + }, + "id": 268, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(histogram_quantile($percentile/100, sum(rate(pf_source_latency_abs_diff_bucket{source=~\"$source\"}[$__range])) by (le, source)))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{source}}", + "range": false, + "refId": "A" + } + ], + "title": "Load Latency Diff vs. Predicted (p$percentile)", + "transformations": [], + "type": "bargauge" + }, + { + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "id": 305, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "gridPos": { + "h": 2, + "w": 4.8, + "x": 0, + "y": 26 + }, + "id": 411, + "maxPerRow": 12, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# ${source}", + "mode": "markdown" + }, + "pluginVersion": "9.4.7", + "repeat": "source", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (rate(pf_source_latency_bucket{source=~\"$source\"}[$__rate_interval]))", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4.8, + "x": 0, + "y": 28 + }, + "id": 349, + "maxPerRow": 12, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "µs" + } + }, + "pluginVersion": "9.4.7", + "repeat": "source", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (rate(pf_source_latency_bucket{source=~\"$source\"}[$__rate_interval]))", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "$source - Latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4.8, + "x": 0, + "y": 35 + }, + "id": 377, + "maxPerRow": 12, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "µs" + } + }, + "pluginVersion": "9.4.7", + "repeat": "source", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(pf_source_per_original_megapixel_latency_bucket{source=~\"$source\"}[$__rate_interval]))", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "$source - Latency / Original MP", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4.8, + "x": 0, + "y": 42 + }, + "id": 378, + "maxPerRow": 12, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "µs" + } + }, + "pluginVersion": "9.4.7", + "repeat": "source", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(pf_source_per_resized_megapixel_latency_bucket{source=~\"$source\"}[$__rate_interval]))", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "$source - Latency / Resized MP", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4.8, + "x": 0, + "y": 49 + }, + "id": 379, + "maxPerRow": 12, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "µs" + } + }, + "pluginVersion": "9.4.7", + "repeat": "source", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(pf_source_latency_abs_diff_bucket{source=~\"$source\"}[$__rate_interval]))", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "$source - Latency Diff vs. Predicted", + "type": "heatmap" + } + ], + "title": "Source Details", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "RmeUbDMnz" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, "id": 143, "targets": [ { @@ -702,7 +1267,7 @@ "h": 5, "w": 12, "x": 0, - "y": 26 + "y": 27 }, "id": 93, "options": { @@ -768,7 +1333,7 @@ "h": 5, "w": 6, "x": 12, - "y": 26 + "y": 27 }, "id": 109, "options": { @@ -833,7 +1398,7 @@ "h": 5, "w": 6, "x": 18, - "y": 26 + "y": 27 }, "id": 126, "options": { @@ -879,7 +1444,7 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 32 }, "id": 56, "targets": [ @@ -909,8 +1474,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "yellow", @@ -926,7 +1490,7 @@ "h": 5, "w": 6, "x": 0, - "y": 32 + "y": 33 }, "id": 38, "options": { @@ -976,8 +1540,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -993,7 +1556,7 @@ "h": 5, "w": 6, "x": 6, - "y": 32 + "y": 33 }, "id": 25, "options": { @@ -1043,8 +1606,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1056,7 +1618,7 @@ "h": 5, "w": 12, "x": 12, - "y": 32 + "y": 33 }, "id": 33, "options": { @@ -1120,8 +1682,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -1136,7 +1697,7 @@ "h": 5, "w": 12, "x": 0, - "y": 37 + "y": 38 }, "id": 37, "options": { @@ -1198,8 +1759,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -1215,7 +1775,7 @@ "h": 5, "w": 12, "x": 12, - "y": 37 + "y": 38 }, "id": 35, "options": { @@ -1269,8 +1829,7 @@ "mode": "absolute", "steps": [ { - "color": "text", - "value": null + "color": "text" }, { "color": "green", @@ -1286,7 +1845,7 @@ "h": 6, "w": 12, "x": 0, - "y": 42 + "y": 43 }, "id": 75, "options": { @@ -1354,8 +1913,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1367,7 +1925,7 @@ "h": 6, "w": 12, "x": 12, - "y": 42 + "y": 43 }, "id": 92, "options": { @@ -1415,7 +1973,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 49 }, "id": 12, "panels": [ @@ -1496,7 +2054,7 @@ "h": 6, "w": 12, "x": 0, - "y": 47 + "y": 55 }, "id": 13, "options": { @@ -1600,7 +2158,7 @@ "h": 6, "w": 12, "x": 12, - "y": 47 + "y": 55 }, "id": 17, "options": { @@ -1724,7 +2282,7 @@ "h": 6, "w": 12, "x": 0, - "y": 53 + "y": 61 }, "id": 14, "options": { @@ -1862,7 +2420,7 @@ "h": 6, "w": 12, "x": 12, - "y": 53 + "y": 61 }, "id": 15, "options": { @@ -1922,7 +2480,7 @@ "type": "row" } ], - "refresh": "2s", + "refresh": "", "revision": 1, "schemaVersion": 38, "style": "dark", @@ -1977,7 +2535,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": [ "All" ], @@ -2070,11 +2628,31 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "current": { + "selected": true, + "text": "50", + "value": "50" + }, + "hide": 0, + "label": "Latency p", + "name": "percentile", + "options": [ + { + "selected": true, + "text": "50", + "value": "50" + } + ], + "query": "50", + "skipUrlSync": false, + "type": "textbox" } ] }, "time": { - "from": "now-15m", + "from": "now-2d", "to": "now" }, "timepicker": { @@ -2096,6 +2674,6 @@ "timezone": "", "title": "Photofield", "uid": "9sQ5hGGnk", - "version": 13, + "version": 24, "weekStart": "" } \ No newline at end of file diff --git a/internal/image/source.go b/internal/image/source.go index d820f6c..906a3a6 100644 --- a/internal/image/source.go +++ b/internal/image/source.go @@ -16,7 +16,7 @@ import ( "photofield/internal/queue" "photofield/io" "photofield/io/ffmpeg" - ioristretto "photofield/io/ristretto" + "photofield/io/ristretto" "photofield/io/sqlite" "photofield/tag" @@ -137,6 +137,7 @@ type Source struct { Sources io.Sources SourceLatencyHistogram *prometheus.HistogramVec + SourceLatencyAbsDiffHistogram *prometheus.HistogramVec SourcePerOriginalMegapixelLatencyHistogram *prometheus.HistogramVec SourcePerResizedMegapixelLatencyHistogram *prometheus.HistogramVec @@ -167,7 +168,15 @@ func NewSource(config Config, migrations embed.FS, migrationsThumbs embed.FS) *S source.SourceLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metrics.Namespace, Name: "source_latency", - Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, + Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 150000, 200000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, + }, + []string{"source"}, + ) + + source.SourceLatencyAbsDiffHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metrics.Namespace, + Name: "source_latency_abs_diff", + Buckets: []float64{50, 100, 250, 500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 200000, 500000, 1000000}, }, []string{"source"}, ) @@ -175,7 +184,7 @@ func NewSource(config Config, migrations embed.FS, migrationsThumbs embed.FS) *S source.SourcePerOriginalMegapixelLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metrics.Namespace, Name: "source_per_original_megapixel_latency", - Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, + Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 150000, 200000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, }, []string{"source"}, ) @@ -183,7 +192,7 @@ func NewSource(config Config, migrations embed.FS, migrationsThumbs embed.FS) *S source.SourcePerResizedMegapixelLatencyHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metrics.Namespace, Name: "source_per_resized_megapixel_latency", - Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, + Buckets: []float64{500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, 150000, 200000, 250000, 500000, 1000000, 2000000, 5000000, 10000000}, }, []string{"source"}, ) @@ -192,7 +201,7 @@ func NewSource(config Config, migrations embed.FS, migrationsThumbs embed.FS) *S SourceTypes: config.SourceTypes, FFmpegPath: ffmpeg.FindPath(), Migrations: migrationsThumbs, - ImageCache: ioristretto.New(), + ImageCache: ristretto.New(), DataDir: config.DataDir, } diff --git a/internal/render/photo.go b/internal/render/photo.go index 26d8746..ec4395e 100644 --- a/internal/render/photo.go +++ b/internal/render/photo.go @@ -5,6 +5,7 @@ import ( "fmt" "image/color" "log" + "math" "photofield/internal/image" "photofield/io" "time" @@ -77,6 +78,9 @@ func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales } sources := srcs.EstimateCost(io.Size(size), io.Size(rsize)) sources.Sort() + + var errs []error + for i, s := range sources { if drawn { break @@ -87,13 +91,21 @@ func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales img, err := r.Image, r.Error if img == nil || err != nil { + if err != nil { + errs = append(errs, err) + } continue } - name := s.Name() - source.SourceLatencyHistogram.WithLabelValues(name).Observe(float64(elapsed.Microseconds())) - source.SourcePerOriginalMegapixelLatencyHistogram.WithLabelValues(name).Observe(float64(elapsed) * 1e6 / (float64(size.X) * float64(size.Y))) - source.SourcePerResizedMegapixelLatencyHistogram.WithLabelValues(name).Observe(float64(elapsed) * 1e6 / float64(s.EstimatedArea)) + if !r.FromCache { + name := s.Name() + elapsedus := float64(elapsed.Microseconds()) + elapsedabsdiff := math.Abs(float64(s.EstimatedDuration.Microseconds()) - elapsedus) + source.SourceLatencyHistogram.WithLabelValues(name).Observe(elapsedus) + source.SourceLatencyAbsDiffHistogram.WithLabelValues(name).Observe(elapsedabsdiff) + source.SourcePerOriginalMegapixelLatencyHistogram.WithLabelValues(name).Observe(elapsedus * 1e6 / (float64(size.X) * float64(size.Y))) + source.SourcePerResizedMegapixelLatencyHistogram.WithLabelValues(name).Observe(elapsedus * 1e6 / float64(s.EstimatedArea)) + } if r.Orientation == io.SourceInfoOrientation { r.Orientation = io.Orientation(info.Orientation) @@ -138,6 +150,10 @@ func (photo *Photo) Draw(config *Render, scene *Scene, c *canvas.Context, scales } if !drawn { + if len(errs) > 0 { + log.Printf("Unable to draw photo %v: %v", photo.Id, errs) + } + style := c.Style style.FillColor = canvas.Red photo.Sprite.DrawWithStyle(c, style) diff --git a/io/bench/bench.go b/io/bench/bench.go new file mode 100644 index 0000000..ae75196 --- /dev/null +++ b/io/bench/bench.go @@ -0,0 +1,87 @@ +package bench + +import ( + "context" + "fmt" + "log" + "math/rand" + "photofield/io" + "testing" +) + +type Sample struct { + Id io.ImageId + Path string + Size io.Size +} + +type sourceWithSamples struct { + source io.Source + samples []Sample +} + +func BenchmarkSources(seed int64, sources io.Sources, samples []Sample, count int) { + log.Printf("benchmark build samples") + workingSources := make([]sourceWithSamples, 0, len(sources)) + for _, source := range sources { + workingSources = append(workingSources, sourceWithSamples{ + source, + workingSamples(source, samples), + }) + } + log.Printf("benchmark run") + maxLen := 20 + for i := 0; i < count; i++ { + for _, s := range workingSources { + if len(s.samples) == 0 { + fmt.Printf("# BenchmarkSourceGet/%-*s\t%s\n", maxLen, s.source.Name(), "no samples") + continue + } + r := testing.Benchmark(func(b *testing.B) { + BenchmarkSource(b, seed, s.source, s.samples) + }) + if r.N == 0 { + fmt.Printf("# BenchmarkSourceGet/%-*s\t%s\n", maxLen, s.source.Name(), "error") + continue + } + fmt.Printf("BenchmarkSourceGet/%-*s\t%s\t%s\n", maxLen, s.source.Name(), r.String(), r.MemString()) + } + } +} + +func workingSamples(source io.Source, samples []Sample) []Sample { + working := make([]Sample, 0, len(samples)) + for _, sample := range samples { + if source.Exists(context.Background(), sample.Id, sample.Path) { + working = append(working, sample) + } + } + return working +} + +func BenchmarkSource(b *testing.B, seed int64, source io.Source, samples []Sample) { + b.StopTimer() + ctx := context.Background() + b.ReportMetric(float64(len(samples)), "samples") + + rnd := rand.New(rand.NewSource(seed)) + + for i := 0; i < b.N; i++ { + sample := samples[rnd.Intn(len(samples))] + resized := source.Size(sample.Size) + b.StartTimer() + r := source.Get(ctx, sample.Id, sample.Path) + b.StopTimer() + if r.Error != nil { + b.Fatal(r.Error) + } + ns := float64(b.Elapsed().Nanoseconds()) + origmp := float64(sample.Size.Area()) / 1e6 + b.ReportMetric(ns/origmp/float64(b.N), "ns/origmp/op") + resmp := float64(resized.Area()) / 1e6 + b.ReportMetric(ns/resmp/float64(b.N), "ns/resmp/op") + gotsize := io.Size{X: r.Image.Bounds().Dx(), Y: r.Image.Bounds().Dy()} + gotmp := float64(gotsize.Area()) / 1e6 + b.ReportMetric(ns/gotmp/float64(b.N), "ns/gotmp/op") + } +} diff --git a/io/cached/cached.go b/io/cached/cached.go index 58a22bd..af6a38b 100644 --- a/io/cached/cached.go +++ b/io/cached/cached.go @@ -52,6 +52,7 @@ func (c *Cached) Get(ctx context.Context, id io.ImageId, path string) io.Result if r.Image != nil || r.Error != nil { // fmt.Printf("%v cache found\n", id) // println("found in cache") + r.FromCache = true return r } // r = c.Source.Get(ctx, id, path) diff --git a/io/io.go b/io/io.go index 1978562..f26c3e2 100644 --- a/io/io.go +++ b/io/io.go @@ -95,6 +95,7 @@ func (s Size) Fit(original Size, fit AspectRatioFit) Size { type Result struct { Image image.Image Orientation Orientation + FromCache bool Error error } @@ -128,34 +129,49 @@ type ReadDecoder interface { type Sources []Source -func (sources Sources) EstimateCost(original Size, target Size) SourceCosts { +// Original +// var UnderdrawPenaltyMultiplier = 15. +// var SizeCostMultiplier = 0.00001 +// var DurationCostMultiplier = 0.003 + +// Optimized for 0.9 max width ratio + square duration +var UnderdrawPenaltyMultiplier = 59.851585 +var SizeCostMultiplier = 0.000281 +var DurationCostMultiplier = 0.011857 + +func SizeCost(source Size, original Size, target Size) (cost float64, area int64) { + if source.X == 0 && source.Y == 0 { + source = target + } + area = source.Area() targetArea := target.Area() + diff := float64(targetArea) - float64(area) + if targetArea > area { + diff *= UnderdrawPenaltyMultiplier + } + cost = diff * diff * SizeCostMultiplier + return +} + +func DurationCost(dur time.Duration) float64 { + us := float64(dur.Microseconds()) + return us * us * DurationCostMultiplier +} + +func (sources Sources) EstimateCost(original Size, target Size) SourceCosts { costs := make([]SourceCost, len(sources)) for i := range sources { s := sources[i] - ssize := s.Size(original) - if ssize.X == 0 && ssize.Y == 0 { - ssize = target - } - sarea := ssize.Area() - sizecost := math.Abs(float64(targetArea)-float64(sarea)) * 0.001 - if targetArea > sarea { - // areacost = math.Sqrt(float64(targetArea)-float64(sarea)) * 3 - // areacost = math.Sqrt(float64(targetArea)-float64(sarea)) * 3 - sizecost *= 15 - } - // dx := float64(target.X - ssize.X) - // dy := float64(target.Y - ssize.Y) - // sizecost := math.Sqrt(dx*dx + dy*dy) + sizecost, sarea := SizeCost(s.Size(original), original, target) dur := s.GetDurationEstimate(original) - durcost := math.Pow(float64(dur.Microseconds()), 1) * 0.003 - // durcost := float64(dur.Microseconds()) * 0.001 + durcost := DurationCost(dur) cost := sizecost + durcost - // fmt.Printf("%4d %30s %12s %12s %12s %12d %12f %10s %12f %12f\n", i, s.Name(), original, target, ssize, sarea, sizecost, dur, durcost, cost) costs[i] = SourceCost{ Source: s, EstimatedArea: sarea, EstimatedDuration: dur, + SizeCost: sizecost, + DurationCost: durcost, Cost: cost, } } @@ -166,6 +182,8 @@ type SourceCost struct { Source EstimatedArea int64 EstimatedDuration time.Duration + SizeCost float64 + DurationCost float64 Cost float64 } @@ -178,3 +196,11 @@ func (costs SourceCosts) Sort() { return a.Cost < b.Cost }) } + +func (costs SourceCosts) SortSize() { + sort.Slice(costs, func(i, j int) bool { + a := costs[i] + b := costs[j] + return a.SizeCost < b.SizeCost + }) +} diff --git a/io/thumb/thumb.go b/io/thumb/thumb.go index 674638c..7546806 100644 --- a/io/thumb/thumb.go +++ b/io/thumb/thumb.go @@ -117,7 +117,7 @@ func (t Thumb) Size(size io.Size) io.Size { } func (t Thumb) GetDurationEstimate(size io.Size) time.Duration { - return 31 * time.Nanosecond * time.Duration(size.Area()) + return 31 * time.Nanosecond * time.Duration(t.Width*t.Height) } func (t *Thumb) resolvePath(originalPath string) string { diff --git a/justfile b/justfile index ec55e94..befdee5 100644 --- a/justfile +++ b/justfile @@ -22,6 +22,9 @@ run-static *args: go build -tags embedstatic ./photofield {{args}} +bench collection: build + ./photofield -bench -bench.collection {{collection}} -test.benchtime 1s -test.count 6 + ui: cd ui && npm run dev diff --git a/main.go b/main.go index 5bb0c74..30a146c 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "io/fs" "io/ioutil" "math" + "math/rand" "mime" "path" "path/filepath" @@ -21,6 +22,7 @@ import ( "sort" "strings" "sync" + "testing" "time" "io" @@ -60,6 +62,7 @@ import ( "photofield/internal/render" "photofield/internal/scene" pfio "photofield/io" + "photofield/io/bench" "photofield/tag" ) @@ -1126,15 +1129,59 @@ func IndexHTML() func(next http.Handler) http.Handler { } } +// benchmarkSources runs a benchmark on image sources +// +// It's not very usable right now as it doesn't use a representative sample of images, +// but it's a start. +func benchmarkSources(collection *collection.Collection, seed int64, sampleSize int, count int) { + ids := make([]image.ImageId, 0) + for id := range collection.GetIds(imageSource) { + ids = append(ids, id) + } + randGen := rand.New(rand.NewSource(seed)) + randGen.Shuffle(len(ids), func(i, j int) { ids[i], ids[j] = ids[j], ids[i] }) + if len(ids) > sampleSize { + ids = ids[:sampleSize] + } + samples := make([]bench.Sample, 0) + for _, id := range ids { + path, err := imageSource.GetImagePath(id) + if err != nil { + panic(err) + } + info := imageSource.GetInfo(id) + samples = append(samples, bench.Sample{ + Id: pfio.ImageId(id), + Path: path, + Size: pfio.Size{ + X: info.Width, + Y: info.Height, + }, + }) + } + sources := imageSource.Sources + bench.BenchmarkSources(seed, sources, samples, count) +} + func main() { startupTime = time.Now() - versionPtr := flag.Bool("version", false, "print version and exit") - vacuumPtr := flag.Bool("vacuum", false, "clean database for smaller size and better performance, and exit") + testing.Init() + versionFlag := flag.Bool("version", false, "print version and exit") + vacuumFlag := flag.Bool("vacuum", false, "clean database for smaller size and better performance, and exit") + benchFlag := flag.Bool("bench", false, "benchmark sources and exit") + benchCollectionId := flag.String("bench.collection", "vacation-photos", "id of the collection to benchmark") + benchSeed := flag.Int64("bench.seed", 123, "seed for random number generator") + benchSample := flag.Int("bench.sample", 10000, "number of images from the collection to use as a sample") + flag.Parse() flag.Parse() - if *versionPtr { + if *benchFlag { + log.SetOutput(os.Stderr) + } + + if *versionFlag { fmt.Printf("photofield %s, commit %s, built on %s by %s\n", version, commit, date, builtBy) return } @@ -1200,7 +1247,7 @@ func main() { imageSource = image.NewSource(appConfig.Media, migrations, migrationsThumbs) defer imageSource.Close() - if *vacuumPtr { + if *vacuumFlag { err := imageSource.Vacuum() if err != nil { panic(err) @@ -1239,6 +1286,19 @@ func main() { log.Printf(" %v - %v files indexed %v ago", collection.Name, collection.IndexedCount, indexedAgo) } + if *benchFlag { + log.Printf("benchmark sources") + + count := flag.Lookup("test.count").Value.(flag.Getter).Get().(uint) + + c := getCollectionById(*benchCollectionId) + if c == nil { + panic(fmt.Errorf("collection %v not found", *benchCollectionId)) + } + benchmarkSources(c, *benchSeed, *benchSample, int(count)) + return + } + metadataTask := Task{ Type: string(openapi.TaskTypeINDEXMETADATA), Id: "index-metadata",