fix qp range for av1_nvenc

The range is [0, 255] per `ffmpeg/src/libavcodec/nvenc.c`
mltframework · Dec 11, 2023 · d9bf8b5 · d9bf8b5
1 parent 9e0c1de
commit d9bf8b5
Showing 1 changed file with 24 additions and 19 deletions.
diff --git a/src/docks/encodedock.cpp b/src/docks/encodedock.cpp
@@ -52,6 +52,10 @@ static const QString kNullTarget = "/dev/null";
 #endif
 
 static double getBufferSize(Mlt::Properties &preset, const char *property);
+static int maxQpNvenc(const QString &vcodec)
+{
+    return vcodec == "av1_nvenc" ? 255 : 51;
+}
 
 EncodeDock::EncodeDock(QWidget *parent) :
     QDockWidget(parent),
@@ -153,6 +157,18 @@ void EncodeDock::loadPresetFromProperties(Mlt::Properties &preset)
     QString acodec = QString::fromLatin1(preset.get("acodec"));
     QString vcodec = QString::fromLatin1(preset.get("vcodec"));
 
+    if (ui->hwencodeCheckBox->isChecked()) {
+        foreach (const QString &hw, Settings.encodeHardware()) {
+            if ((vcodec == "libx264" && hw.startsWith("h264"))
+                    || (vcodec == "libx265" && hw.startsWith("hevc"))
+                    || (vcodec == "libvpx-vp9" && hw.startsWith("vp9"))
+                    || (vcodec == "libaom-av1" && hw.startsWith("av1"))) {
+                vcodec = hw;
+                break;
+            }
+        }
+    }
+
     ui->disableAudioCheckbox->setChecked(preset.get_int("an"));
     ui->disableVideoCheckbox->setChecked(preset.get_int("vn"));
     m_extension.clear();
@@ -190,17 +206,6 @@ void EncodeDock::loadPresetFromProperties(Mlt::Properties &preset)
                 if (ui->audioCodecCombo->itemText(j) == value)
                     ui->audioCodecCombo->setCurrentIndex(j);
         } else if (name == "vcodec") {
-            if (ui->hwencodeCheckBox->isChecked()) {
-                foreach (const QString &hw, Settings.encodeHardware()) {
-                    if ((vcodec == "libx264" && hw.startsWith("h264"))
-                            || (vcodec == "libx265" && hw.startsWith("hevc"))
-                            || (vcodec == "libvpx-vp9" && hw.startsWith("vp9"))
-                            || (vcodec == "libaom-av1" && hw.startsWith("av1"))) {
-                        vcodec = hw;
-                        break;
-                    }
-                }
-            }
             for (int j = 0; j < ui->videoCodecCombo->count(); j++)
                 if (ui->videoCodecCombo->itemText(j) == vcodec)
                     ui->videoCodecCombo->setCurrentIndex(j);
@@ -307,7 +312,7 @@ void EncodeDock::loadPresetFromProperties(Mlt::Properties &preset)
             videoQuality = preset.get_int(name.toUtf8().constData());
         } else if (name == "crf") {
             ui->videoRateControlCombo->setCurrentIndex(preset.get("vbufsize") ? RateControlConstrained
-                                                       : ui->videoCodecCombo->currentText().endsWith("_ videotoolbox") ? RateControlAverage :
+                                                       : vcodec.endsWith("_videotoolbox") ? RateControlAverage :
                                                        RateControlQuality);
             videoQuality = preset.get_int("crf");
         } else if (name == "bufsize" || name == "vbufsize") {
@@ -374,12 +379,11 @@ void EncodeDock::loadPresetFromProperties(Mlt::Properties &preset)
     if ((ui->videoRateControlCombo->currentIndex() == RateControlQuality
             || ui->videoRateControlCombo->currentIndex() == RateControlConstrained)
             && videoQuality > -1) {
-        const QString &vcodec = ui->videoCodecCombo->currentText();
         //val = min + (max - min) * paramval;
         if (vcodec.startsWith("libx264") || vcodec == "libx265" || vcodec.contains("nvenc")
                 || vcodec.endsWith("_amf") || vcodec.endsWith("_vaapi"))
             // 0 (best, 100%) - 51 (worst)
-            ui->videoQualitySpinner->setValue(TO_RELATIVE(51, 0, videoQuality));
+            ui->videoQualitySpinner->setValue(TO_RELATIVE((vcodec == "av1_nvenc") ? 63 : 51, 0, videoQuality));
         else if (vcodec.endsWith("_videotoolbox"))
 #if defined(Q_OS_MAC) && defined(Q_PROCESSOR_ARM)
             ui->videoQualitySpinner->setValue(ui->hwencodeCheckBox->isChecked() ? videoQuality * 55.0 / 23.0 :
@@ -699,13 +703,14 @@ Mlt::Properties *EncodeDock::collectProperties(int realtime, bool includeProfile
                 }
                 case RateControlQuality: {
                     setIfNotSet(p, "rc", "constqp");
-                    setIfNotSet(p, "vqp", TO_ABSOLUTE(51, 0, vq));
-                    setIfNotSet(p, "vq", TO_ABSOLUTE(51, 0, vq));
+                    auto qmax = maxQpNvenc(vcodec);
+                    setIfNotSet(p, "vqp", TO_ABSOLUTE(qmax, 0, vq));
+                    setIfNotSet(p, "vq", TO_ABSOLUTE(qmax, 0, vq));
                     break;
                 }
                 case RateControlConstrained: {
                     const QString &b = ui->videoBitrateCombo->currentText();
-                    setIfNotSet(p, "qmin", TO_ABSOLUTE(51, 0, vq));
+                    setIfNotSet(p, "qmin", TO_ABSOLUTE(maxQpNvenc(vcodec), 0, vq));
                     setIfNotSet(p, "vb", qRound(cvbr));
                     setIfNotSet(p, "vmaxrate", b.toLatin1().constData());
                     setIfNotSet(p, "vbufsize", int(ui->videoBufferSizeSpinner->value() * 8 * 1024));
@@ -2131,9 +2136,9 @@ void EncodeDock::on_videoQualitySpinner_valueChanged(int vq)
         s = QString("crf=%1").arg(TO_ABSOLUTE(63, 0, vq));
     } else if (vcodec.contains("nvenc")) {
         if (ui->videoRateControlCombo->currentIndex() == RateControlQuality)
-            s = QString("vqp=%1").arg(TO_ABSOLUTE(51, 0, vq));
+            s = QString("vqp=%1").arg(TO_ABSOLUTE(maxQpNvenc(vcodec), 0, vq));
         else
-            s = QString("qmin=%1").arg(TO_ABSOLUTE(51, 0, vq));
+            s = QString("qmin=%1").arg(TO_ABSOLUTE(maxQpNvenc(vcodec), 0, vq));
     } else if (vcodec.endsWith("_amf")) {
         s = QString("qp_i=qp_p=qp_b=%1").arg(TO_ABSOLUTE(51, 0, vq));
     } else if (vcodec.endsWith("_vaapi")) {