diff --git a/migrator/migrator.go b/migrator/migrator.go
index 189a141f56..cec4e30fc2 100644
--- a/migrator/migrator.go
+++ b/migrator/migrator.go
@@ -524,8 +524,8 @@ func (m Migrator) MigrateColumn(value interface{}, field *schema.Field, columnTy
 
 	// check nullable
 	if nullable, ok := columnType.Nullable(); ok && nullable == field.NotNull {
-		// not primary key & database is nullable
-		if !field.PrimaryKey && nullable {
+		// not primary key & current database is non-nullable(to be nullable)
+		if !field.PrimaryKey && !nullable {
 			alterColumn = true
 		}
 	}
diff --git a/tests/go.mod b/tests/go.mod
index ba8a84a868..3728611017 100644
--- a/tests/go.mod
+++ b/tests/go.mod
@@ -10,7 +10,7 @@ require (
 	gorm.io/driver/mysql v1.5.7
 	gorm.io/driver/postgres v1.5.9
 	gorm.io/driver/sqlite v1.5.6
-	gorm.io/driver/sqlserver v1.5.3
+	gorm.io/driver/sqlserver v1.5.4
 	gorm.io/gorm v1.25.12
 )
 
diff --git a/tests/migrate_test.go b/tests/migrate_test.go
index d955c8d7f2..6c8a4216de 100644
--- a/tests/migrate_test.go
+++ b/tests/migrate_test.go
@@ -141,6 +141,48 @@ func TestAutoMigrateSelfReferential(t *testing.T) {
 	}
 }
 
+func TestAutoMigrateNullable(t *testing.T) {
+	type MigrateNullableColumn struct {
+		ID    uint
+		Bonus float64 `gorm:"not null"`
+		Stock float64
+	}
+
+	DB.Migrator().DropTable(&MigrateNullableColumn{})
+
+	DB.AutoMigrate(&MigrateNullableColumn{})
+
+	type MigrateNullableColumn2 struct {
+		ID    uint
+		Bonus float64
+		Stock float64 `gorm:"not null"`
+	}
+
+	if err := DB.Table("migrate_nullable_columns").AutoMigrate(&MigrateNullableColumn2{}); err != nil {
+		t.Fatalf("failed to auto migrate, got error: %v", err)
+	}
+
+	columnTypes, err := DB.Table("migrate_nullable_columns").Migrator().ColumnTypes(&MigrateNullableColumn{})
+	if err != nil {
+		t.Fatalf("failed to get column types, got error: %v", err)
+	}
+
+	for _, columnType := range columnTypes {
+		switch columnType.Name() {
+		case "bonus":
+			// allow to change non-nullable to nullable
+			if nullable, _ := columnType.Nullable(); !nullable {
+				t.Fatalf("bonus's nullable should be true, bug got %t", nullable)
+			}
+		case "stock":
+			// do not allow to change nullable to non-nullable
+			if nullable, _ := columnType.Nullable(); !nullable {
+				t.Fatalf("stock's nullable should be true, bug got %t", nullable)
+			}
+		}
+	}
+}
+
 func TestSmartMigrateColumn(t *testing.T) {
 	fullSupported := map[string]bool{"mysql": true, "postgres": true}[DB.Dialector.Name()]