Merge branch 'main' into Update-readme

.github/workflows/link_checker.yml

@@ -14,9 +14,9 @@ jobs:
   link-checker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.10.*
       - name: Install dependencies

.github/workflows/nightly.yml

@@ -9,9 +9,9 @@ jobs:
     if: github.repository == 'sony/model_optimization' # Don't do this in forks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 

.github/workflows/publish_release.yml

@@ -6,9 +6,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Install dependencies

.github/workflows/run_keras_sony_custom_layers.yml

@@ -15,9 +15,9 @@ jobs:
   run-tensorflow-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Install dependencies

.github/workflows/run_keras_tests.yml

@@ -14,9 +14,9 @@ jobs:
   run-tensorflow-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python-version }}
       - name: Install dependencies

.github/workflows/run_pytorch_tests.yml

@@ -14,9 +14,9 @@ jobs:
   run-pytorch-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python-version }}
       - name: Install dependencies

.github/workflows/run_tests_python310_pytorch25.yml

@@ -0,0 +1,19 @@
+name: Python 3.10, Pytorch 2.5
+on:
+  workflow_dispatch: # Allow manual triggers
+  schedule:
+    - cron: 0 0 * * *
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/run_pytorch_tests.yml
+    with:
+      python-version: "3.10"
+      torch-version: "2.5.*"

.github/workflows/run_tests_python311_pytorch25.yml

@@ -0,0 +1,19 @@
+name: Python 3.11, Pytorch 2.5
+on:
+  workflow_dispatch: # Allow manual triggers
+  schedule:
+    - cron: 0 0 * * *
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/run_pytorch_tests.yml
+    with:
+      python-version: "3.11"
+      torch-version: "2.5.*"

.github/workflows/run_tests_python312_pytorch22.yml

@@ -0,0 +1,19 @@
+name: Python 3.12, Pytorch 2.2
+on:
+  workflow_dispatch: # Allow manual triggers
+  schedule:
+    - cron: 0 0 * * *
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/run_pytorch_tests.yml
+    with:
+      python-version: "3.12"
+      torch-version: "2.2.*"

.github/workflows/run_tests_python312_pytorch23.yml

@@ -0,0 +1,19 @@
+name: Python 3.12, Pytorch 2.3
+on:
+  workflow_dispatch: # Allow manual triggers
+  schedule:
+    - cron: 0 0 * * *
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/run_pytorch_tests.yml
+    with:
+      python-version: "3.12"
+      torch-version: "2.3.*"

.github/workflows/run_tests_python312_pytorch24.yml

@@ -0,0 +1,19 @@
+name: Python 3.12, Pytorch 2.4
+on:
+  workflow_dispatch: # Allow manual triggers
+  schedule:
+    - cron: 0 0 * * *
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/run_pytorch_tests.yml
+    with:
+      python-version: "3.12"
+      torch-version: "2.4.*"

.github/workflows/run_tests_python312_pytorch25.yml

@@ -0,0 +1,19 @@
+name: Python 3.2, Pytorch 2.5
+on:
+  workflow_dispatch: # Allow manual triggers
+  schedule:
+    - cron: 0 0 * * *
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/run_pytorch_tests.yml
+    with:
+      python-version: "3.12"
+      torch-version: "2.5.*"

.github/workflows/run_tests_python39_pytorch25.yml

@@ -0,0 +1,19 @@
+name: Python 3.9, Pytorch 2.5
+on:
+  workflow_dispatch: # Allow manual triggers
+  schedule:
+    - cron: 0 0 * * *
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/run_pytorch_tests.yml
+    with:
+      python-version: "3.9"
+      torch-version: "2.5.*"

.github/workflows/run_tests_suite_coverage.yml

@@ -18,9 +18,9 @@ jobs:
     env:
       COVERAGE_THRESHOLD: 98
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 

.github/workflows/stale.yml

@@ -18,7 +18,7 @@ jobs:
       pull-requests: write
 
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'Stale issue message'

.github/workflows/test_whl_install.yml

@@ -11,11 +11,11 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{  inputs.python_version  }}
       - name: Install dependencies

.github/workflows/tests_common.yml

@@ -15,9 +15,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
       - name: Install dependencies

README.md

@@ -97,7 +97,6 @@ Currently, MCT is being tested on various Python, Pytorch and TensorFlow version
 | Python 3.11 | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch22.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch22.yml) | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch23.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch23.yml) | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch24.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch24.yml) | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch25.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python311_pytorch25.yml) |
 | Python 3.12 | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch22.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch22.yml) | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch23.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch23.yml) | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch24.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch24.yml) | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch25.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python312_pytorch25.yml) |
 
-
 |             | TensorFlow 2.12                                                                                                                                                                                                        | TensorFlow 2.13                                                                                                                                                                                                        | TensorFlow 2.14                                                                                                                                                                                                        | TensorFlow 2.15                                                                                                                                                                                                        |
 |-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | Python 3.9  | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras212.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras212.yml)   | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras213.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras213.yml)   | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras214.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras214.yml)   | [![Run Tests](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras215.yml/badge.svg)](https://github.com/sony/model_optimization/actions/workflows/run_tests_python39_keras215.yml)   |

model_compression_toolkit/core/common/hessian/hessian_info_service.py

@@ -204,7 +204,7 @@ def _fetch_hessians_with_compute(self, request: HessianScoresRequest, n_iteratio
             target_nodes = [n for n in orig_request.target_nodes if n.name in missing]
             request = request.clone(target_nodes=target_nodes)
         self._compute_hessians(request, n_iterations, count_by_cache=True)
-        res, missing = self.cache.fetch_hessian(request)
+        res, missing = self.cache.fetch_hessian(orig_request)
         assert not missing
         return res
 

model_compression_toolkit/gptq/common/gptq_config.py

@@ -16,8 +16,7 @@
 from enum import Enum
 from typing import Callable, Any, Dict, Optional
 
-from model_compression_toolkit.constants import GPTQ_HESSIAN_NUM_SAMPLES, ACT_HESSIAN_DEFAULT_BATCH_SIZE
-from model_compression_toolkit.gptq.common.gptq_constants import REG_DEFAULT
+from model_compression_toolkit.constants import ACT_HESSIAN_DEFAULT_BATCH_SIZE
 
 
 class RoundingType(Enum):
@@ -39,20 +38,26 @@ class GPTQHessianScoresConfig:
     Configuration to use for computing the Hessian-based scores for GPTQ loss metric.
 
     Args:
+        per_sample (bool): Whether to use per sample attention score.
         hessians_num_samples (int|None): Number of samples to use for computing the Hessian-based scores.
           If None, compute Hessian for all images.
         norm_scores (bool): Whether to normalize the returned scores of the weighted loss function (to get values between 0 and 1).
         log_norm (bool): Whether to use log normalization for the GPTQ Hessian-based scores.
         scale_log_norm (bool): Whether to scale the final vector of the Hessian-based scores.
         hessian_batch_size (int): The Hessian computation batch size. used only if using GPTQ with Hessian-based objective.
-        per_sample (bool): Whether to use per sample attention score.
     """
-    hessians_num_samples: Optional[int] = GPTQ_HESSIAN_NUM_SAMPLES
-    norm_scores: bool = True
-    log_norm: bool = True
+    per_sample: bool
+    hessians_num_samples: Optional[int]
+    norm_scores: bool = None
+    log_norm: bool = None
     scale_log_norm: bool = False
     hessian_batch_size: int = ACT_HESSIAN_DEFAULT_BATCH_SIZE
-    per_sample: bool = False
+
+    def __post_init__(self):
+        if self.norm_scores is None:
+            self.norm_scores = not self.per_sample
+        if self.log_norm is None:
+            self.log_norm = not self.per_sample
 
 
 @dataclass
@@ -107,32 +112,30 @@ class GradientPTQConfig:
 
     Args:
         n_epochs: Number of representative dataset epochs to train.
-        optimizer: Optimizer to use.
-        optimizer_rest: Optimizer to use for bias and quantizer parameters.
         loss: The loss to use. See 'multiple_tensors_mse_loss' for the expected interface.
-        log_function: Function to log information about the GPTQ process.
+        optimizer: Optimizer to use.
+        optimizer_rest: Default optimizer to use for bias and quantizer parameters.
         train_bias: Whether to update the bias during the training or not.
-        rounding_type: An enum that defines the rounding type.
-        use_hessian_based_weights: Whether to use Hessian-based weights for weighted average loss.
-        optimizer_quantization_parameter: Optimizer to override the rest optimizer  for quantizer parameters.
-        optimizer_bias: Optimizer to override the rest optimizer for bias.
-        regularization_factor: A floating point number that defines the regularization factor.
         hessian_weights_config: A configuration that include all necessary arguments to run a computation of
             Hessian scores for the GPTQ loss.
         gradual_activation_quantization_config: A configuration for Gradual Activation Quantization.
+        regularization_factor: A floating point number that defines the regularization factor.
+        rounding_type: An enum that defines the rounding type.
+        optimizer_quantization_parameter: Optimizer to override the rest optimizer for quantizer parameters.
+        optimizer_bias: Optimizer to override the rest optimizer for bias.
+        log_function: Function to log information about the GPTQ process.
         gptq_quantizer_params_override: A dictionary of parameters to override in GPTQ quantizer instantiation.
     """
     n_epochs: int
+    loss: Callable
     optimizer: Any
-    optimizer_rest: Any = None
-    loss: Callable = None
-    log_function: Callable = None
-    train_bias: bool = True
+    optimizer_rest: Any
+    train_bias: bool
+    hessian_weights_config: Optional[GPTQHessianScoresConfig]
+    gradual_activation_quantization_config: Optional[GradualActivationQuantizationConfig]
+    regularization_factor: float
     rounding_type: RoundingType = RoundingType.SoftQuantizer
-    use_hessian_based_weights: bool = True
     optimizer_quantization_parameter: Any = None
     optimizer_bias: Any = None
-    regularization_factor: float = REG_DEFAULT
-    hessian_weights_config: GPTQHessianScoresConfig = field(default_factory=GPTQHessianScoresConfig)
-    gradual_activation_quantization_config: Optional[GradualActivationQuantizationConfig] = None
+    log_function: Callable = None
     gptq_quantizer_params_override: Dict[str, Any] = field(default_factory=dict)

model_compression_toolkit/gptq/common/gptq_constants.py

@@ -14,6 +14,7 @@
 MIM_TEMP = 0.5
 MAX_TEMP = 1.0
 REG_DEFAULT = 0.01
+REG_DEFAULT_SLA = 10
 MAX_LSB_CHANGE = 1
 
 # Soft rounding arguments values
@@ -27,6 +28,5 @@
 # GPTQ learning hyperparameters
 LR_DEFAULT = 3e-2
 LR_REST_DEFAULT = 1e-4
-LR_BIAS_DEFAULT = 1e-3
-LR_QUANTIZATION_PARAM_DEFAULT = 1e-3
+LR_BIAS_DEFAULT = 1e-4
 GPTQ_MOMENTUM = 0.9

model_compression_toolkit/gptq/common/gptq_training.py

@@ -75,7 +75,7 @@ def __init__(self,
                                                                        fw_info=self.fw_info)
 
         self.fxp_model, self.gptq_user_info = self.build_gptq_model()
-        if self.gptq_config.use_hessian_based_weights:
+        if self.gptq_config.hessian_weights_config:
             if not isinstance(hessian_info_service, HessianInfoService):
                 Logger.critical(f"When using Hessian-based approximations for sensitivity evaluation, "
                                 f"an 'HessianInfoService' object must be provided, but received: {hessian_info_service}.")   # pragma: no cover