Score changes for default - failed responses punished more heavily (#18)

* Score changes for default - failed responses punished more heavily

* Turn down midjourney just a little

validation/validator_api_server/constants.py

@@ -58,8 +58,8 @@
 
 SCORE_QUERY_PROBABILITY = 0.01
 NUMBER_OF_SECONDARY_AXONS_TO_COMPARE_WHEN_SCORING = 1  # head - head comparison
-BONUS_FOR_WINNING_MINER = 0.2
-FAILED_RESPONSE_SCORE = 0.5
+BONUS_FOR_WINNING_MINER = 0.25
+FAILED_RESPONSE_SCORE = 0.25
 
 CHANCE_TO_CHECK_OUTPUT_WHEN_IMAGES_FROM_MINERS_WERE_SIMILAR = 0.02
 

validation/validator_api_server/core_validator.py

@@ -149,7 +149,9 @@ async def _store_synthetic_result(self, sota_request: utility_models.SotaCheckin
                     },
                 )
         except Exception as e:
-            bt.logging.debug(f"Error when storing sota image: {e}. Doesn't matter too much, unless this repeatedly happens")
+            bt.logging.debug(
+                f"Error when storing sota image: {e}. Doesn't matter too much, unless this repeatedly happens"
+            )
 
     async def _query_checking_server_for_expected_result(
         self, endpoint: str, synapse: bt.Synapse, outgoing_model: BaseModel
@@ -222,9 +224,7 @@ async def synthetically_score(self) -> None:
         while True:
             # Weight Sota slightly higher so we do Sota 22% of the time, non Sota 78% of the time
             operation = random.choices(
-                cst.OPERATIONS_TO_SCORE_SYNTHETICALLY,
-                weights=[0.13, 0.13, 0.13, 0.13, 0.13, 0.13, 0.16],
-                k=1
+                cst.OPERATIONS_TO_SCORE_SYNTHETICALLY, weights=[0.13, 0.13, 0.13, 0.13, 0.13, 0.13, 0.13], k=1
             )[0]
             synthetic_data = await self._query_checking_server_for_synthetic_data(operation)
 
@@ -780,7 +780,7 @@ def set_weights(self):
             bt.logging.info("No uids found to score, nothing to set")
             return
         for uid, periods_for_uid in scoring_periods_uid_was_in.items():
-            scores = uid_scores.get(uid, [0.5])
+            scores = uid_scores.get(uid, [cst.FAILED_RESPONSE_SCORE])
             average_score = sum(scores) / len(scores)
 
             uid_weights[uid] = average_score * (periods_for_uid / max_periods) ** 0.5

validation/validator_checking_server/utils.py

@@ -8,7 +8,7 @@
 import cv2
 import diskcache
 import numpy as np
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 from validation.validator_api_server import similarity_comparisons
 import httpx
 from core import constants as cst
@@ -48,8 +48,10 @@ async def is_sota_image_valid(image_url: str, prompt: str) -> bool:
     clip_model, clip_processor = resource_management.SingletonResourceManager().get_resource(cst.MODEL_CLIP)
     clip_device = resource_management.SingletonResourceManager()._config.get(cst.MODEL_CLIP)
 
-
-    original_image = Image.open(io.BytesIO(image_bytes))
+    try:
+        original_image = Image.open(io.BytesIO(image_bytes))
+    except UnidentifiedImageError:
+        bt.logging.warning(f"Error when fetching image {image_url}, can't parse the bytes into a PIL for some reason")
 
     width, height = original_image.size
     image_1 = original_image.crop((0, 0, width // 2, height // 2))