feature-extraction-rsna2024-sumaniitm.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":8236972,"sourceType":"datasetVersion","datasetId":4885707},{"sourceId":9155320,"sourceType":"datasetVersion","datasetId":5530686},{"sourceId":99960,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":83858,"modelId":108105}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import required libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport polars as pl\n#import duckdb as dd\n#from tqdm import tqdm\nimport matplotlib.pyplot as plt\n#import cv2\n#from pydicom import dcmread\nimport warnings\n#from sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import log_loss\nimport pickle\n#import gc\nimport ctypes\nimport tensorflow as tf\n#tf.compat.v1.disable_eager_execution()\n#tf.keras.backend.clear_session()\n\n\"\"\"for gpu in tf.config.experimental.list_physical_devices(\"GPU\"):\n    tf.config.experimental.set_memory_growth(gpu, True)\"\"\"\n    \nimport tensorflow_io as tfio\nfrom tensorflow import keras\n\n#tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-04T04:21:20.312431Z","iopub.execute_input":"2024-09-04T04:21:20.312822Z","iopub.status.idle":"2024-09-04T04:21:33.641346Z","shell.execute_reply.started":"2024-09-04T04:21:20.312783Z","shell.execute_reply":"2024-09-04T04:21:33.640527Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:21:43.793957Z","iopub.execute_input":"2024-09-04T04:21:43.795336Z","iopub.status.idle":"2024-09-04T04:21:44.837019Z","shell.execute_reply.started":"2024-09-04T04:21:43.795282Z","shell.execute_reply":"2024-09-04T04:21:44.836009Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Number of accelerators:  2\n","output_type":"stream"}]},{"cell_type":"code","source":"# strategy = tf.distribute.MirroredStrategy()\nprint('DEVICES AVAILABLE: {}'.format(strategy.num_replicas_in_sync))\n\nBATCH_SIZE_PER_REPLICA = 48\n\n#We obtain the BATCH_SIZE dividing by the number of devices. \n#BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\nBATCH_SIZE = BATCH_SIZE_PER_REPLICA * 2","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:21:52.690312Z","iopub.execute_input":"2024-09-04T04:21:52.691029Z","iopub.status.idle":"2024-09-04T04:21:52.696333Z","shell.execute_reply.started":"2024-09-04T04:21:52.690988Z","shell.execute_reply":"2024-09-04T04:21:52.695289Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"DEVICES AVAILABLE: 2\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Feature extraction from image files\n## Only if not using tensorflow\n### kept here as backward compatibility with earlier work","metadata":{}},{"cell_type":"code","source":"metadata_root_path = '/kaggle/input/spinal-canal-stenosis-metadata'\n\nfor file in os.listdir(metadata_root_path):\n    labels = []\n    features = []\n    data = {}\n    metadata_file_path = os.path.join(metadata_root_path, file)\n    print(metadata_file_path)\n    metadata_df = pl.read_csv(metadata_file_path, low_memory=True)\n    for j in tqdm(range(metadata_df.shape[0])):\n        dcm_image_path = metadata_df.item(j,0)\n        dicom_ds = dcmread(dcm_image_path)\n        img_array = dicom_ds.pixel_array\n        features.append(np.mean(img_array.T, axis=0))\n        # features.append(dcmread(dcm_image_path).pixel_array)\n        labels.append(metadata_df.item(j,2))\n\n    print(\"feature list length --> \", len(features))\n    print(\"label list length --> \", len(labels))\n\n    #extracted_training_features = np.array([np.resize(img,(128,128)) for img in features])\n    #training_labels = np.array(labels)\n\n    #del labels\n    #del features\n    #gc.collect()\n\n    extracted_training_features_file_name = \"{0}_training_features\".format(file.replace('_feature_metadata.csv',''))\n    labels_file_name = \"{0}_labels\".format(file.replace('.csv',''))\n\n    with open(extracted_training_features_file_name, \"wb\") as file:\n        pickle.dump(features, file)\n    with open(labels_file_name, \"wb\") as file:\n        pickle.dump(labels, file)\n\n    del labels\n    del features\n    gc.collect()\n    libc = ctypes.CDLL(\"libc.so.6\") # clearing cache \n    libc.malloc_trim(0)\n\n    #data[\"image_array\"] = features\n    #data[\"encoded_severity\"] = labels\n    #extracted_training_data = pd.DataFrame(data)\n    #extracted_training_data.to_csv(\"{0}.csv.gz\".format(file.replace('.csv','')), index=False, compression='gzip')\n\n    print('finished dumping features & labels for {0}'.format(file))","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Review pipeline with one sample image","metadata":{}},{"cell_type":"code","source":"image_bytes \\\n= tf.io.read_file('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/1.dcm')\n\nimage = tfio.image.decode_dicom_image(image_bytes, scale='auto', dtype=tf.float32)\n\nm, M=tf.math.reduce_min(image), tf.math.reduce_max(image)\nimage = (tf.image.grayscale_to_rgb(image)-m)/(M-m)\nimage = tf.image.resize(image, (128,128))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing functions","metadata":{}},{"cell_type":"code","source":"def read_and_parse_dicom_files(full_file_path):\n    tf.config.run_functions_eagerly(True)\n    raw_image = tf.io.read_file(full_file_path)\n    sp = tf.strings.split(tf.gather(tf.strings.split(full_file_path, 'images/'), 1), '/')\n    N = tf.size(sp)\n    LEN = tf.strings.length(tf.gather(sp, 0))+tf.strings.length(tf.gather(sp, 2))\n    \n    # Add missing file metadata to avoid warnnigs flooding\n    if   LEN==12: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x92\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==13: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x92\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==14: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x94\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==15: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x94\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==16: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x96\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==17: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x96\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==18: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x98\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    \n    #image_bytes = tf.io.read_file(full_file_path)\n    #image = tfio.image.decode_dicom_image(image_bytes, scale='auto', dtype=tf.float32)\n    image = tfio.image.decode_dicom_image(raw_image, scale='auto', dtype=tf.float32)\n    m, M=tf.math.reduce_min(image), tf.math.reduce_max(image)\n    image = (tf.image.grayscale_to_rgb(image)-m)/(M-m)\n    image = tf.image.resize(image, (128,128))\n    return tf.squeeze(image)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:21:58.548524Z","iopub.execute_input":"2024-09-04T04:21:58.549251Z","iopub.status.idle":"2024-09-04T04:21:58.560458Z","shell.execute_reply.started":"2024-09-04T04:21:58.549211Z","shell.execute_reply":"2024-09-04T04:21:58.559477Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"def load_dataset(image_path, labels):\n    image = read_and_parse_dicom_files(image_path)\n    return {\"images\": tf.cast(image, tf.float32), \"labels\": tf.cast(labels, tf.float32)}\n\ndef dict_to_tuple(inputs):\n    return inputs[\"images\"], inputs[\"labels\"]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:00.289684Z","iopub.execute_input":"2024-09-04T04:22:00.290145Z","iopub.status.idle":"2024-09-04T04:22:00.295656Z","shell.execute_reply.started":"2024-09-04T04:22:00.290097Z","shell.execute_reply":"2024-09-04T04:22:00.294551Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"# Train, Test, Validation & holdout splits\n## holdout set to be used for CV","metadata":{}},{"cell_type":"code","source":"metadata_file_path = '/kaggle/input/spinal-canal-stenosis-metadata/spinal_canal_stenosis_l1_l2_feature_metadata.csv'\nmetadata_df = pl.read_csv(metadata_file_path, low_memory=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:05.467135Z","iopub.execute_input":"2024-09-04T04:22:05.467520Z","iopub.status.idle":"2024-09-04T04:22:05.664430Z","shell.execute_reply.started":"2024-09-04T04:22:05.467482Z","shell.execute_reply":"2024-09-04T04:22:05.663626Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"metadata_df.columns","metadata":{"execution":{"iopub.status.busy":"2024-09-02T18:01:47.797769Z","iopub.execute_input":"2024-09-02T18:01:47.798526Z","iopub.status.idle":"2024-09-02T18:01:47.806604Z","shell.execute_reply.started":"2024-09-02T18:01:47.798486Z","shell.execute_reply":"2024-09-02T18:01:47.805628Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"['full_img_path', 'severity', 'encoded_severity']"},"metadata":{}}]},{"cell_type":"code","source":"\"\"\"def attach_weights(label):\n    if label == 'Normal/Mild':\n        return 1\n    elif label == 'Severe':\n        return 4\n    else:\n        return 2\n    \nmetadata_df = metadata_df.with_columns([\n    pl.col(\"severity\").map_elements(attach_weights, return_dtype=pl.Int32).alias(\"sample_weight\")\n])\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-09-02T16:30:51.977775Z","iopub.execute_input":"2024-09-02T16:30:51.978744Z","iopub.status.idle":"2024-09-02T16:30:52.097069Z","shell.execute_reply.started":"2024-09-02T16:30:51.978701Z","shell.execute_reply":"2024-09-02T16:30:52.096362Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"for_train, holdout = train_test_split(metadata_df, test_size=0.4, random_state=42)\n\nx_train, x_test_val = train_test_split(for_train, test_size=0.3, random_state=42)\nx_test, x_valid = train_test_split(x_test_val, test_size=0.2, random_state=42)\n\nprint(\"Training data shape : {0}\".format(x_train.shape))\nprint(\"Test data shape : {0}\".format(x_test.shape))\nprint(\"Validation data shape : {0}\".format(x_valid.shape))\nprint(\"Holdout data shape : {0}\".format(holdout.shape))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:11.484814Z","iopub.execute_input":"2024-09-04T04:22:11.485201Z","iopub.status.idle":"2024-09-04T04:22:11.555958Z","shell.execute_reply.started":"2024-09-04T04:22:11.485166Z","shell.execute_reply":"2024-09-04T04:22:11.554937Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"Training data shape : (61794, 3)\nTest data shape : (21187, 3)\nValidation data shape : (5297, 3)\nHoldout data shape : (58853, 3)\n","output_type":"stream"}]},{"cell_type":"code","source":"holdout_image_filenames = pl.Series(holdout.select(pl.col('full_img_path'))).to_list()\nholdout_image_labels = pl.Series(holdout.select(pl.col('encoded_severity'))).to_list()\n#holdout_weights = pl.Series(holdout.select(pl.col('sample_weight'))).to_list()\n\ntrain_image_filenames = pl.Series(x_train.select(pl.col('full_img_path'))).to_list()\ntrain_image_labels = pl.Series(x_train.select(pl.col('encoded_severity'))).to_list()\n#train_weights = pl.Series(x_train.select(pl.col('sample_weight'))).to_list()\n\ntest_image_filenames = pl.Series(x_test.select(pl.col('full_img_path'))).to_list()\ntest_image_labels = pl.Series(x_test.select(pl.col('encoded_severity'))).to_list()\n#test_weights = pl.Series(x_test.select(pl.col('sample_weight'))).to_list()\n\nvalid_image_filenames = pl.Series(x_valid.select(pl.col('full_img_path'))).to_list()\nvalid_image_labels = pl.Series(x_valid.select(pl.col('encoded_severity'))).to_list()\n#valid_weights = pl.Series(x_valid.select(pl.col('sample_weight'))).to_list()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:17.423951Z","iopub.execute_input":"2024-09-04T04:22:17.424321Z","iopub.status.idle":"2024-09-04T04:22:17.501670Z","shell.execute_reply.started":"2024-09-04T04:22:17.424287Z","shell.execute_reply":"2024-09-04T04:22:17.500699Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"train_dataset = tf.data.Dataset.from_tensor_slices((train_image_filenames, train_image_labels))\n\ntest_dataset = tf.data.Dataset.from_tensor_slices((test_image_filenames, test_image_labels))\n\nvalid_dataset = tf.data.Dataset.from_tensor_slices((valid_image_filenames, valid_image_labels))\n\nholdout_dataset = tf.data.Dataset.from_tensor_slices((holdout_image_filenames, holdout_image_labels))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:20.129199Z","iopub.execute_input":"2024-09-04T04:22:20.129952Z","iopub.status.idle":"2024-09-04T04:22:21.080543Z","shell.execute_reply.started":"2024-09-04T04:22:20.129888Z","shell.execute_reply":"2024-09-04T04:22:21.079725Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"train_ds = train_dataset.map(load_dataset, num_parallel_calls=tf.data.AUTOTUNE)\ntrain_ds = train_ds.map(dict_to_tuple, num_parallel_calls=tf.data.AUTOTUNE)\ntrain_ds = train_ds.batch(batch_size=BATCH_SIZE, drop_remainder=True)\ntrain_ds = train_ds.prefetch(tf.data.AUTOTUNE)\n\ntest_ds = test_dataset.map(load_dataset, num_parallel_calls=tf.data.AUTOTUNE)\ntest_ds = test_ds.map(dict_to_tuple, num_parallel_calls=tf.data.AUTOTUNE)\ntest_ds = test_ds.batch(batch_size=BATCH_SIZE, drop_remainder=True)\ntest_ds = test_ds.prefetch(tf.data.AUTOTUNE)\n\nvalid_ds = valid_dataset.map(load_dataset, num_parallel_calls=tf.data.AUTOTUNE)\nvalid_ds = valid_ds.map(dict_to_tuple, num_parallel_calls=tf.data.AUTOTUNE)\nvalid_ds = valid_ds.batch(batch_size=BATCH_SIZE, drop_remainder=True)\nvalid_ds = valid_ds.prefetch(tf.data.AUTOTUNE)\n\nholdout_ds = holdout_dataset.map(load_dataset, num_parallel_calls=tf.data.AUTOTUNE)\nholdout_ds = holdout_ds.map(dict_to_tuple, num_parallel_calls=tf.data.AUTOTUNE)\nholdout_ds = holdout_ds.batch(batch_size=BATCH_SIZE, drop_remainder=True)\nholdout_ds = holdout_ds.prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:23.535265Z","iopub.execute_input":"2024-09-04T04:22:23.536023Z","iopub.status.idle":"2024-09-04T04:22:25.280348Z","shell.execute_reply.started":"2024-09-04T04:22:23.535982Z","shell.execute_reply":"2024-09-04T04:22:25.279311Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/tensorflow/python/data/ops/structured_function.py:258: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n  warnings.warn(\n","output_type":"stream"}]},{"cell_type":"code","source":"elem = next(iter(train_ds))\ntype(elem)","metadata":{"execution":{"iopub.status.busy":"2024-09-02T18:02:47.583975Z","iopub.execute_input":"2024-09-02T18:02:47.584796Z","iopub.status.idle":"2024-09-02T18:02:48.726882Z","shell.execute_reply.started":"2024-09-02T18:02:47.584756Z","shell.execute_reply":"2024-09-02T18:02:48.726071Z"},"trusted":true},"execution_count":12,"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"tuple"},"metadata":{}}]},{"cell_type":"code","source":"elem[1]","metadata":{"execution":{"iopub.status.busy":"2024-09-02T18:02:52.591773Z","iopub.execute_input":"2024-09-02T18:02:52.592443Z","iopub.status.idle":"2024-09-02T18:02:52.601414Z","shell.execute_reply.started":"2024-09-02T18:02:52.592401Z","shell.execute_reply":"2024-09-02T18:02:52.600388Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"<tf.Tensor: shape=(96,), dtype=float32, numpy=\narray([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n       1., 1., 1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 2., 1., 1.,\n       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n       1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], dtype=float32)>"},"metadata":{}}]},{"cell_type":"code","source":"tf.constant(np.ones(48))","metadata":{"execution":{"iopub.status.busy":"2024-09-02T18:09:15.937378Z","iopub.execute_input":"2024-09-02T18:09:15.938282Z","iopub.status.idle":"2024-09-02T18:09:15.945696Z","shell.execute_reply.started":"2024-09-02T18:09:15.938239Z","shell.execute_reply":"2024-09-02T18:09:15.944733Z"},"trusted":true},"execution_count":22,"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"<tf.Tensor: shape=(48,), dtype=float64, numpy=\narray([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n       1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])>"},"metadata":{}}]},{"cell_type":"markdown","source":"#### Create a solution dataframe with predictions as 1 or 0 depending on the labels\n#### using this solution dataframe and the actual predictions and sample weights (given in the problem statement)\n#### build a custom keras metric","metadata":{}},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=1000)\nx_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-08-29T04:50:27.480296Z","iopub.execute_input":"2024-08-29T04:50:27.480727Z","iopub.status.idle":"2024-08-29T04:50:27.503403Z","shell.execute_reply.started":"2024-08-29T04:50:27.480685Z","shell.execute_reply":"2024-08-29T04:50:27.501993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_study_id(full_image_path):\n    return full_image_path.split('/')[-3]\n\nx_train_studies = x_train.with_columns(\n    pl.col(\"full_img_path\")\n    .map_elements(get_study_id, return_dtype=pl.String)\n    .alias(\"study_id\")\n)\nx_train_studies.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-08-28T04:55:10.255514Z","iopub.execute_input":"2024-08-28T04:55:10.256076Z","iopub.status.idle":"2024-08-28T04:55:10.321302Z","shell.execute_reply.started":"2024-08-28T04:55:10.256024Z","shell.execute_reply":"2024-08-28T04:55:10.320254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.select([pl.col('severity'), pl.col('encoded_severity')]).unique()","metadata":{"execution":{"iopub.status.busy":"2024-08-29T04:50:36.370773Z","iopub.execute_input":"2024-08-29T04:50:36.371255Z","iopub.status.idle":"2024-08-29T04:50:36.429731Z","shell.execute_reply.started":"2024-08-29T04:50:36.371206Z","shell.execute_reply":"2024-08-29T04:50:36.428044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.select(pl.col('encoded_severity')).unique().to_series().to_list()","metadata":{"execution":{"iopub.status.busy":"2024-08-29T05:25:53.879991Z","iopub.execute_input":"2024-08-29T05:25:53.881519Z","iopub.status.idle":"2024-08-29T05:25:53.892644Z","shell.execute_reply.started":"2024-08-29T05:25:53.881400Z","shell.execute_reply":"2024-08-29T05:25:53.891178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Code to create custom metric aligned with Competition evaluation criteria","metadata":{}},{"cell_type":"code","source":"train_studies_metadata_file_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv'\ntrain_studies_metadata_df = pl.read_csv(train_studies_metadata_file_path, low_memory=True)\nprint(\"before dropping nulls :\", train_studies_metadata_df.shape)\ntrain_studies_metadata_df = train_studies_metadata_df.drop_nulls()\nprint(\"after dropping nulls :\", train_studies_metadata_df.shape)\n\nstudies_full = train_studies_metadata_df.select(pl.col('study_id')).unique().to_series().to_list()\nprint(len(studies_full))\n#train_studies_metadata_df.select([pl.col('study_id'),pl.col('spinal_canal_stenosis_l1_l2')]).head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:39.657093Z","iopub.execute_input":"2024-09-04T04:22:39.657464Z","iopub.status.idle":"2024-09-04T04:22:39.703510Z","shell.execute_reply.started":"2024-09-04T04:22:39.657429Z","shell.execute_reply":"2024-09-04T04:22:39.702517Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stdout","text":"before dropping nulls : (1975, 26)\nafter dropping nulls : (1790, 26)\n1790\n","output_type":"stream"}]},{"cell_type":"code","source":"config = {}\nconfig['root_file_path'] = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\nconfig['start'] = 10\nconfig['end'] = 110\n#studies = os.listdir(config['root_file_path'])[config['start']:config['end']]\nstudies = studies_full[config['start']:config['end']]\nprint(len(studies))\n\ntest_dict = {}\nfor study in studies:\n    image_files = []\n    for dirname, _, filenames in os.walk(config['root_file_path']+'/'+str(study)):\n        for filename in filenames:\n            test_dict[os.path.join(dirname, filename).split('/')[-3]] = image_files\n            image_files.append(os.path.join(dirname, filename))\n            \nprint(len(test_dict))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:43.248093Z","iopub.execute_input":"2024-09-04T04:22:43.248523Z","iopub.status.idle":"2024-09-04T04:22:45.243877Z","shell.execute_reply.started":"2024-09-04T04:22:43.248482Z","shell.execute_reply":"2024-09-04T04:22:45.242946Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"100\n100\n","output_type":"stream"}]},{"cell_type":"code","source":"def label_encoder(label):\n    if label == 'Normal/Mild':\n        return 2\n    elif label == 'Severe':\n        return 3\n    else:\n        return 1\n    \ndef attach_weights(label):\n    if label == 'Normal/Mild':\n        return 1\n    elif label == 'Severe':\n        return 4\n    else:\n        return 2\n\ntrain_studies_metadata_df_up = train_studies_metadata_df.unpivot(index=\"study_id\")\ntrain_studies_metadata_df_up.columns = ['study_id', 'condition', 'severity']\n\ntrain_studies_metadata_df_up = train_studies_metadata_df_up.with_columns([\n    pl.col(\"severity\").map_elements(label_encoder, return_dtype=pl.Int32).alias(\"encoded_severity\"),\n    pl.col(\"severity\").map_elements(attach_weights, return_dtype=pl.Int32).alias(\"sample_weight\"),\n    (pl.col(\"study_id\").cast(pl.String)+'_'+pl.col(\"condition\")).alias(\"row_id\")\n])\n\nprint(train_studies_metadata_df_up.shape)\ntrain_studies_metadata_df_up.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:51.011504Z","iopub.execute_input":"2024-09-04T04:22:51.012389Z","iopub.status.idle":"2024-09-04T04:22:51.077906Z","shell.execute_reply.started":"2024-09-04T04:22:51.012346Z","shell.execute_reply":"2024-09-04T04:22:51.076976Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stdout","text":"(44750, 6)\n","output_type":"stream"},{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"shape: (10, 6)\n┌──────────┬───────────────────┬─────────────┬──────────────────┬───────────────┬──────────────────┐\n│ study_id ┆ condition         ┆ severity    ┆ encoded_severity ┆ sample_weight ┆ row_id           │\n│ ---      ┆ ---               ┆ ---         ┆ ---              ┆ ---           ┆ ---              │\n│ i64      ┆ str               ┆ str         ┆ i32              ┆ i32           ┆ str              │\n╞══════════╪═══════════════════╪═════════════╪══════════════════╪═══════════════╪══════════════════╡\n│ 4003253  ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 4003253_spinal_c │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ anal_stenosis_…  │\n│ 4646740  ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 4646740_spinal_c │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ anal_stenosis_…  │\n│ 7143189  ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 7143189_spinal_c │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ anal_stenosis_…  │\n│ 8785691  ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 8785691_spinal_c │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ anal_stenosis_…  │\n│ 10728036 ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 10728036_spinal_ │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ canal_stenosis…  │\n│ 11340341 ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 11340341_spinal_ │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ canal_stenosis…  │\n│ 11943292 ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 11943292_spinal_ │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ canal_stenosis…  │\n│ 13317052 ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 13317052_spinal_ │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ canal_stenosis…  │\n│ 22191399 ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 22191399_spinal_ │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ canal_stenosis…  │\n│ 26342422 ┆ spinal_canal_sten ┆ Normal/Mild ┆ 2                ┆ 1             ┆ 26342422_spinal_ │\n│          ┆ osis_l1_l2        ┆             ┆                  ┆               ┆ canal_stenosis…  │\n└──────────┴───────────────────┴─────────────┴──────────────────┴───────────────┴──────────────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (10, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>condition</th><th>severity</th><th>encoded_severity</th><th>sample_weight</th><th>row_id</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>i32</td><td>i32</td><td>str</td></tr></thead><tbody><tr><td>4003253</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;4003253_spinal_canal_stenosis_…</td></tr><tr><td>4646740</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;4646740_spinal_canal_stenosis_…</td></tr><tr><td>7143189</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;7143189_spinal_canal_stenosis_…</td></tr><tr><td>8785691</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;8785691_spinal_canal_stenosis_…</td></tr><tr><td>10728036</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;10728036_spinal_canal_stenosis…</td></tr><tr><td>11340341</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;11340341_spinal_canal_stenosis…</td></tr><tr><td>11943292</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;11943292_spinal_canal_stenosis…</td></tr><tr><td>13317052</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;13317052_spinal_canal_stenosis…</td></tr><tr><td>22191399</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;22191399_spinal_canal_stenosis…</td></tr><tr><td>26342422</td><td>&quot;spinal_canal_stenosis_l1_l2&quot;</td><td>&quot;Normal/Mild&quot;</td><td>2</td><td>1</td><td>&quot;26342422_spinal_canal_stenosis…</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"temp = train_studies_metadata_df_up\\\n.select([pl.col('study_id'), pl.col('row_id'), pl.col('encoded_severity'), pl.col('severity'), pl.col('sample_weight')])\ntemp.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:55.309556Z","iopub.execute_input":"2024-09-04T04:22:55.310004Z","iopub.status.idle":"2024-09-04T04:22:55.318802Z","shell.execute_reply.started":"2024-09-04T04:22:55.309962Z","shell.execute_reply":"2024-09-04T04:22:55.317767Z"},"trusted":true},"execution_count":14,"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"shape: (10, 5)\n┌──────────┬─────────────────────────────────┬──────────────────┬─────────────┬───────────────┐\n│ study_id ┆ row_id                          ┆ encoded_severity ┆ severity    ┆ sample_weight │\n│ ---      ┆ ---                             ┆ ---              ┆ ---         ┆ ---           │\n│ i64      ┆ str                             ┆ i32              ┆ str         ┆ i32           │\n╞══════════╪═════════════════════════════════╪══════════════════╪═════════════╪═══════════════╡\n│ 4003253  ┆ 4003253_spinal_canal_stenosis_… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 4646740  ┆ 4646740_spinal_canal_stenosis_… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 7143189  ┆ 7143189_spinal_canal_stenosis_… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 8785691  ┆ 8785691_spinal_canal_stenosis_… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 10728036 ┆ 10728036_spinal_canal_stenosis… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 11340341 ┆ 11340341_spinal_canal_stenosis… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 11943292 ┆ 11943292_spinal_canal_stenosis… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 13317052 ┆ 13317052_spinal_canal_stenosis… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 22191399 ┆ 22191399_spinal_canal_stenosis… ┆ 2                ┆ Normal/Mild ┆ 1             │\n│ 26342422 ┆ 26342422_spinal_canal_stenosis… ┆ 2                ┆ Normal/Mild ┆ 1             │\n└──────────┴─────────────────────────────────┴──────────────────┴─────────────┴───────────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (10, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>row_id</th><th>encoded_severity</th><th>severity</th><th>sample_weight</th></tr><tr><td>i64</td><td>str</td><td>i32</td><td>str</td><td>i32</td></tr></thead><tbody><tr><td>4003253</td><td>&quot;4003253_spinal_canal_stenosis_…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>4646740</td><td>&quot;4646740_spinal_canal_stenosis_…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>7143189</td><td>&quot;7143189_spinal_canal_stenosis_…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>8785691</td><td>&quot;8785691_spinal_canal_stenosis_…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>10728036</td><td>&quot;10728036_spinal_canal_stenosis…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>11340341</td><td>&quot;11340341_spinal_canal_stenosis…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>11943292</td><td>&quot;11943292_spinal_canal_stenosis…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>13317052</td><td>&quot;13317052_spinal_canal_stenosis…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>22191399</td><td>&quot;22191399_spinal_canal_stenosis…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>26342422</td><td>&quot;26342422_spinal_canal_stenosis…</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"train_studies_metadata_df_final = temp.pivot(\"severity\", index=[\"study_id\",\"row_id\"], values=\"encoded_severity\")\n#.select([pl.col('study_id'), pl.col('row_id'), pl.col('encoded_severity'), pl.col('severity')])\\\n#.pivot(\"severity\", index=[\"study_id\",\"row_id\"], values=\"encoded_severity\")\n\ntrain_studies_metadata_df_final.columns = ['study_id', 'row_id', 'normal_mild', 'moderate', 'severe']\n\ntrain_studies_metadata_df_final.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:22:59.705830Z","iopub.execute_input":"2024-09-04T04:22:59.706853Z","iopub.status.idle":"2024-09-04T04:22:59.753725Z","shell.execute_reply.started":"2024-09-04T04:22:59.706801Z","shell.execute_reply":"2024-09-04T04:22:59.752725Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"shape: (10, 5)\n┌──────────┬─────────────────────────────────┬─────────────┬──────────┬────────┐\n│ study_id ┆ row_id                          ┆ normal_mild ┆ moderate ┆ severe │\n│ ---      ┆ ---                             ┆ ---         ┆ ---      ┆ ---    │\n│ i64      ┆ str                             ┆ i32         ┆ i32      ┆ i32    │\n╞══════════╪═════════════════════════════════╪═════════════╪══════════╪════════╡\n│ 4003253  ┆ 4003253_spinal_canal_stenosis_… ┆ 2           ┆ null     ┆ null   │\n│ 4646740  ┆ 4646740_spinal_canal_stenosis_… ┆ 2           ┆ null     ┆ null   │\n│ 7143189  ┆ 7143189_spinal_canal_stenosis_… ┆ 2           ┆ null     ┆ null   │\n│ 8785691  ┆ 8785691_spinal_canal_stenosis_… ┆ 2           ┆ null     ┆ null   │\n│ 10728036 ┆ 10728036_spinal_canal_stenosis… ┆ 2           ┆ null     ┆ null   │\n│ 11340341 ┆ 11340341_spinal_canal_stenosis… ┆ 2           ┆ null     ┆ null   │\n│ 11943292 ┆ 11943292_spinal_canal_stenosis… ┆ 2           ┆ null     ┆ null   │\n│ 13317052 ┆ 13317052_spinal_canal_stenosis… ┆ 2           ┆ null     ┆ null   │\n│ 22191399 ┆ 22191399_spinal_canal_stenosis… ┆ 2           ┆ null     ┆ null   │\n│ 26342422 ┆ 26342422_spinal_canal_stenosis… ┆ 2           ┆ null     ┆ null   │\n└──────────┴─────────────────────────────────┴─────────────┴──────────┴────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (10, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>row_id</th><th>normal_mild</th><th>moderate</th><th>severe</th></tr><tr><td>i64</td><td>str</td><td>i32</td><td>i32</td><td>i32</td></tr></thead><tbody><tr><td>4003253</td><td>&quot;4003253_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>4646740</td><td>&quot;4646740_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>7143189</td><td>&quot;7143189_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>8785691</td><td>&quot;8785691_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>10728036</td><td>&quot;10728036_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>11340341</td><td>&quot;11340341_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>11943292</td><td>&quot;11943292_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>13317052</td><td>&quot;13317052_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>22191399</td><td>&quot;22191399_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td></tr><tr><td>26342422</td><td>&quot;26342422_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"train_studies_metadata_df_final_2 = train_studies_metadata_df_final.join(temp, on=[\"study_id\",\"row_id\"], how=\"inner\")\n#train_studies_metadata_df_final.columns = ['study_id', 'row_id', 'normal_mild', 'moderate', 'severe', 'sample_weight']\ntrain_studies_metadata_df_final_2.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:03.660145Z","iopub.execute_input":"2024-09-04T04:23:03.660924Z","iopub.status.idle":"2024-09-04T04:23:03.686635Z","shell.execute_reply.started":"2024-09-04T04:23:03.660862Z","shell.execute_reply":"2024-09-04T04:23:03.685700Z"},"trusted":true},"execution_count":16,"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"shape: (10, 8)\n┌──────────┬─────────────┬─────────────┬──────────┬────────┬─────────────┬────────────┬────────────┐\n│ study_id ┆ row_id      ┆ normal_mild ┆ moderate ┆ severe ┆ encoded_sev ┆ severity   ┆ sample_wei │\n│ ---      ┆ ---         ┆ ---         ┆ ---      ┆ ---    ┆ erity       ┆ ---        ┆ ght        │\n│ i64      ┆ str         ┆ i32         ┆ i32      ┆ i32    ┆ ---         ┆ str        ┆ ---        │\n│          ┆             ┆             ┆          ┆        ┆ i32         ┆            ┆ i32        │\n╞══════════╪═════════════╪═════════════╪══════════╪════════╪═════════════╪════════════╪════════════╡\n│ 4003253  ┆ 4003253_spi ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ nal_canal_s ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ tenosis_…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 4646740  ┆ 4646740_spi ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ nal_canal_s ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ tenosis_…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 7143189  ┆ 7143189_spi ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ nal_canal_s ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ tenosis_…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 8785691  ┆ 8785691_spi ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ nal_canal_s ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ tenosis_…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 10728036 ┆ 10728036_sp ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ inal_canal_ ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ stenosis…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 11340341 ┆ 11340341_sp ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ inal_canal_ ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ stenosis…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 11943292 ┆ 11943292_sp ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ inal_canal_ ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ stenosis…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 13317052 ┆ 13317052_sp ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ inal_canal_ ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ stenosis…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 22191399 ┆ 22191399_sp ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ inal_canal_ ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ stenosis…   ┆             ┆          ┆        ┆             ┆            ┆            │\n│ 26342422 ┆ 26342422_sp ┆ 2           ┆ null     ┆ null   ┆ 2           ┆ Normal/Mil ┆ 1          │\n│          ┆ inal_canal_ ┆             ┆          ┆        ┆             ┆ d          ┆            │\n│          ┆ stenosis…   ┆             ┆          ┆        ┆             ┆            ┆            │\n└──────────┴─────────────┴─────────────┴──────────┴────────┴─────────────┴────────────┴────────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (10, 8)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>row_id</th><th>normal_mild</th><th>moderate</th><th>severe</th><th>encoded_severity</th><th>severity</th><th>sample_weight</th></tr><tr><td>i64</td><td>str</td><td>i32</td><td>i32</td><td>i32</td><td>i32</td><td>str</td><td>i32</td></tr></thead><tbody><tr><td>4003253</td><td>&quot;4003253_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>4646740</td><td>&quot;4646740_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>7143189</td><td>&quot;7143189_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>8785691</td><td>&quot;8785691_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>10728036</td><td>&quot;10728036_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>11340341</td><td>&quot;11340341_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>11943292</td><td>&quot;11943292_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>13317052</td><td>&quot;13317052_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>22191399</td><td>&quot;22191399_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr><tr><td>26342422</td><td>&quot;26342422_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>2</td><td>&quot;Normal/Mild&quot;</td><td>1</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"train_studies_metadata_df_final_2 = train_studies_metadata_df_final_2.drop(['encoded_severity', 'severity'])","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:10.362064Z","iopub.execute_input":"2024-09-04T04:23:10.362779Z","iopub.status.idle":"2024-09-04T04:23:10.368407Z","shell.execute_reply.started":"2024-09-04T04:23:10.362742Z","shell.execute_reply":"2024-09-04T04:23:10.367420Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"train_studies_metadata_df_final_2 = train_studies_metadata_df_final_2.with_columns([\n    pl.when(pl.col('normal_mild').is_not_null()).then(1).otherwise(0).alias('true_normal_mild'),\n    pl.when(pl.col('moderate').is_not_null()).then(1).otherwise(0).alias('true_moderate'),\n    pl.when(pl.col('severe').is_not_null()).then(1).otherwise(0).alias('true_severe'),\n])\n\ntrain_studies_metadata_df_final_2.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:13.522763Z","iopub.execute_input":"2024-09-04T04:23:13.523940Z","iopub.status.idle":"2024-09-04T04:23:13.539809Z","shell.execute_reply.started":"2024-09-04T04:23:13.523864Z","shell.execute_reply":"2024-09-04T04:23:13.538802Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"shape: (10, 9)\n┌──────────┬────────────┬───────────┬──────────┬───┬───────────┬───────────┬───────────┬───────────┐\n│ study_id ┆ row_id     ┆ normal_mi ┆ moderate ┆ … ┆ sample_we ┆ true_norm ┆ true_mode ┆ true_seve │\n│ ---      ┆ ---        ┆ ld        ┆ ---      ┆   ┆ ight      ┆ al_mild   ┆ rate      ┆ re        │\n│ i64      ┆ str        ┆ ---       ┆ i32      ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---       │\n│          ┆            ┆ i32       ┆          ┆   ┆ i32       ┆ i32       ┆ i32       ┆ i32       │\n╞══════════╪════════════╪═══════════╪══════════╪═══╪═══════════╪═══════════╪═══════════╪═══════════╡\n│ 4003253  ┆ 4003253_sp ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ inal_canal ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ _stenosis_ ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 4646740  ┆ 4646740_sp ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ inal_canal ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ _stenosis_ ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 7143189  ┆ 7143189_sp ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ inal_canal ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ _stenosis_ ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 8785691  ┆ 8785691_sp ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ inal_canal ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ _stenosis_ ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 10728036 ┆ 10728036_s ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ pinal_cana ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ l_stenosis ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 11340341 ┆ 11340341_s ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ pinal_cana ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ l_stenosis ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 11943292 ┆ 11943292_s ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ pinal_cana ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ l_stenosis ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 13317052 ┆ 13317052_s ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ pinal_cana ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ l_stenosis ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 22191399 ┆ 22191399_s ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ pinal_cana ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ l_stenosis ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│ 26342422 ┆ 26342422_s ┆ 2         ┆ null     ┆ … ┆ 1         ┆ 1         ┆ 0         ┆ 0         │\n│          ┆ pinal_cana ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ l_stenosis ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n│          ┆ …          ┆           ┆          ┆   ┆           ┆           ┆           ┆           │\n└──────────┴────────────┴───────────┴──────────┴───┴───────────┴───────────┴───────────┴───────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (10, 9)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>row_id</th><th>normal_mild</th><th>moderate</th><th>severe</th><th>sample_weight</th><th>true_normal_mild</th><th>true_moderate</th><th>true_severe</th></tr><tr><td>i64</td><td>str</td><td>i32</td><td>i32</td><td>i32</td><td>i32</td><td>i32</td><td>i32</td><td>i32</td></tr></thead><tbody><tr><td>4003253</td><td>&quot;4003253_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>4646740</td><td>&quot;4646740_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>7143189</td><td>&quot;7143189_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>8785691</td><td>&quot;8785691_spinal_canal_stenosis_…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>10728036</td><td>&quot;10728036_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>11340341</td><td>&quot;11340341_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>11943292</td><td>&quot;11943292_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>13317052</td><td>&quot;13317052_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>22191399</td><td>&quot;22191399_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>26342422</td><td>&quot;26342422_spinal_canal_stenosis…</td><td>2</td><td>null</td><td>null</td><td>1</td><td>1</td><td>0</td><td>0</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"train_studies_metadata_df_final_2 = train_studies_metadata_df_final_2.drop(['normal_mild', 'moderate', 'severe'])\ntrain_studies_metadata_df_final_2.columns = ['study_id', 'row_id', 'sample_weight', 'normal_mild', 'moderate', 'severe']\ntrain_studies_metadata_df_final_2.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:19.785806Z","iopub.execute_input":"2024-09-04T04:23:19.786663Z","iopub.status.idle":"2024-09-04T04:23:19.795187Z","shell.execute_reply.started":"2024-09-04T04:23:19.786623Z","shell.execute_reply":"2024-09-04T04:23:19.794219Z"},"trusted":true},"execution_count":19,"outputs":[{"execution_count":19,"output_type":"execute_result","data":{"text/plain":"shape: (10, 6)\n┌──────────┬─────────────────────────────────┬───────────────┬─────────────┬──────────┬────────┐\n│ study_id ┆ row_id                          ┆ sample_weight ┆ normal_mild ┆ moderate ┆ severe │\n│ ---      ┆ ---                             ┆ ---           ┆ ---         ┆ ---      ┆ ---    │\n│ i64      ┆ str                             ┆ i32           ┆ i32         ┆ i32      ┆ i32    │\n╞══════════╪═════════════════════════════════╪═══════════════╪═════════════╪══════════╪════════╡\n│ 4003253  ┆ 4003253_spinal_canal_stenosis_… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 4646740  ┆ 4646740_spinal_canal_stenosis_… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 7143189  ┆ 7143189_spinal_canal_stenosis_… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 8785691  ┆ 8785691_spinal_canal_stenosis_… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 10728036 ┆ 10728036_spinal_canal_stenosis… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 11340341 ┆ 11340341_spinal_canal_stenosis… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 11943292 ┆ 11943292_spinal_canal_stenosis… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 13317052 ┆ 13317052_spinal_canal_stenosis… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 22191399 ┆ 22191399_spinal_canal_stenosis… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n│ 26342422 ┆ 26342422_spinal_canal_stenosis… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n└──────────┴─────────────────────────────────┴───────────────┴─────────────┴──────────┴────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (10, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>row_id</th><th>sample_weight</th><th>normal_mild</th><th>moderate</th><th>severe</th></tr><tr><td>i64</td><td>str</td><td>i32</td><td>i32</td><td>i32</td><td>i32</td></tr></thead><tbody><tr><td>4003253</td><td>&quot;4003253_spinal_canal_stenosis_…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>4646740</td><td>&quot;4646740_spinal_canal_stenosis_…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>7143189</td><td>&quot;7143189_spinal_canal_stenosis_…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>8785691</td><td>&quot;8785691_spinal_canal_stenosis_…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>10728036</td><td>&quot;10728036_spinal_canal_stenosis…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>11340341</td><td>&quot;11340341_spinal_canal_stenosis…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>11943292</td><td>&quot;11943292_spinal_canal_stenosis…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>13317052</td><td>&quot;13317052_spinal_canal_stenosis…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>22191399</td><td>&quot;22191399_spinal_canal_stenosis…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr><tr><td>26342422</td><td>&quot;26342422_spinal_canal_stenosis…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"train_studies_metadata_df_final_2.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:25.074937Z","iopub.execute_input":"2024-09-04T04:23:25.075311Z","iopub.status.idle":"2024-09-04T04:23:25.081693Z","shell.execute_reply.started":"2024-09-04T04:23:25.075276Z","shell.execute_reply":"2024-09-04T04:23:25.080623Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"(44750, 6)"},"metadata":{}}]},{"cell_type":"code","source":"solutions = train_studies_metadata_df_final_2.filter(pl.col('study_id').is_in(studies))\nsolutions = solutions.drop(['study_id'])\nprint(solutions.shape)\n#solutions.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:33.774293Z","iopub.execute_input":"2024-09-04T04:23:33.774670Z","iopub.status.idle":"2024-09-04T04:23:33.788618Z","shell.execute_reply.started":"2024-09-04T04:23:33.774634Z","shell.execute_reply":"2024-09-04T04:23:33.787480Z"},"trusted":true},"execution_count":21,"outputs":[{"name":"stdout","text":"(2500, 5)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Now generate the predictions","metadata":{}},{"cell_type":"code","source":"model = keras.models.\\\nload_model(\"/kaggle/input/keras_base_scs_l1_l2/tensorflow2/default/1/keras_base_spinal_canal_stenosis_l1_l2.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:39.877493Z","iopub.execute_input":"2024-09-04T04:23:39.878400Z","iopub.status.idle":"2024-09-04T04:23:42.194711Z","shell.execute_reply.started":"2024-09-04T04:23:39.878357Z","shell.execute_reply":"2024-09-04T04:23:42.193936Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"def read_and_parse_dicom_files_for_inf(full_file_path):\n    tf.config.run_functions_eagerly(True)\n    raw_image = tf.io.read_file(full_file_path)\n    sp = tf.strings.split(tf.gather(tf.strings.split(full_file_path, 'images/'), 1), '/')\n    N = tf.size(sp)\n    LEN = tf.strings.length(tf.gather(sp, 0))+tf.strings.length(tf.gather(sp, 2))\n    \n    # Add missing file metadata to avoid warnnigs flooding\n    if   LEN==12: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x92\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==13: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x92\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==14: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x94\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==15: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x94\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==16: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x96\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==17: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x96\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==18: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x98\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    \n    #image_bytes = tf.io.read_file(full_file_path)\n    #image = tfio.image.decode_dicom_image(image_bytes, scale='auto', dtype=tf.float32)\n    image = tfio.image.decode_dicom_image(raw_image, scale='auto', dtype=tf.float32)\n    m, M=tf.math.reduce_min(image), tf.math.reduce_max(image)\n    image = (tf.image.grayscale_to_rgb(image)-m)/(M-m)\n    image = tf.image.resize(image, (128,128))\n    return tf.squeeze(image)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:47.030734Z","iopub.execute_input":"2024-09-04T04:23:47.031377Z","iopub.status.idle":"2024-09-04T04:23:47.043757Z","shell.execute_reply.started":"2024-09-04T04:23:47.031337Z","shell.execute_reply":"2024-09-04T04:23:47.042746Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"vfunc = np.vectorize(read_and_parse_dicom_files_for_inf, otypes=[object])\n\ndef get_predictions(key, model_to_use):\n    final_feature_list = vfunc(test_dict[key]).tolist()\n    final = np.array(final_feature_list)\n    return model_to_use.predict(final)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:51.574382Z","iopub.execute_input":"2024-09-04T04:23:51.574758Z","iopub.status.idle":"2024-09-04T04:23:51.580565Z","shell.execute_reply.started":"2024-09-04T04:23:51.574723Z","shell.execute_reply":"2024-09-04T04:23:51.579404Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"from multiprocessing import cpu_count\nn_cores = cpu_count()\nprint(f'Number of Logical CPU cores: {n_cores}')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:23:55.228410Z","iopub.execute_input":"2024-09-04T04:23:55.229141Z","iopub.status.idle":"2024-09-04T04:23:55.234065Z","shell.execute_reply.started":"2024-09-04T04:23:55.229101Z","shell.execute_reply":"2024-09-04T04:23:55.232926Z"},"trusted":true},"execution_count":25,"outputs":[{"name":"stdout","text":"Number of Logical CPU cores: 4\n","output_type":"stream"}]},{"cell_type":"code","source":"#from joblib import Parallel, delayed\nfrom tqdm import tqdm\n\n#y_proba = (Parallel(n_jobs=90)(delayed(get_predictions)(st, model) for st in tqdm(test_dict.keys())))\ny_proba = [get_predictions(st, model) for st in tqdm(test_dict.keys())]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:24:04.407244Z","iopub.execute_input":"2024-09-04T04:24:04.407629Z","iopub.status.idle":"2024-09-04T04:30:10.072513Z","shell.execute_reply.started":"2024-09-04T04:24:04.407592Z","shell.execute_reply":"2024-09-04T04:30:10.071500Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":26,"outputs":[{"name":"stderr","text":"  0%|          | 0/100 [00:00<?, ?it/s]/opt/conda/lib/python3.10/site-packages/numpy/lib/function_base.py:2455: RuntimeWarning: invalid value encountered in read_and_parse_dicom_files_for_inf (vectorized)\n  outputs = ufunc(*inputs)\n/opt/conda/lib/python3.10/site-packages/tensorflow/python/data/ops/structured_function.py:258: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n  warnings.warn(\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 277ms/step\n","output_type":"stream"},{"name":"stderr","text":"  1%|          | 1/100 [00:04<08:03,  4.89s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 249ms/step\n","output_type":"stream"},{"name":"stderr","text":"  2%|▏         | 2/100 [00:08<06:50,  4.18s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 250ms/step\n","output_type":"stream"},{"name":"stderr","text":"  3%|▎         | 3/100 [00:11<05:54,  3.66s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 256ms/step\n","output_type":"stream"},{"name":"stderr","text":"  4%|▍         | 4/100 [00:17<07:00,  4.38s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 270ms/step\n","output_type":"stream"},{"name":"stderr","text":"  5%|▌         | 5/100 [00:19<05:58,  3.77s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 255ms/step\n","output_type":"stream"},{"name":"stderr","text":"  6%|▌         | 6/100 [00:24<06:21,  4.06s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 251ms/step\n","output_type":"stream"},{"name":"stderr","text":"  7%|▋         | 7/100 [00:27<05:45,  3.72s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 260ms/step\n","output_type":"stream"},{"name":"stderr","text":"  8%|▊         | 8/100 [00:29<04:55,  3.21s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 239ms/step\n","output_type":"stream"},{"name":"stderr","text":"  9%|▉         | 9/100 [00:31<04:30,  2.97s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 256ms/step\n","output_type":"stream"},{"name":"stderr","text":" 10%|█         | 10/100 [00:35<04:43,  3.15s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 230ms/step\n","output_type":"stream"},{"name":"stderr","text":" 11%|█         | 11/100 [00:39<05:10,  3.49s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 12%|█▏        | 12/100 [00:43<05:07,  3.49s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 251ms/step\n","output_type":"stream"},{"name":"stderr","text":" 13%|█▎        | 13/100 [00:47<05:19,  3.68s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 242ms/step\n","output_type":"stream"},{"name":"stderr","text":" 14%|█▍        | 14/100 [00:50<05:00,  3.49s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 238ms/step\n","output_type":"stream"},{"name":"stderr","text":" 15%|█▌        | 15/100 [00:52<04:29,  3.17s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 267ms/step\n","output_type":"stream"},{"name":"stderr","text":" 16%|█▌        | 16/100 [00:56<04:27,  3.18s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 276ms/step\n","output_type":"stream"},{"name":"stderr","text":" 17%|█▋        | 17/100 [00:58<04:05,  2.96s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 254ms/step\n","output_type":"stream"},{"name":"stderr","text":" 18%|█▊        | 18/100 [01:00<03:48,  2.79s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 248ms/step\n","output_type":"stream"},{"name":"stderr","text":" 19%|█▉        | 19/100 [01:03<03:40,  2.72s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 235ms/step\n","output_type":"stream"},{"name":"stderr","text":" 20%|██        | 20/100 [01:07<03:59,  3.00s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 267ms/step\n","output_type":"stream"},{"name":"stderr","text":" 21%|██        | 21/100 [01:10<03:55,  2.98s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 239ms/step\n","output_type":"stream"},{"name":"stderr","text":" 22%|██▏       | 22/100 [01:13<03:50,  2.96s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 260ms/step\n","output_type":"stream"},{"name":"stderr","text":" 23%|██▎       | 23/100 [01:17<04:26,  3.46s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 251ms/step\n","output_type":"stream"},{"name":"stderr","text":" 24%|██▍       | 24/100 [01:21<04:26,  3.51s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 25%|██▌       | 25/100 [01:27<05:24,  4.33s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 252ms/step\n","output_type":"stream"},{"name":"stderr","text":" 26%|██▌       | 26/100 [01:30<04:51,  3.94s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 231ms/step\n","output_type":"stream"},{"name":"stderr","text":" 27%|██▋       | 27/100 [01:36<05:22,  4.42s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 28%|██▊       | 28/100 [01:40<05:14,  4.36s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 236ms/step\n","output_type":"stream"},{"name":"stderr","text":" 29%|██▉       | 29/100 [01:44<05:06,  4.31s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 237ms/step\n","output_type":"stream"},{"name":"stderr","text":" 30%|███       | 30/100 [01:49<05:16,  4.52s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 233ms/step\n","output_type":"stream"},{"name":"stderr","text":" 31%|███       | 31/100 [01:52<04:50,  4.21s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 233ms/step\n","output_type":"stream"},{"name":"stderr","text":" 32%|███▏      | 32/100 [01:55<04:11,  3.69s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 241ms/step\n","output_type":"stream"},{"name":"stderr","text":" 33%|███▎      | 33/100 [01:58<03:45,  3.36s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 274ms/step\n","output_type":"stream"},{"name":"stderr","text":" 34%|███▍      | 34/100 [02:00<03:30,  3.20s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 238ms/step\n","output_type":"stream"},{"name":"stderr","text":" 35%|███▌      | 35/100 [02:03<03:17,  3.04s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 243ms/step\n","output_type":"stream"},{"name":"stderr","text":" 36%|███▌      | 36/100 [02:06<03:10,  2.97s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 233ms/step\n","output_type":"stream"},{"name":"stderr","text":" 37%|███▋      | 37/100 [02:11<03:40,  3.50s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 239ms/step\n","output_type":"stream"},{"name":"stderr","text":" 38%|███▊      | 38/100 [02:14<03:32,  3.42s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 238ms/step\n","output_type":"stream"},{"name":"stderr","text":" 39%|███▉      | 39/100 [02:21<04:35,  4.51s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 240ms/step\n","output_type":"stream"},{"name":"stderr","text":" 40%|████      | 40/100 [02:27<04:53,  4.89s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 238ms/step\n","output_type":"stream"},{"name":"stderr","text":" 41%|████      | 41/100 [02:30<04:27,  4.54s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 240ms/step\n","output_type":"stream"},{"name":"stderr","text":" 42%|████▏     | 42/100 [02:34<04:05,  4.23s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 264ms/step\n","output_type":"stream"},{"name":"stderr","text":" 43%|████▎     | 43/100 [02:37<03:41,  3.88s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 246ms/step\n","output_type":"stream"},{"name":"stderr","text":" 44%|████▍     | 44/100 [02:41<03:41,  3.96s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 264ms/step\n","output_type":"stream"},{"name":"stderr","text":" 45%|████▌     | 45/100 [02:44<03:17,  3.58s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 46%|████▌     | 46/100 [02:47<03:12,  3.56s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 233ms/step\n","output_type":"stream"},{"name":"stderr","text":" 47%|████▋     | 47/100 [02:50<02:51,  3.24s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 245ms/step\n","output_type":"stream"},{"name":"stderr","text":" 48%|████▊     | 48/100 [02:53<02:41,  3.10s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 227ms/step\n","output_type":"stream"},{"name":"stderr","text":" 49%|████▉     | 49/100 [02:56<02:45,  3.25s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 50%|█████     | 50/100 [03:00<02:57,  3.55s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 236ms/step\n","output_type":"stream"},{"name":"stderr","text":" 51%|█████     | 51/100 [03:05<03:15,  3.99s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 52%|█████▏    | 52/100 [03:09<03:10,  3.97s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 241ms/step\n","output_type":"stream"},{"name":"stderr","text":" 53%|█████▎    | 53/100 [03:14<03:16,  4.19s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 54%|█████▍    | 54/100 [03:17<02:54,  3.79s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 231ms/step\n","output_type":"stream"},{"name":"stderr","text":" 55%|█████▌    | 55/100 [03:21<02:56,  3.92s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 243ms/step\n","output_type":"stream"},{"name":"stderr","text":" 56%|█████▌    | 56/100 [03:24<02:38,  3.61s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 235ms/step\n","output_type":"stream"},{"name":"stderr","text":" 57%|█████▋    | 57/100 [03:29<02:51,  3.99s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 58%|█████▊    | 58/100 [03:32<02:33,  3.65s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 243ms/step\n","output_type":"stream"},{"name":"stderr","text":" 59%|█████▉    | 59/100 [03:38<03:00,  4.41s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 248ms/step\n","output_type":"stream"},{"name":"stderr","text":" 60%|██████    | 60/100 [03:41<02:37,  3.93s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 239ms/step\n","output_type":"stream"},{"name":"stderr","text":" 61%|██████    | 61/100 [03:45<02:32,  3.92s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 229ms/step\n","output_type":"stream"},{"name":"stderr","text":" 62%|██████▏   | 62/100 [03:47<02:08,  3.38s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 63%|██████▎   | 63/100 [03:52<02:26,  3.95s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 236ms/step\n","output_type":"stream"},{"name":"stderr","text":" 64%|██████▍   | 64/100 [03:55<02:12,  3.67s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 236ms/step\n","output_type":"stream"},{"name":"stderr","text":" 65%|██████▌   | 65/100 [03:59<02:13,  3.81s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 235ms/step\n","output_type":"stream"},{"name":"stderr","text":" 66%|██████▌   | 66/100 [04:02<02:02,  3.61s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 236ms/step\n","output_type":"stream"},{"name":"stderr","text":" 67%|██████▋   | 67/100 [04:05<01:45,  3.21s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 245ms/step\n","output_type":"stream"},{"name":"stderr","text":" 68%|██████▊   | 68/100 [04:08<01:46,  3.32s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 256ms/step\n","output_type":"stream"},{"name":"stderr","text":" 69%|██████▉   | 69/100 [04:12<01:44,  3.36s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 70%|███████   | 70/100 [04:16<01:47,  3.59s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 241ms/step\n","output_type":"stream"},{"name":"stderr","text":" 71%|███████   | 71/100 [04:19<01:44,  3.61s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 72%|███████▏  | 72/100 [04:23<01:41,  3.63s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 229ms/step\n","output_type":"stream"},{"name":"stderr","text":" 73%|███████▎  | 73/100 [04:26<01:32,  3.43s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 239ms/step\n","output_type":"stream"},{"name":"stderr","text":" 74%|███████▍  | 74/100 [04:30<01:31,  3.50s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 265ms/step\n","output_type":"stream"},{"name":"stderr","text":" 75%|███████▌  | 75/100 [04:33<01:23,  3.33s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m8/8\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 245ms/step\n","output_type":"stream"},{"name":"stderr","text":" 76%|███████▌  | 76/100 [04:45<02:25,  6.05s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 233ms/step\n","output_type":"stream"},{"name":"stderr","text":" 77%|███████▋  | 77/100 [04:49<02:02,  5.32s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 78%|███████▊  | 78/100 [04:53<01:49,  4.99s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 230ms/step\n","output_type":"stream"},{"name":"stderr","text":" 79%|███████▉  | 79/100 [04:55<01:28,  4.20s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 242ms/step\n","output_type":"stream"},{"name":"stderr","text":" 80%|████████  | 80/100 [04:58<01:13,  3.66s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 81%|████████  | 81/100 [05:02<01:11,  3.74s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 255ms/step\n","output_type":"stream"},{"name":"stderr","text":" 82%|████████▏ | 82/100 [05:04<01:00,  3.39s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 230ms/step\n","output_type":"stream"},{"name":"stderr","text":" 83%|████████▎ | 83/100 [05:08<01:02,  3.66s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 227ms/step\n","output_type":"stream"},{"name":"stderr","text":" 84%|████████▍ | 84/100 [05:12<00:58,  3.65s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 252ms/step\n","output_type":"stream"},{"name":"stderr","text":" 85%|████████▌ | 85/100 [05:14<00:49,  3.28s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 231ms/step\n","output_type":"stream"},{"name":"stderr","text":" 86%|████████▌ | 86/100 [05:17<00:44,  3.16s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 231ms/step\n","output_type":"stream"},{"name":"stderr","text":" 87%|████████▋ | 87/100 [05:20<00:38,  2.93s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 227ms/step\n","output_type":"stream"},{"name":"stderr","text":" 88%|████████▊ | 88/100 [05:22<00:32,  2.73s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 234ms/step\n","output_type":"stream"},{"name":"stderr","text":" 89%|████████▉ | 89/100 [05:26<00:34,  3.17s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 236ms/step\n","output_type":"stream"},{"name":"stderr","text":" 90%|█████████ | 90/100 [05:30<00:32,  3.21s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 247ms/step\n","output_type":"stream"},{"name":"stderr","text":" 91%|█████████ | 91/100 [05:33<00:29,  3.26s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 92%|█████████▏| 92/100 [05:36<00:26,  3.30s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 235ms/step\n","output_type":"stream"},{"name":"stderr","text":" 93%|█████████▎| 93/100 [05:41<00:26,  3.81s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 246ms/step\n","output_type":"stream"},{"name":"stderr","text":" 94%|█████████▍| 94/100 [05:45<00:22,  3.72s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 246ms/step\n","output_type":"stream"},{"name":"stderr","text":" 95%|█████████▌| 95/100 [05:48<00:17,  3.52s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 232ms/step\n","output_type":"stream"},{"name":"stderr","text":" 96%|█████████▌| 96/100 [05:50<00:12,  3.16s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 231ms/step\n","output_type":"stream"},{"name":"stderr","text":" 97%|█████████▋| 97/100 [05:55<00:11,  3.75s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 240ms/step\n","output_type":"stream"},{"name":"stderr","text":" 98%|█████████▊| 98/100 [06:00<00:07,  3.91s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 231ms/step\n","output_type":"stream"},{"name":"stderr","text":" 99%|█████████▉| 99/100 [06:02<00:03,  3.46s/it]","output_type":"stream"},{"name":"stdout","text":"\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 241ms/step\n","output_type":"stream"},{"name":"stderr","text":"100%|██████████| 100/100 [06:05<00:00,  3.66s/it]\n","output_type":"stream"}]},{"cell_type":"code","source":"rows = {}\n\nfor i in range(len(y_proba)):\n    rows[list(test_dict.keys())[i]+'_spinal_canal_stenosis_l1_l2'] = np.mean(y_proba[i], axis=0)\n    \nsubmission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\nsubmission['row_id'] = 'samples'\n\nconditions = ['spinal_canal_stenosis', 'neural_foraminal_narrowing', 'subarticular_stenosis']\nsides = ['left', 'right']\nvertebrae_levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\nseverity_levels = ['normal_mild', 'moderate', 'severe']\n\nfor c in conditions:\n    for v in vertebrae_levels:\n        if c != 'spinal_canal_stenosis':\n            for s in sides:\n                for st in test_dict.keys():\n                    rows[st+'_'+s+'_'+c+'_'+v] = np.array([0.333333, 0.333333, 0.333333])\n        else:\n            for st in test_dict.keys():\n                if c+'_'+v == 'spinal_canal_stenosis_l1_l2':\n                    pass\n                else:\n                    rows[st+'_'+c+'_'+v] = np.array([0.333333, 0.333333, 0.333333])\n                    \nfor row_id, feature in tqdm(rows.items()):\n    feature_set_reshaped = feature.reshape(1, -1)\n    predictions = np.ascontiguousarray(feature_set_reshaped)\n    df = pd.DataFrame(predictions, columns=severity_levels)\n    df.insert(loc=0, column='row_id', value=row_id)\n    submission = pd.concat([submission,df]).reset_index(drop=True)\n\ni = submission[(submission.row_id == 'samples')].index\nsubmission = submission.drop(i).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:30:34.517821Z","iopub.execute_input":"2024-09-04T04:30:34.518719Z","iopub.status.idle":"2024-09-04T04:30:36.432638Z","shell.execute_reply.started":"2024-09-04T04:30:34.518679Z","shell.execute_reply":"2024-09-04T04:30:36.431625Z"},"trusted":true},"execution_count":27,"outputs":[{"name":"stderr","text":"100%|██████████| 2500/2500 [00:01<00:00, 1335.09it/s]\n","output_type":"stream"}]},{"cell_type":"code","source":"solutions_pd = solutions.to_pandas()\nprint(submission.shape)\nprint(solutions_pd.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:30:41.109798Z","iopub.execute_input":"2024-09-04T04:30:41.110658Z","iopub.status.idle":"2024-09-04T04:30:41.134868Z","shell.execute_reply.started":"2024-09-04T04:30:41.110616Z","shell.execute_reply":"2024-09-04T04:30:41.134029Z"},"trusted":true},"execution_count":28,"outputs":[{"name":"stdout","text":"(2500, 4)\n(2500, 5)\n","output_type":"stream"}]},{"cell_type":"code","source":"def get_condition(full_location: str) -> str:\n    # Given an input like spinal_canal_stenosis_l1_l2 extracts 'spinal'\n    for injury_condition in ['spinal', 'foraminal', 'subarticular']:\n        if injury_condition in full_location:\n            return injury_condition\n    raise ValueError(f'condition not found in {full_location}')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:30:45.738803Z","iopub.execute_input":"2024-09-04T04:30:45.739562Z","iopub.status.idle":"2024-09-04T04:30:45.744425Z","shell.execute_reply.started":"2024-09-04T04:30:45.739524Z","shell.execute_reply":"2024-09-04T04:30:45.743426Z"},"trusted":true},"execution_count":29,"outputs":[]},{"cell_type":"code","source":"target_levels = ['normal_mild', 'moderate', 'severe']\n\nif not pd.api.types.is_numeric_dtype(submission[target_levels].values):\n        raise ParticipantVisibleError('All submission values must be numeric')\n\nif not np.isfinite(submission[target_levels].values).all():\n    raise ParticipantVisibleError('All submission values must be finite')\n\nif solutions_pd[target_levels].min().min() < 0:\n    raise ParticipantVisibleError('All labels must be at least zero')\nif submission[target_levels].min().min() < 0:\n    raise ParticipantVisibleError('All predictions must be at least zero')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:30:53.140693Z","iopub.execute_input":"2024-09-04T04:30:53.141096Z","iopub.status.idle":"2024-09-04T04:30:53.153578Z","shell.execute_reply.started":"2024-09-04T04:30:53.141059Z","shell.execute_reply":"2024-09-04T04:30:53.152442Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"solutions_pd['study_id'] = solutions_pd['row_id'].apply(lambda x: x.split('_')[0])\nsolutions_pd['location'] = solutions_pd['row_id'].apply(lambda x: '_'.join(x.split('_')[1:]))\nsolutions_pd['condition'] = solutions_pd['row_id'].apply(get_condition)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:30:57.084759Z","iopub.execute_input":"2024-09-04T04:30:57.085155Z","iopub.status.idle":"2024-09-04T04:30:57.098762Z","shell.execute_reply.started":"2024-09-04T04:30:57.085118Z","shell.execute_reply":"2024-09-04T04:30:57.097839Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"code","source":"row_id_column_name = 'row_id'\n\ndel solutions_pd[row_id_column_name]\ndel submission[row_id_column_name]\nassert sorted(submission.columns) == sorted(target_levels)\n\nsubmission['study_id'] = solutions_pd['study_id']\nsubmission['location'] = solutions_pd['location']\nsubmission['condition'] = solutions_pd['condition']","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:31:00.018778Z","iopub.execute_input":"2024-09-04T04:31:00.019490Z","iopub.status.idle":"2024-09-04T04:31:00.028156Z","shell.execute_reply.started":"2024-09-04T04:31:00.019449Z","shell.execute_reply":"2024-09-04T04:31:00.027091Z"},"trusted":true},"execution_count":32,"outputs":[]},{"cell_type":"markdown","source":"#### Testing for one condition to understand log loss","metadata":{}},{"cell_type":"code","source":"condition_losses = []\ncondition_weights = []\n\ncondition_indices = solutions_pd.loc[solutions_pd['condition'] == 'spinal'].index.values","metadata":{"execution":{"iopub.status.busy":"2024-09-04T04:46:35.005946Z","iopub.execute_input":"2024-09-04T04:46:35.006341Z","iopub.status.idle":"2024-09-04T04:46:35.014400Z","shell.execute_reply.started":"2024-09-04T04:46:35.006305Z","shell.execute_reply":"2024-09-04T04:46:35.013235Z"},"trusted":true},"execution_count":33,"outputs":[]},{"cell_type":"code","source":"y_true=solutions_pd.loc[condition_indices, target_levels].values\ny_true","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:32:37.527669Z","iopub.execute_input":"2024-09-04T05:32:37.528090Z","iopub.status.idle":"2024-09-04T05:32:37.536537Z","shell.execute_reply.started":"2024-09-04T05:32:37.528052Z","shell.execute_reply":"2024-09-04T05:32:37.535518Z"},"trusted":true},"execution_count":73,"outputs":[{"execution_count":73,"output_type":"execute_result","data":{"text/plain":"array([[1, 0, 0],\n       [1, 0, 0],\n       [1, 0, 0],\n       ...,\n       [1, 0, 0],\n       [1, 0, 0],\n       [1, 0, 0]], dtype=int32)"},"metadata":{}}]},{"cell_type":"code","source":"y_pred=submission.loc[condition_indices, target_levels].values\ny_pred","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:32:34.611450Z","iopub.execute_input":"2024-09-04T05:32:34.612121Z","iopub.status.idle":"2024-09-04T05:32:34.620112Z","shell.execute_reply.started":"2024-09-04T05:32:34.612081Z","shell.execute_reply":"2024-09-04T05:32:34.619182Z"},"trusted":true},"execution_count":72,"outputs":[{"execution_count":72,"output_type":"execute_result","data":{"text/plain":"array([[0.06724037, 0.87429702, 0.05846261],\n       [0.06496824, 0.87956232, 0.05546964],\n       [0.06813553, 0.87197536, 0.059889  ],\n       ...,\n       [0.333333  , 0.333333  , 0.333333  ],\n       [0.333333  , 0.333333  , 0.333333  ],\n       [0.333333  , 0.333333  , 0.333333  ]])"},"metadata":{}}]},{"cell_type":"code","source":"sample_weight=solutions_pd.loc[condition_indices, 'sample_weight'].values\nsample_weight","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:30:41.405041Z","iopub.execute_input":"2024-09-04T05:30:41.405824Z","iopub.status.idle":"2024-09-04T05:30:41.414687Z","shell.execute_reply.started":"2024-09-04T05:30:41.405782Z","shell.execute_reply":"2024-09-04T05:30:41.413748Z"},"trusted":true},"execution_count":59,"outputs":[{"execution_count":59,"output_type":"execute_result","data":{"text/plain":"array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1,\n       1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1,\n       1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 4,\n       1, 1, 1, 1, 1, 1, 1, 4, 4, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 4, 1,\n       4, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1,\n       2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 4, 1, 1, 1, 2, 1,\n       1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 2, 1, 1, 1, 4, 1, 1, 4, 1, 1,\n       1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 4, 1, 1, 1, 4, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 2, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32)"},"metadata":{}}]},{"cell_type":"code","source":"print(sample_weight[163:165])\nprint(y_true[163:165])\nprint(y_pred[163:165])","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:32:52.992845Z","iopub.execute_input":"2024-09-04T05:32:52.993580Z","iopub.status.idle":"2024-09-04T05:32:53.000132Z","shell.execute_reply.started":"2024-09-04T05:32:52.993537Z","shell.execute_reply":"2024-09-04T05:32:52.998970Z"},"trusted":true},"execution_count":74,"outputs":[{"name":"stdout","text":"[1 4]\n[[1 0 0]\n [0 0 1]]\n[[0.333333 0.333333 0.333333]\n [0.333333 0.333333 0.333333]]\n","output_type":"stream"}]},{"cell_type":"code","source":"condition_losses = []\ncondition_weights = []\nfor condition in ['spinal', 'foraminal', 'subarticular']:\n    condition_indices = solutions_pd.loc[solutions_pd['condition'] == condition].index.values\n    condition_loss = log_loss(\n        y_true=solutions_pd.loc[condition_indices, target_levels].values,\n        y_pred=submission.loc[condition_indices, target_levels].values,\n        sample_weight=solutions_pd.loc[condition_indices, 'sample_weight'].values\n    )\n    condition_losses.append(condition_loss)\n    condition_weights.append(1)","metadata":{"execution":{"iopub.status.busy":"2024-08-30T05:58:43.709968Z","iopub.execute_input":"2024-08-30T05:58:43.710346Z","iopub.status.idle":"2024-08-30T05:58:43.732602Z","shell.execute_reply.started":"2024-08-30T05:58:43.710307Z","shell.execute_reply":"2024-08-30T05:58:43.731873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"condition_losses","metadata":{"execution":{"iopub.status.busy":"2024-08-30T05:58:47.077571Z","iopub.execute_input":"2024-08-30T05:58:47.077994Z","iopub.status.idle":"2024-08-30T05:58:47.084351Z","shell.execute_reply.started":"2024-08-30T05:58:47.077954Z","shell.execute_reply":"2024-08-30T05:58:47.083358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"any_severe_spinal_labels = pd.Series(solutions_pd.loc[solutions_pd['condition'] == 'spinal'].groupby('study_id')['severe'].max())\nany_severe_spinal_weights = pd.Series(solutions_pd.loc[solutions_pd['condition'] == 'spinal'].groupby('study_id')['sample_weight'].max())\nany_severe_spinal_predictions = pd.Series(submission.loc[submission['condition'] == 'spinal'].groupby('study_id')['severe'].max())","metadata":{"execution":{"iopub.status.busy":"2024-08-30T05:58:49.992104Z","iopub.execute_input":"2024-08-30T05:58:49.992819Z","iopub.status.idle":"2024-08-30T05:58:50.005420Z","shell.execute_reply.started":"2024-08-30T05:58:49.992756Z","shell.execute_reply":"2024-08-30T05:58:50.004507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"any_severe_scalar = 1.0\n\nany_severe_spinal_loss = log_loss(\n    y_true=any_severe_spinal_labels,\n    y_pred=any_severe_spinal_predictions,\n    sample_weight=any_severe_spinal_weights\n)\ncondition_losses.append(any_severe_spinal_loss)\ncondition_weights.append(any_severe_scalar)\n\nprint(\"final score during training : \", np.average(condition_losses, weights=condition_weights))","metadata":{"execution":{"iopub.status.busy":"2024-08-30T05:58:53.231921Z","iopub.execute_input":"2024-08-30T05:58:53.232365Z","iopub.status.idle":"2024-08-30T05:58:53.241481Z","shell.execute_reply.started":"2024-08-30T05:58:53.232309Z","shell.execute_reply":"2024-08-30T05:58:53.240486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"### Custom Loss function with weights","metadata":{}},{"cell_type":"code","source":"def create_wtd_log_loss(sample_weights):\n    def wtd_log_loss(y_true, y_pred):\n        return log_loss(y_true, y_pred, sample_weight=sample_weights)\n    return wtd_log_loss","metadata":{"execution":{"iopub.status.busy":"2024-09-02T18:15:59.764706Z","iopub.execute_input":"2024-09-02T18:15:59.765494Z","iopub.status.idle":"2024-09-02T18:15:59.770600Z","shell.execute_reply.started":"2024-09-02T18:15:59.765442Z","shell.execute_reply":"2024-09-02T18:15:59.769556Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras import layers\n\nwith strategy.scope():\n    \n    rsna_input = layers.Input(shape=(128,128,3), name=\"rsna_input\")\n    #weights = layers.Input(shape=(1,3), name=\"weights\")\n    \n    conv_base = EfficientNetB0(include_top=False, weights=\"imagenet\", input_tensor=rsna_input)\n    conv_base.trainable = False\n    \n    x = layers.GlobalAveragePooling2D(name=\"avg_pool\")(conv_base.output)\n    x = layers.BatchNormalization()(x)\n    \n    hidden_layer1 = layers.Dense(200, activation=\"relu\", kernel_initializer=keras.initializers.LecunNormal(seed=None))(x)\n    hidden_layer2 = layers.Dense(100, activation=\"selu\")(hidden_layer1)\n    hidden_layer3 = layers.Dense(50, activation=\"selu\")(hidden_layer2)\n    rsna_output = layers.Dense(3, activation=\"softmax\")(hidden_layer3)\n    #model = tf.keras.models.Model(inputs=rsna_input, outputs=rsna_output)\n    model = tf.keras.Model(rsna_input, rsna_output)\n    \n    checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"keras_effnet_spinal_canal_stenosis_l1_l2.keras\")\n    early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)\n\n    #model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adamax\", metrics=[\"accuracy\"])\n    model.compile(loss=create_wtd_log_loss(z), optimizer=\"adamax\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2024-09-02T18:19:57.533443Z","iopub.execute_input":"2024-09-02T18:19:57.533844Z","iopub.status.idle":"2024-09-02T18:19:59.292908Z","shell.execute_reply.started":"2024-09-02T18:19:57.533808Z","shell.execute_reply":"2024-09-02T18:19:59.292108Z"},"trusted":true},"execution_count":34,"outputs":[]},{"cell_type":"code","source":"config = model.get_config()\nprint(config[\"layers\"][0][\"config\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_ds, epochs=5, validation_data=valid_ds, callbacks=[checkpoint_cb, early_stopping_cb])","metadata":{"execution":{"iopub.status.busy":"2024-09-02T18:22:02.940164Z","iopub.execute_input":"2024-09-02T18:22:02.940954Z","iopub.status.idle":"2024-09-02T18:22:04.684707Z","shell.execute_reply.started":"2024-09-02T18:22:02.940914Z","shell.execute_reply":"2024-09-02T18:22:04.683409Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":37,"outputs":[{"name":"stdout","text":"Epoch 1/5\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/tensorflow/python/data/ops/structured_function.py:258: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n  warnings.warn(\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","Cell \u001b[0;32mIn[37], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#history = model.fit(train_ds, class_weight={1:1.0,2:2.0,3:4.0}, epochs=5, validation_data=valid_ds, callbacks=[checkpoint_cb, early_stopping_cb])\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m#competition_weights = list(map(lambda x: x[2], train_ds))\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m history \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_ds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalid_ds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mcheckpoint_cb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mearly_stopping_cb\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py:122\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    119\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m    120\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m    121\u001b[0m     \u001b[38;5;66;03m# `keras.config.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m--> 122\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    123\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    124\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n","Cell \u001b[0;32mIn[31], line 3\u001b[0m, in \u001b[0;36mcreate_wtd_log_loss.<locals>.wtd_log_loss\u001b[0;34m(y_true, y_pred)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwtd_log_loss\u001b[39m(y_true, y_pred):\n\u001b[0;32m----> 3\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlog_loss\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_true\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weights\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:2656\u001b[0m, in \u001b[0;36mlog_loss\u001b[0;34m(y_true, y_pred, eps, normalize, sample_weight, labels)\u001b[0m\n\u001b[1;32m   2653\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m y_pred \u001b[38;5;241m/\u001b[39m y_pred_sum[:, np\u001b[38;5;241m.\u001b[39mnewaxis]\n\u001b[1;32m   2654\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m-\u001b[39mxlogy(transformed_labels, y_pred)\u001b[38;5;241m.\u001b[39msum(axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m-> 2656\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_weighted_sum\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnormalize\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:139\u001b[0m, in \u001b[0;36m_weighted_sum\u001b[0;34m(sample_score, sample_weight, normalize)\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_weighted_sum\u001b[39m(sample_score, sample_weight, normalize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m    138\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m normalize:\n\u001b[0;32m--> 139\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maverage\u001b[49m\u001b[43m(\u001b[49m\u001b[43msample_score\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweights\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    140\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m sample_weight \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    141\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mdot(sample_score, sample_weight)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/numpy/lib/function_base.py:534\u001b[0m, in \u001b[0;36maverage\u001b[0;34m(a, axis, weights, returned, keepdims)\u001b[0m\n\u001b[1;32m    532\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m a\u001b[38;5;241m.\u001b[39mshape \u001b[38;5;241m!=\u001b[39m wgt\u001b[38;5;241m.\u001b[39mshape:\n\u001b[1;32m    533\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m axis \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 534\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m    535\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAxis must be specified when shapes of a and weights \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    536\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdiffer.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    537\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m wgt\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m    538\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m    539\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m1D weights expected when shapes of a and weights differ.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n","\u001b[0;31mTypeError\u001b[0m: Axis must be specified when shapes of a and weights differ."],"ename":"TypeError","evalue":"Axis must be specified when shapes of a and weights differ.","output_type":"error"}]},{"cell_type":"code","source":"pd.DataFrame(history.history).plot(figsize=(10,6))\nplt.grid(True)\nplt.gca().set_ylim(0.60,1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-23T13:14:50.992661Z","iopub.execute_input":"2024-08-23T13:14:50.993057Z","iopub.status.idle":"2024-08-23T13:14:51.290181Z","shell.execute_reply.started":"2024-08-23T13:14:50.993019Z","shell.execute_reply":"2024-08-23T13:14:51.289255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"keras_base_spinal_canal_stenosis_l1_l2.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cropped_image = tf.image.resize_with_crop_or_pad(image, 100, 100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1,1, figsize=(5,5))\naxes.imshow(np.squeeze(cropped_image.numpy()), cmap='gray')\naxes.set_title('image')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing inference with the trained model","metadata":{}},{"cell_type":"code","source":"model = keras.models.\\\nload_model(\"/kaggle/input/keras_base_scs_l1_l2/tensorflow2/default/1/keras_base_spinal_canal_stenosis_l1_l2.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-09-03T04:05:45.102333Z","iopub.execute_input":"2024-09-03T04:05:45.102775Z","iopub.status.idle":"2024-09-03T04:05:47.132988Z","shell.execute_reply.started":"2024-09-03T04:05:45.102732Z","shell.execute_reply":"2024-09-03T04:05:47.132203Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"### Remove the squeeze from image file preprocessing","metadata":{}},{"cell_type":"code","source":"def read_and_parse_dicom_files_for_inf(full_file_path):\n    tf.config.run_functions_eagerly(True)\n    raw_image = tf.io.read_file(full_file_path)\n    sp = tf.strings.split(tf.gather(tf.strings.split(full_file_path, 'images/'), 1), '/')\n    N = tf.size(sp)\n    LEN = tf.strings.length(tf.gather(sp, 0))+tf.strings.length(tf.gather(sp, 2))\n    \n    # Add missing file metadata to avoid warnnigs flooding\n    if   LEN==12: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x92\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==13: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x92\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==14: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x94\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==15: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x94\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==16: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x96\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==17: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x96\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    elif LEN==18: raw_image = tf.strings.regex_replace(raw_image, pattern=b'DICM\\x02\\x00\\x01\\x00', rewrite=b'DICM\\x02\\x00\\x00\\x00UL\\x04\\x00\\x98\\x00\\x00\\x00\\x02\\x00\\x01\\x00')\n    \n    #image_bytes = tf.io.read_file(full_file_path)\n    #image = tfio.image.decode_dicom_image(image_bytes, scale='auto', dtype=tf.float32)\n    image = tfio.image.decode_dicom_image(raw_image, scale='auto', dtype=tf.float32)\n    m, M=tf.math.reduce_min(image), tf.math.reduce_max(image)\n    image = (tf.image.grayscale_to_rgb(image)-m)/(M-m)\n    image = tf.image.resize(image, (128,128))\n    return image","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:33:50.676363Z","iopub.execute_input":"2024-09-04T05:33:50.677307Z","iopub.status.idle":"2024-09-04T05:33:50.690727Z","shell.execute_reply.started":"2024-09-04T05:33:50.677248Z","shell.execute_reply":"2024-09-04T05:33:50.689545Z"},"trusted":true},"execution_count":75,"outputs":[]},{"cell_type":"code","source":"\"\"\"test_image = \\\nread_and_parse_dicom_files_for_inf('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/44036939/3481971518/1.dcm')\n\"\"\"\ntest_image = \\\nread_and_parse_dicom_files_for_inf('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/20.dcm')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:33:53.379144Z","iopub.execute_input":"2024-09-04T05:33:53.379893Z","iopub.status.idle":"2024-09-04T05:33:53.425189Z","shell.execute_reply.started":"2024-09-04T05:33:53.379858Z","shell.execute_reply":"2024-09-04T05:33:53.424411Z"},"trusted":true},"execution_count":76,"outputs":[]},{"cell_type":"code","source":"test_image.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:33:55.201434Z","iopub.execute_input":"2024-09-04T05:33:55.202110Z","iopub.status.idle":"2024-09-04T05:33:55.208088Z","shell.execute_reply.started":"2024-09-04T05:33:55.202070Z","shell.execute_reply":"2024-09-04T05:33:55.207039Z"},"trusted":true},"execution_count":77,"outputs":[{"execution_count":77,"output_type":"execute_result","data":{"text/plain":"TensorShape([1, 128, 128, 3])"},"metadata":{}}]},{"cell_type":"code","source":"y_proba = model.predict(test_image)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:33:56.473448Z","iopub.execute_input":"2024-09-04T05:33:56.474155Z","iopub.status.idle":"2024-09-04T05:33:56.841057Z","shell.execute_reply.started":"2024-09-04T05:33:56.474113Z","shell.execute_reply":"2024-09-04T05:33:56.840049Z"},"trusted":true},"execution_count":78,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/tensorflow/python/data/ops/structured_function.py:258: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n  warnings.warn(\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 252ms/step\n","output_type":"stream"}]},{"cell_type":"code","source":"y_proba","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:34:00.103459Z","iopub.execute_input":"2024-09-04T05:34:00.104188Z","iopub.status.idle":"2024-09-04T05:34:00.110508Z","shell.execute_reply.started":"2024-09-04T05:34:00.104150Z","shell.execute_reply":"2024-09-04T05:34:00.109301Z"},"trusted":true},"execution_count":79,"outputs":[{"execution_count":79,"output_type":"execute_result","data":{"text/plain":"array([[0.0680268 , 0.87121695, 0.06075624]], dtype=float32)"},"metadata":{}}]},{"cell_type":"code","source":"train_studies_metadata_df.filter(pl.col('study_id')==100206310)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:34:01.602068Z","iopub.execute_input":"2024-09-04T05:34:01.602437Z","iopub.status.idle":"2024-09-04T05:34:01.612591Z","shell.execute_reply.started":"2024-09-04T05:34:01.602401Z","shell.execute_reply":"2024-09-04T05:34:01.611489Z"},"trusted":true},"execution_count":80,"outputs":[{"execution_count":80,"output_type":"execute_result","data":{"text/plain":"shape: (1, 26)\n┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n│ study_id  ┆ spinal_ca ┆ spinal_ca ┆ spinal_ca ┆ … ┆ right_sub ┆ right_sub ┆ right_sub ┆ right_su │\n│ ---       ┆ nal_steno ┆ nal_steno ┆ nal_steno ┆   ┆ articular ┆ articular ┆ articular ┆ barticul │\n│ i64       ┆ sis_l1_l2 ┆ sis_l2_l3 ┆ sis_l3_l4 ┆   ┆ _stenosis ┆ _stenosis ┆ _stenosis ┆ ar_steno │\n│           ┆ ---       ┆ ---       ┆ ---       ┆   ┆ _l2…      ┆ _l3…      ┆ _l4…      ┆ sis_l5…  │\n│           ┆ str       ┆ str       ┆ str       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---      │\n│           ┆           ┆           ┆           ┆   ┆ str       ┆ str       ┆ str       ┆ str      │\n╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n│ 100206310 ┆ Normal/Mi ┆ Normal/Mi ┆ Moderate  ┆ … ┆ Normal/Mi ┆ Moderate  ┆ Moderate  ┆ Moderate │\n│           ┆ ld        ┆ ld        ┆           ┆   ┆ ld        ┆           ┆           ┆          │\n└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (1, 26)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>spinal_canal_stenosis_l1_l2</th><th>spinal_canal_stenosis_l2_l3</th><th>spinal_canal_stenosis_l3_l4</th><th>spinal_canal_stenosis_l4_l5</th><th>spinal_canal_stenosis_l5_s1</th><th>left_neural_foraminal_narrowing_l1_l2</th><th>left_neural_foraminal_narrowing_l2_l3</th><th>left_neural_foraminal_narrowing_l3_l4</th><th>left_neural_foraminal_narrowing_l4_l5</th><th>left_neural_foraminal_narrowing_l5_s1</th><th>right_neural_foraminal_narrowing_l1_l2</th><th>right_neural_foraminal_narrowing_l2_l3</th><th>right_neural_foraminal_narrowing_l3_l4</th><th>right_neural_foraminal_narrowing_l4_l5</th><th>right_neural_foraminal_narrowing_l5_s1</th><th>left_subarticular_stenosis_l1_l2</th><th>left_subarticular_stenosis_l2_l3</th><th>left_subarticular_stenosis_l3_l4</th><th>left_subarticular_stenosis_l4_l5</th><th>left_subarticular_stenosis_l5_s1</th><th>right_subarticular_stenosis_l1_l2</th><th>right_subarticular_stenosis_l2_l3</th><th>right_subarticular_stenosis_l3_l4</th><th>right_subarticular_stenosis_l4_l5</th><th>right_subarticular_stenosis_l5_s1</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>100206310</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Severe&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Severe&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Severe&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Severe&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Normal/Mild&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Moderate&quot;</td><td>&quot;Moderate&quot;</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"train_studies_metadata_df_final_2.filter((pl.col('study_id')==100206310) & (pl.col('row_id')=='100206310_spinal_canal_stenosis_l1_l2'))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:37:46.436004Z","iopub.execute_input":"2024-09-04T05:37:46.436774Z","iopub.status.idle":"2024-09-04T05:37:46.445502Z","shell.execute_reply.started":"2024-09-04T05:37:46.436731Z","shell.execute_reply":"2024-09-04T05:37:46.444432Z"},"trusted":true},"execution_count":84,"outputs":[{"execution_count":84,"output_type":"execute_result","data":{"text/plain":"shape: (1, 6)\n┌───────────┬─────────────────────────────────┬───────────────┬─────────────┬──────────┬────────┐\n│ study_id  ┆ row_id                          ┆ sample_weight ┆ normal_mild ┆ moderate ┆ severe │\n│ ---       ┆ ---                             ┆ ---           ┆ ---         ┆ ---      ┆ ---    │\n│ i64       ┆ str                             ┆ i32           ┆ i32         ┆ i32      ┆ i32    │\n╞═══════════╪═════════════════════════════════╪═══════════════╪═════════════╪══════════╪════════╡\n│ 100206310 ┆ 100206310_spinal_canal_stenosi… ┆ 1             ┆ 1           ┆ 0        ┆ 0      │\n└───────────┴─────────────────────────────────┴───────────────┴─────────────┴──────────┴────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (1, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>study_id</th><th>row_id</th><th>sample_weight</th><th>normal_mild</th><th>moderate</th><th>severe</th></tr><tr><td>i64</td><td>str</td><td>i32</td><td>i32</td><td>i32</td><td>i32</td></tr></thead><tbody><tr><td>100206310</td><td>&quot;100206310_spinal_canal_stenosi…</td><td>1</td><td>1</td><td>0</td><td>0</td></tr></tbody></table></div>"},"metadata":{}}]},{"cell_type":"code","source":"print(np.array([[1,0,0]]).shape)\nprint(y_proba.shape)\nprint(np.array([1]).shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:41:13.560166Z","iopub.execute_input":"2024-09-04T05:41:13.560566Z","iopub.status.idle":"2024-09-04T05:41:13.566335Z","shell.execute_reply.started":"2024-09-04T05:41:13.560527Z","shell.execute_reply":"2024-09-04T05:41:13.565281Z"},"trusted":true},"execution_count":89,"outputs":[{"name":"stdout","text":"(1, 3)\n(1, 3)\n(1,)\n","output_type":"stream"}]},{"cell_type":"code","source":"print(log_loss(np.array([[1,0,0]]), y_proba, sample_weight=np.array([1])))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:40:23.088156Z","iopub.execute_input":"2024-09-04T05:40:23.088538Z","iopub.status.idle":"2024-09-04T05:40:23.095948Z","shell.execute_reply.started":"2024-09-04T05:40:23.088502Z","shell.execute_reply":"2024-09-04T05:40:23.095075Z"},"trusted":true},"execution_count":86,"outputs":[{"name":"stdout","text":"2.68785348332437\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## How weighted log loss works for N samples\n### Using the example below\n#### Two samples, each sample has 3 classes with true labels (y_true) & predicted probabilities (y_pred)\n#### the 1st class has a weight of 1 and 2nd class has a weight 2. The number of weights correspond to the true labels of the two samples, \n#### the weight associated with the 1st class is 1 and with the 2nd class is 2, since the first sample belongs to class 1, hence 1 will be multiplied with the predicted probability","metadata":{}},{"cell_type":"code","source":"y_true = np.array([[1,0,0],[0,1,0]])\ny_pred = np.array([[0.0680268 , 0.87121695, 0.06075624],[0.0680268 , 0.87121695, 0.06075624]])\nsample_weight = np.array([1,2])\n\nprint(log_loss(y_true, y_pred, sample_weight=sample_weight))\nprint(log_loss(y_true, y_pred))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:21:09.525238Z","iopub.execute_input":"2024-09-04T05:21:09.526069Z","iopub.status.idle":"2024-09-04T05:21:09.536341Z","shell.execute_reply.started":"2024-09-04T05:21:09.526024Z","shell.execute_reply":"2024-09-04T05:21:09.535232Z"},"trusted":true},"execution_count":54,"outputs":[{"name":"stdout","text":"0.9878606690208339\n1.4128588827163582\n","output_type":"stream"}]},{"cell_type":"code","source":"# Loss of 1st sample\nloss_1 = -(1 * np.log(0.0680268) + 0 * np.log(0.87121695) + 0 * np.log(0.06075624))\n# Loss of 2nd sample\nloss_2 = -(0 * np.log(0.0680268) + 1 * np.log(0.87121695) + 0 * np.log(0.06075624))\n\nloss_without_wt = np.average([loss_1, loss_2])\nloss_with_wt = np.average([loss_1, loss_2], weights=[1,2])\n\nprint(\"loss_without_wt : \", loss_without_wt)\nprint(\"loss_with_wt : \", loss_with_wt)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T05:41:56.455337Z","iopub.execute_input":"2024-09-04T05:41:56.456065Z","iopub.status.idle":"2024-09-04T05:41:56.462830Z","shell.execute_reply.started":"2024-09-04T05:41:56.456023Z","shell.execute_reply":"2024-09-04T05:41:56.461832Z"},"trusted":true},"execution_count":91,"outputs":[{"name":"stdout","text":"loss_without_wt :  1.4128588927163583\nloss_with_wt :  0.9878606790208341\n","output_type":"stream"}]},{"cell_type":"code","source":"model.evaluate(test_ds)","metadata":{"execution":{"iopub.status.busy":"2024-08-23T15:42:13.478962Z","iopub.execute_input":"2024-08-23T15:42:13.479450Z","iopub.status.idle":"2024-08-23T15:50:35.534686Z","shell.execute_reply.started":"2024-08-23T15:42:13.479405Z","shell.execute_reply":"2024-08-23T15:50:35.533446Z"},"trusted":true},"execution_count":null,"outputs":[]}]}