create-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jupyter environment detected. Enabling Open3D WebVisualizer.\n",
      "[Open3D INFO] WebRTC GUI backend enabled.\n",
      "[Open3D INFO] WebRTCWindowSystem: HTTP handshake server disabled.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import open3d as o3d\n",
    "import numpy as np\n",
    "import h5py\n",
    "import random\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Print grid in readable format\n",
    "def pretty_print(grid):\n",
    "    for row in grid:\n",
    "        print(row)\n",
    "\n",
    "#from path of description file get the coupling grid, model data and \n",
    "#remove models with error\n",
    "def process_file(path):\n",
    "    with open(path,'r') as f:\n",
    "        #don't consider first two lines\n",
    "        f.readline()           #segmenti x x x\n",
    "        f.readline()           #rotazioni x x x\n",
    "\n",
    "        line=f.readline()      #numero pezzi x  \n",
    "\n",
    "        #get number of fragments\n",
    "        number=int(re.findall(\"\\d+\",line)[0])\n",
    "\n",
    "        #coupling matrix\n",
    "        grid=[]\n",
    "        for i in range(number):\n",
    "            line=f.readline()\n",
    "            ret=re.findall('-1|0|1',line)\n",
    "            grid.append(list(map(int,ret)))\n",
    "\n",
    "        #model data\n",
    "        model_names=[]\n",
    "        non_valid_indeces=[]\n",
    "        for m in range(number):\n",
    "            f.readline()        #blank line\n",
    "            name=f.readline()   #model name\n",
    "            mesh=f.readline()   #mesh n\n",
    "            f.readline()        #external n\n",
    "            f.readline()        #internal n\n",
    "\n",
    "            #if mesh=0 I don't consider the element\n",
    "            mesh_n=int(mesh.rstrip().split(\" \")[1]) \n",
    "            if mesh_n!=0: \n",
    "                #try create file name\n",
    "                file_name= name.rstrip().replace(\".\",\"_\")\n",
    "                model_names.append(file_name)          \n",
    "            else:\n",
    "                #saving indices to remove later\n",
    "                non_valid_indeces.append(m)\n",
    "\n",
    "        #removing elements from grid\n",
    "        #sorted in reverse to avoid wrong index\n",
    "        for index in sorted(non_valid_indeces, reverse=True):\n",
    "            #remove row\n",
    "            del grid[index]\n",
    "            #remove columns\n",
    "            for row in grid:\n",
    "                del row[index]\n",
    "        \n",
    "        return grid,model_names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get a set of fragments (i.e. a subfolder)\n",
    "def get_set(folder,verbose=True):\n",
    "    folder_path=os.path.join(main_path,folder)\n",
    "    models_file=[]\n",
    "\n",
    "    #files are either text files or models\n",
    "    for file in os.listdir(folder_path):\n",
    "        if '.txt' in file:\n",
    "            description_file=file\n",
    "        elif '.stl' in file:\n",
    "            #not used, they are not in the same order of the file\n",
    "            models_file.append(file)\n",
    "\n",
    "    #parsing description file\n",
    "    full_description_file=os.path.join(folder_path,description_file)\n",
    "    grid,model_names=process_file(full_description_file)\n",
    "\n",
    "    #get path of models of current set\n",
    "    model_prefix=folder.replace(\"generatedTest_\",\"\")\n",
    "    complete_models_path=[]\n",
    "    for i in range(len(model_names)):\n",
    "        #e.g. 2021_11_29_10_00_40_Cube_001.stl\n",
    "        correct_model_name=f\"{model_prefix}_{model_names[i]}.stl\"\n",
    "        #saving only names, could be useful but now not used\n",
    "        model_names[i]=correct_model_name\n",
    "\n",
    "        complete_models_path.append(os.path.join(folder_path,correct_model_name))\n",
    "\n",
    "    #create set and put into list of sets\n",
    "    set={\"models\":complete_models_path,\"grid\":grid}\n",
    "    \n",
    "    if verbose:\n",
    "        print(\"A set: \\n\")\n",
    "        print(\"Models: \",set[\"models\"])\n",
    "        print(\"Grid: \")\n",
    "        pretty_print(set[\"grid\"])\n",
    "\n",
    "    return set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 1), (0, 2), (0, 3), (0, 4), (1, 2), (1, 3), (1, 4), (2, 3), (2, 4), (3, 4)]\n"
     ]
    }
   ],
   "source": [
    "#Create all the possible pairs of fragments\n",
    "#duplicates are not considered, e.g.: (i,j) - (j,i)\n",
    "def create_pairs(num):\n",
    "    lista = []\n",
    "    for i in range(num):\n",
    "        for j in range(i+1, num):\n",
    "            lista.append((i, j))\n",
    "\n",
    "    return lista\n",
    "\n",
    "print(create_pairs(5))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_dataset_pairs(main_path,max_elements=-1,alpha=1):\n",
    "    num_total_pairs=0\n",
    "    \n",
    "    pairs_at_folder=[]  # save number of total pairs at the end of each folder, needed to split in train-test\n",
    "    #get total number of pairs we consider:\n",
    "    for folder in os.listdir(main_path):\n",
    "        #check only folders, not files\n",
    "        if '.' not in folder:\n",
    "            fragment_set=get_set(folder,verbose=False)  #set of fragments of one model\n",
    "\n",
    "            #get total number of adjacent pairs of fragments\n",
    "            #i.e. number of 1 in the grid\n",
    "            grid = np.array(fragment_set[\"grid\"])\n",
    "            unique, counts = np.unique(grid, return_counts=True)\n",
    "            dic = dict(zip(unique, counts))\n",
    "            #divide by two because we consider half of the pairs, not both of (i,j),(j,i)\n",
    "            num_zeros = int(dic[0]/2)\n",
    "            num_ones = int(dic[1]/2)\n",
    "\n",
    "            #considerare solo alpha*N coppie non adiacenti\n",
    "            max_not_adjacent_pairs=alpha*num_ones\n",
    "            estimated_num_pairs=int(min(max_not_adjacent_pairs,num_zeros)+num_ones)\n",
    "            num_total_pairs+=estimated_num_pairs\n",
    "            pairs_at_folder.append(num_total_pairs)\n",
    "\n",
    "    if max_elements>0:\n",
    "        num_total_pairs=min(max_elements,num_total_pairs)\n",
    "    print(\"dataset will contain:\", num_total_pairs,\" total pairs\")\n",
    "    return num_total_pairs,pairs_at_folder\n",
    "   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def get_pointcloud(mesh,num_points):\n",
    "    pointcloud=mesh.sample_points_poisson_disk(num_points)\n",
    "\n",
    "    center=pointcloud.get_center()\n",
    "    pointcloud=pointcloud.translate(center*-1)\n",
    "\n",
    "    return pointcloud\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00000000e+00, 1.94289029e-17, 5.55111512e-18])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path=r\"produzione_29112021\\generatedTest_2021_11_29_10_00_40\\2021_11_29_10_00_40_Cube.stl\"\n",
    "mesh=o3d.io.read_triangle_mesh(path)\n",
    "p=get_pointcloud(mesh,10)\n",
    "p.get_center()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataset(main_path,num_points,max_elements=-1,alpha=1):\n",
    "    \n",
    "    num_elements=0\n",
    "    num_pairs=0             # number of pairs we put in dataset (train+test)\n",
    "    num_train_pairs=0       # number of pairs in train\n",
    "    num_test_pairs=0        # number of pairs in test\n",
    "    num_total_pairs=0       # total number of pairs, including those we don't put in dataset (~1.5 Millions)\n",
    "    folder_index = 0\n",
    "    tot_num_adjacent=0\n",
    "\n",
    "    sets=[]\n",
    "\n",
    "    \n",
    "\n",
    "    print(\"Creating dataset...\")\n",
    "    dataset_total_pairs,pairs_at_folder=count_dataset_pairs(main_path,max_elements,alpha)\n",
    "\n",
    "    dataset_file=None\n",
    "    dataset_file_name=f\"dataset_{dataset_total_pairs}pairs_{num_points}points_center_{int(time.time())}.hdf5\"\n",
    "\n",
    "    with open(\".gitignore\",\"a\") as f:\n",
    "        f.write(dataset_file_name)\n",
    "\n",
    "    try: \n",
    "        dataset_file.close()\n",
    "        os.remove(dataset_file_name)\n",
    "    except:\n",
    "        print(\"file dataset not found, creating\")\n",
    "    \n",
    "    test_percent=0.15\n",
    "    approximate_train_total_pairs=dataset_total_pairs*(1-test_percent)\n",
    "\n",
    "    #indexes of folder that we use as test\n",
    "    test_folder_indexes=np.where(np.array(pairs_at_folder)>=approximate_train_total_pairs)[0]\n",
    "    \n",
    "    train_total_pairs=pairs_at_folder[test_folder_indexes[0]-1]\n",
    "\n",
    "    dataset_file=h5py.File(dataset_file_name, 'w')\n",
    "    train_data=dataset_file.create_dataset(\"train_data\", (train_total_pairs,2,num_points,3))\n",
    "    train_labels=dataset_file.create_dataset(\"train_labels\", (train_total_pairs,),dtype='i')\n",
    "    \n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    total_dataset_start_time=time.time()\n",
    "\n",
    "    print(f\"Each point cloud is sampled with {num_points} points\\n\\n\")\n",
    "    for folder in os.listdir(main_path):\n",
    "        #check only folders, not files\n",
    "        if '.' not in folder:\n",
    "            \n",
    "            if folder_index==test_folder_indexes[0]:\n",
    "                # from now on I consider test set, I create the datasets in the file\n",
    "                print(\"\\n\\n\\nTEST SET STARTING....\\n\\n\\n\")\n",
    "                test_total_pairs=dataset_total_pairs-train_total_pairs\n",
    "                test_data=dataset_file.create_dataset(\"test_data\", (test_total_pairs,2,num_points,3))\n",
    "                test_labels=dataset_file.create_dataset(\"test_labels\", (test_total_pairs,),dtype='i')\n",
    "\n",
    "            \n",
    "            set_data=[]\n",
    "            set_labels=[]\n",
    "\n",
    "            set_start_time=time.time()\n",
    "            print(f\"\\n\\nStarting set {folder_index}: folder - {folder}\")            \n",
    "\n",
    "            fragment_set=get_set(folder,verbose=False)  #set of fragments of one model\n",
    "            num_elements+=len(fragment_set[\"models\"])\n",
    "            sets.append(fragment_set)\n",
    "            \n",
    "            #save first pointcloud of meshes\n",
    "            set_pointcloud=[]\n",
    "            for path in fragment_set[\"models\"]:\n",
    "                mesh=o3d.io.read_triangle_mesh(path)\n",
    "                pointcloud=get_pointcloud(mesh,num_points)\n",
    "                set_pointcloud.append(pointcloud)\n",
    "\n",
    "            #get indeces of pairs\n",
    "            pairs=create_pairs(len(fragment_set[\"models\"]))\n",
    "\n",
    "            #shuffle to get random pairs not in order\n",
    "            random.shuffle(pairs)\n",
    "\n",
    "            #get total number of adjacent pairs of fragments\n",
    "            #i.e. number of 1 in the grid\n",
    "            grid = np.array(fragment_set[\"grid\"])\n",
    "            unique, counts = np.unique(grid, return_counts=True)\n",
    "            dic = dict(zip(unique, counts))\n",
    "            #divide by two because we consider half of the pairs, not both of (i,j),(j,i)\n",
    "            num_zeros = int(dic[0]/2)\n",
    "            num_ones = int(dic[1]/2)\n",
    "\n",
    "            #considerare solo a*N coppie non adiacenti\n",
    "            max_not_adjacent_pairs=alpha*num_ones\n",
    "\n",
    "            estimated_num_pairs=int(min(max_not_adjacent_pairs,num_zeros)+num_ones)\n",
    "\n",
    "            print(\"Set stats: \")\n",
    "            print(f\"  --number of fragments: {len(fragment_set['models'])}\")\n",
    "            print(f\"  --total adjacent pairs: {num_ones}; total not adjacent pairs: {num_zeros}\")\n",
    "            print(f\"  --dataset for set will contain: {estimated_num_pairs} pairs\")\n",
    "            print(f\"     --> adj: {num_ones}    n-adj: {int(min(max_not_adjacent_pairs,num_zeros))}\")\n",
    "\n",
    "            #some stats\n",
    "            set_num_not_adjacent=0\n",
    "            set_num_adjacent=0\n",
    "            current_set_pairs=0\n",
    "            current_set_not_adj=0\n",
    "            \n",
    "            for pair in pairs:\n",
    "                #if limit of maximum pairs is not exceeded\n",
    "                if max_elements<0 or num_pairs<=max_elements:\n",
    "                        \n",
    "                    num_total_pairs+=1\n",
    "                    idx1,idx2=pair\n",
    "                    label=fragment_set[\"grid\"][idx1][idx2]\n",
    "                    \n",
    "                    #total number of not adjacent\n",
    "                    if label==0:\n",
    "                        set_num_not_adjacent+=1\n",
    "                    else:\n",
    "                        set_num_adjacent+=1\n",
    "                        tot_num_adjacent+=1\n",
    "                    \n",
    "                    #if adjacent pair or not adjacent pairs limit not exceeded\n",
    "                    if label==1 or (label==0 and set_num_not_adjacent<=max_not_adjacent_pairs):\n",
    "                        if folder_index>=test_folder_indexes[0]:\n",
    "                            num_test_pairs+=1\n",
    "                        else:\n",
    "                            num_train_pairs+=1\n",
    "                            \n",
    "                        #add stats for number of fragments in dataset and current set\n",
    "                        num_pairs+=1\n",
    "                        current_set_pairs+=1\n",
    "\n",
    "                        #add stats for adjacent fragment in dataset\n",
    "                        if label==0: current_set_not_adj+=1\n",
    "\n",
    "                        pointcloud1=set_pointcloud[idx1]\n",
    "                        pointcloud2=set_pointcloud[idx2]\n",
    "\n",
    "                        #generate pair and add to dataset\n",
    "                        pointcloud_pair=[ np.asarray(pointcloud1.points) ,  np.asarray(pointcloud2.points) ]\n",
    "\n",
    "                        set_data.append(pointcloud_pair)\n",
    "                        set_labels.append(label)\n",
    "\n",
    "                        \n",
    "                        #print(f\"Completed: {(current_set_pairs/estimated_num_pairs) *100}%\",end='')\n",
    "                        #print('\\r', end='')\n",
    "\n",
    "            set_elapsed_time=time.time()-set_start_time\n",
    "            print(\"Set completed in: %.3f seconds\" % (set_elapsed_time),end=\": \")\n",
    "            print(f\"added {current_set_pairs} pairs --> adj: {set_num_adjacent}   n_adj: {current_set_not_adj}\")\n",
    "            print(f\"currently dataset contains {num_pairs} pairs\",end=\". \")\n",
    "            print()\n",
    "\n",
    "            if folder_index in test_folder_indexes:\n",
    "                test_data[num_test_pairs-current_set_pairs:num_test_pairs]=set_data\n",
    "                test_labels[num_test_pairs-current_set_pairs:num_test_pairs]=set_labels\n",
    "            else:\n",
    "                train_data[num_train_pairs-current_set_pairs:num_train_pairs]=set_data\n",
    "                train_labels[num_train_pairs-current_set_pairs:num_train_pairs]=set_labels\n",
    "                \n",
    "            folder_index+=1\n",
    "\n",
    "\n",
    "    dataset_file.close()\n",
    "    total_dataset_time=time.time()-total_dataset_start_time\n",
    "    print(f\"Dataset contains {num_elements} fragments --> {num_total_pairs} total pairs \")\n",
    "    print(\"we consider only: \",num_pairs,\" pairs, of which \", tot_num_adjacent, \"are adjacent\")\n",
    "    print(f\"Train set: {num_train_pairs},   test set: {num_test_pairs}\")\n",
    "    print(\"total time: %.3f seconds\" % (total_dataset_time))\n",
    "    #return np.array(all_data),np.array(all_labels)\n",
    "\n",
    "\n",
    "\n",
    "#used in class ModelNet40\n",
    "#for now we don't use it\n",
    "def translate_pointcloud(pointcloud):\n",
    "    xyz1 = np.random.uniform(low=2./3., high=3./2., size=[3])\n",
    "    xyz2 = np.random.uniform(low=-0.2, high=0.2, size=[3])\n",
    "       \n",
    "    translated_pointcloud = np.add(np.multiply(pointcloud, xyz1), xyz2).astype('float32')\n",
    "    return translated_pointcloud         \n",
    "\n",
    "def load_dataset(path):\n",
    "    #carica dal file il dataset\n",
    "    return\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#here i save all the sets of fragments\n",
    "sets=[]\n",
    "\n",
    "#root folder\n",
    "main_path=\"produzione_29112021\"\n",
    "\n",
    "num_points=1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating dataset...\n",
      "dataset will contain: 90810  total pairs\n",
      "file dataset not found, creating\n",
      "Each point cloud is sampled with 1000 points\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Starting set 0: folder - generatedTest_2021_11_29_10_00_40\n",
      "Set stats: \n",
      "  --number of fragments: 8\n",
      "  --total adjacent pairs: 19; total not adjacent pairs: 9\n",
      "  --dataset for set will contain: 28 pairs\n",
      "     --> adj: 19    n-adj: 9\n",
      "Set completed in: 3.031 seconds: added 28 pairs --> adj: 19   n_adj: 9\n",
      "currently dataset contains 28 pairs. \n",
      "\n",
      "\n",
      "Starting set 1: folder - generatedTest_2021_11_29_10_01_59\n",
      "Set stats: \n",
      "  --number of fragments: 8\n",
      "  --total adjacent pairs: 19; total not adjacent pairs: 9\n",
      "  --dataset for set will contain: 28 pairs\n",
      "     --> adj: 19    n-adj: 9\n",
      "Set completed in: 2.604 seconds: added 28 pairs --> adj: 19   n_adj: 9\n",
      "currently dataset contains 56 pairs. \n",
      "\n",
      "\n",
      "Starting set 2: folder - generatedTest_2021_11_29_10_02_12\n",
      "Set stats: \n",
      "  --number of fragments: 8\n",
      "  --total adjacent pairs: 18; total not adjacent pairs: 10\n",
      "  --dataset for set will contain: 28 pairs\n",
      "     --> adj: 18    n-adj: 10\n",
      "Set completed in: 2.765 seconds: added 28 pairs --> adj: 18   n_adj: 10\n",
      "currently dataset contains 84 pairs. \n",
      "\n",
      "\n",
      "Starting set 3: folder - generatedTest_2021_11_29_10_02_50\n",
      "Set stats: \n",
      "  --number of fragments: 27\n",
      "  --total adjacent pairs: 97; total not adjacent pairs: 254\n",
      "  --dataset for set will contain: 194 pairs\n",
      "     --> adj: 97    n-adj: 97\n",
      "Set completed in: 9.046 seconds: added 194 pairs --> adj: 97   n_adj: 97\n",
      "currently dataset contains 278 pairs. \n",
      "\n",
      "\n",
      "Starting set 4: folder - generatedTest_2021_11_29_10_02_58\n",
      "Set stats: \n",
      "  --number of fragments: 26\n",
      "  --total adjacent pairs: 86; total not adjacent pairs: 239\n",
      "  --dataset for set will contain: 172 pairs\n",
      "     --> adj: 86    n-adj: 86\n",
      "Set completed in: 10.329 seconds: added 172 pairs --> adj: 86   n_adj: 86\n",
      "currently dataset contains 450 pairs. \n",
      "\n",
      "\n",
      "Starting set 5: folder - generatedTest_2021_11_29_10_03_27\n",
      "Set stats: \n",
      "  --number of fragments: 27\n",
      "  --total adjacent pairs: 94; total not adjacent pairs: 257\n",
      "  --dataset for set will contain: 188 pairs\n",
      "     --> adj: 94    n-adj: 94\n",
      "Set completed in: 10.470 seconds: added 188 pairs --> adj: 94   n_adj: 94\n",
      "currently dataset contains 638 pairs. \n",
      "\n",
      "\n",
      "Starting set 6: folder - generatedTest_2021_11_29_10_03_51\n",
      "Set stats: \n",
      "  --number of fragments: 64\n",
      "  --total adjacent pairs: 276; total not adjacent pairs: 1740\n",
      "  --dataset for set will contain: 552 pairs\n",
      "     --> adj: 276    n-adj: 276\n",
      "Set completed in: 24.456 seconds: added 552 pairs --> adj: 276   n_adj: 276\n",
      "currently dataset contains 1190 pairs. \n",
      "\n",
      "\n",
      "Starting set 7: folder - generatedTest_2021_11_29_10_04_09\n",
      "Set stats: \n",
      "  --number of fragments: 57\n",
      "  --total adjacent pairs: 242; total not adjacent pairs: 1354\n",
      "  --dataset for set will contain: 484 pairs\n",
      "     --> adj: 242    n-adj: 242\n",
      "Set completed in: 26.969 seconds: added 484 pairs --> adj: 242   n_adj: 242\n",
      "currently dataset contains 1674 pairs. \n",
      "\n",
      "\n",
      "Starting set 8: folder - generatedTest_2021_11_29_10_04_25\n",
      "Set stats: \n",
      "  --number of fragments: 59\n",
      "  --total adjacent pairs: 241; total not adjacent pairs: 1470\n",
      "  --dataset for set will contain: 482 pairs\n",
      "     --> adj: 241    n-adj: 241\n",
      "Set completed in: 19.963 seconds: added 482 pairs --> adj: 241   n_adj: 241\n",
      "currently dataset contains 2156 pairs. \n",
      "\n",
      "\n",
      "Starting set 9: folder - generatedTest_2021_11_29_10_05_19\n",
      "Set stats: \n",
      "  --number of fragments: 216\n",
      "  --total adjacent pairs: 1072; total not adjacent pairs: 22148\n",
      "  --dataset for set will contain: 2144 pairs\n",
      "     --> adj: 1072    n-adj: 1072\n",
      "Set completed in: 75.811 seconds: added 2144 pairs --> adj: 1072   n_adj: 1072\n",
      "currently dataset contains 4300 pairs. \n",
      "\n",
      "\n",
      "Starting set 10: folder - generatedTest_2021_11_29_10_06_11\n",
      "Set stats: \n",
      "  --number of fragments: 187\n",
      "  --total adjacent pairs: 930; total not adjacent pairs: 16461\n",
      "  --dataset for set will contain: 1860 pairs\n",
      "     --> adj: 930    n-adj: 930\n",
      "Set completed in: 59.210 seconds: added 1860 pairs --> adj: 930   n_adj: 930\n",
      "currently dataset contains 6160 pairs. \n",
      "\n",
      "\n",
      "Starting set 11: folder - generatedTest_2021_11_29_10_07_02\n",
      "Set stats: \n",
      "  --number of fragments: 208\n",
      "  --total adjacent pairs: 1019; total not adjacent pairs: 20509\n",
      "  --dataset for set will contain: 2038 pairs\n",
      "     --> adj: 1019    n-adj: 1019\n",
      "Set completed in: 74.250 seconds: added 2038 pairs --> adj: 1019   n_adj: 1019\n",
      "currently dataset contains 8198 pairs. \n",
      "\n",
      "\n",
      "Starting set 12: folder - generatedTest_2021_11_29_10_15_14\n",
      "Set stats: \n",
      "  --number of fragments: 120\n",
      "  --total adjacent pairs: 558; total not adjacent pairs: 6582\n",
      "  --dataset for set will contain: 1116 pairs\n",
      "     --> adj: 558    n-adj: 558\n",
      "Set completed in: 41.950 seconds: added 1116 pairs --> adj: 558   n_adj: 558\n",
      "currently dataset contains 9314 pairs. \n",
      "\n",
      "\n",
      "Starting set 13: folder - generatedTest_2021_11_29_10_16_04\n",
      "Set stats: \n",
      "  --number of fragments: 115\n",
      "  --total adjacent pairs: 532; total not adjacent pairs: 6023\n",
      "  --dataset for set will contain: 1064 pairs\n",
      "     --> adj: 532    n-adj: 532\n",
      "Set completed in: 50.627 seconds: added 1064 pairs --> adj: 532   n_adj: 532\n",
      "currently dataset contains 10378 pairs. \n",
      "\n",
      "\n",
      "Starting set 14: folder - generatedTest_2021_11_29_10_16_37\n",
      "Set stats: \n",
      "  --number of fragments: 119\n",
      "  --total adjacent pairs: 554; total not adjacent pairs: 6467\n",
      "  --dataset for set will contain: 1108 pairs\n",
      "     --> adj: 554    n-adj: 554\n",
      "Set completed in: 38.170 seconds: added 1108 pairs --> adj: 554   n_adj: 554\n",
      "currently dataset contains 11486 pairs. \n",
      "\n",
      "\n",
      "Starting set 15: folder - generatedTest_2021_11_29_10_17_44\n",
      "Set stats: \n",
      "  --number of fragments: 192\n",
      "  --total adjacent pairs: 936; total not adjacent pairs: 17400\n",
      "  --dataset for set will contain: 1872 pairs\n",
      "     --> adj: 936    n-adj: 936\n",
      "Set completed in: 68.443 seconds: added 1872 pairs --> adj: 936   n_adj: 936\n",
      "currently dataset contains 13358 pairs. \n",
      "\n",
      "\n",
      "Starting set 16: folder - generatedTest_2021_11_29_10_19_23\n",
      "Set stats: \n",
      "  --number of fragments: 176\n",
      "  --total adjacent pairs: 842; total not adjacent pairs: 14558\n",
      "  --dataset for set will contain: 1684 pairs\n",
      "     --> adj: 842    n-adj: 842\n",
      "Set completed in: 62.287 seconds: added 1684 pairs --> adj: 842   n_adj: 842\n",
      "currently dataset contains 15042 pairs. \n",
      "\n",
      "\n",
      "Starting set 17: folder - generatedTest_2021_11_29_10_20_52\n",
      "Set stats: \n",
      "  --number of fragments: 167\n",
      "  --total adjacent pairs: 778; total not adjacent pairs: 13083\n",
      "  --dataset for set will contain: 1556 pairs\n",
      "     --> adj: 778    n-adj: 778\n",
      "Set completed in: 55.005 seconds: added 1556 pairs --> adj: 778   n_adj: 778\n",
      "currently dataset contains 16598 pairs. \n",
      "\n",
      "\n",
      "Starting set 18: folder - generatedTest_2021_11_29_10_21_45\n",
      "Set stats: \n",
      "  --number of fragments: 160\n",
      "  --total adjacent pairs: 761; total not adjacent pairs: 11959\n",
      "  --dataset for set will contain: 1522 pairs\n",
      "     --> adj: 761    n-adj: 761\n",
      "Set completed in: 52.899 seconds: added 1522 pairs --> adj: 761   n_adj: 761\n",
      "currently dataset contains 18120 pairs. \n",
      "\n",
      "\n",
      "Starting set 19: folder - generatedTest_2021_11_29_10_22_52\n",
      "Set stats: \n",
      "  --number of fragments: 138\n",
      "  --total adjacent pairs: 605; total not adjacent pairs: 8848\n",
      "  --dataset for set will contain: 1210 pairs\n",
      "     --> adj: 605    n-adj: 605\n",
      "Set completed in: 49.177 seconds: added 1210 pairs --> adj: 605   n_adj: 605\n",
      "currently dataset contains 19330 pairs. \n",
      "\n",
      "\n",
      "Starting set 20: folder - generatedTest_2021_11_29_10_23_50\n",
      "Set stats: \n",
      "  --number of fragments: 146\n",
      "  --total adjacent pairs: 684; total not adjacent pairs: 9901\n",
      "  --dataset for set will contain: 1368 pairs\n",
      "     --> adj: 684    n-adj: 684\n",
      "Set completed in: 48.012 seconds: added 1368 pairs --> adj: 684   n_adj: 684\n",
      "currently dataset contains 20698 pairs. \n",
      "\n",
      "\n",
      "Starting set 21: folder - generatedTest_2021_11_29_10_24_57\n",
      "Set stats: \n",
      "  --number of fragments: 216\n",
      "  --total adjacent pairs: 1068; total not adjacent pairs: 22152\n",
      "  --dataset for set will contain: 2136 pairs\n",
      "     --> adj: 1068    n-adj: 1068\n",
      "Set completed in: 84.096 seconds: added 2136 pairs --> adj: 1068   n_adj: 1068\n",
      "currently dataset contains 22834 pairs. \n",
      "\n",
      "\n",
      "Starting set 22: folder - generatedTest_2021_11_29_10_26_01\n",
      "Set stats: \n",
      "  --number of fragments: 180\n",
      "  --total adjacent pairs: 856; total not adjacent pairs: 15254\n",
      "  --dataset for set will contain: 1712 pairs\n",
      "     --> adj: 856    n-adj: 856\n",
      "Set completed in: 59.689 seconds: added 1712 pairs --> adj: 856   n_adj: 856\n",
      "currently dataset contains 24546 pairs. \n",
      "\n",
      "\n",
      "Starting set 23: folder - generatedTest_2021_11_29_10_26_56\n",
      "Set stats: \n",
      "  --number of fragments: 210\n",
      "  --total adjacent pairs: 997; total not adjacent pairs: 20948\n",
      "  --dataset for set will contain: 1994 pairs\n",
      "     --> adj: 997    n-adj: 997\n",
      "Set completed in: 68.813 seconds: added 1994 pairs --> adj: 997   n_adj: 997\n",
      "currently dataset contains 26540 pairs. \n",
      "\n",
      "\n",
      "Starting set 24: folder - generatedTest_2021_11_29_10_27_56\n",
      "Set stats: \n",
      "  --number of fragments: 84\n",
      "  --total adjacent pairs: 360; total not adjacent pairs: 3126\n",
      "  --dataset for set will contain: 720 pairs\n",
      "     --> adj: 360    n-adj: 360\n",
      "Set completed in: 29.178 seconds: added 720 pairs --> adj: 360   n_adj: 360\n",
      "currently dataset contains 27260 pairs. \n",
      "\n",
      "\n",
      "Starting set 25: folder - generatedTest_2021_11_29_10_28_22\n",
      "Set stats: \n",
      "  --number of fragments: 83\n",
      "  --total adjacent pairs: 350; total not adjacent pairs: 3053\n",
      "  --dataset for set will contain: 700 pairs\n",
      "     --> adj: 350    n-adj: 350\n",
      "Set completed in: 30.184 seconds: added 700 pairs --> adj: 350   n_adj: 350\n",
      "currently dataset contains 27960 pairs. \n",
      "\n",
      "\n",
      "Starting set 26: folder - generatedTest_2021_11_29_10_28_46\n",
      "Set stats: \n",
      "  --number of fragments: 83\n",
      "  --total adjacent pairs: 343; total not adjacent pairs: 3060\n",
      "  --dataset for set will contain: 686 pairs\n",
      "     --> adj: 343    n-adj: 343\n",
      "Set completed in: 26.841 seconds: added 686 pairs --> adj: 343   n_adj: 343\n",
      "currently dataset contains 28646 pairs. \n",
      "\n",
      "\n",
      "Starting set 27: folder - generatedTest_2021_11_29_10_29_28\n",
      "Set stats: \n",
      "  --number of fragments: 105\n",
      "  --total adjacent pairs: 470; total not adjacent pairs: 4990\n",
      "  --dataset for set will contain: 940 pairs\n",
      "     --> adj: 470    n-adj: 470\n",
      "Set completed in: 38.891 seconds: added 940 pairs --> adj: 470   n_adj: 470\n",
      "currently dataset contains 29586 pairs. \n",
      "\n",
      "\n",
      "Starting set 28: folder - generatedTest_2021_11_29_10_31_27\n",
      "Set stats: \n",
      "  --number of fragments: 99\n",
      "  --total adjacent pairs: 427; total not adjacent pairs: 4424\n",
      "  --dataset for set will contain: 854 pairs\n",
      "     --> adj: 427    n-adj: 427\n",
      "Set completed in: 34.226 seconds: added 854 pairs --> adj: 427   n_adj: 427\n",
      "currently dataset contains 30440 pairs. \n",
      "\n",
      "\n",
      "Starting set 29: folder - generatedTest_2021_11_29_10_32_18\n",
      "Set stats: \n",
      "  --number of fragments: 103\n",
      "  --total adjacent pairs: 457; total not adjacent pairs: 4796\n",
      "  --dataset for set will contain: 914 pairs\n",
      "     --> adj: 457    n-adj: 457\n",
      "Set completed in: 39.719 seconds: added 914 pairs --> adj: 457   n_adj: 457\n",
      "currently dataset contains 31354 pairs. \n",
      "\n",
      "\n",
      "Starting set 30: folder - generatedTest_2021_11_29_10_36_43\n",
      "Set stats: \n",
      "  --number of fragments: 729\n",
      "  --total adjacent pairs: 3951; total not adjacent pairs: 261405\n",
      "  --dataset for set will contain: 7902 pairs\n",
      "     --> adj: 3951    n-adj: 3951\n",
      "Set completed in: 265.704 seconds: added 7902 pairs --> adj: 3951   n_adj: 3951\n",
      "currently dataset contains 39256 pairs. \n",
      "\n",
      "\n",
      "Starting set 31: folder - generatedTest_2021_11_29_10_41_22\n",
      "Set stats: \n",
      "  --number of fragments: 695\n",
      "  --total adjacent pairs: 3662; total not adjacent pairs: 237503\n",
      "  --dataset for set will contain: 7324 pairs\n",
      "     --> adj: 3662    n-adj: 3662\n",
      "Set completed in: 240.681 seconds: added 7324 pairs --> adj: 3662   n_adj: 3662\n",
      "currently dataset contains 46580 pairs. \n",
      "\n",
      "\n",
      "Starting set 32: folder - generatedTest_2021_11_29_10_45_42\n",
      "Set stats: \n",
      "  --number of fragments: 645\n",
      "  --total adjacent pairs: 3374; total not adjacent pairs: 204316\n",
      "  --dataset for set will contain: 6748 pairs\n",
      "     --> adj: 3374    n-adj: 3374\n",
      "Set completed in: 218.034 seconds: added 6748 pairs --> adj: 3374   n_adj: 3374\n",
      "currently dataset contains 53328 pairs. \n",
      "\n",
      "\n",
      "Starting set 33: folder - generatedTest_2021_11_29_10_48_20\n",
      "Set stats: \n",
      "  --number of fragments: 343\n",
      "  --total adjacent pairs: 1801; total not adjacent pairs: 56852\n",
      "  --dataset for set will contain: 3602 pairs\n",
      "     --> adj: 1801    n-adj: 1801\n",
      "Set completed in: 129.328 seconds: added 3602 pairs --> adj: 1801   n_adj: 1801\n",
      "currently dataset contains 56930 pairs. \n",
      "\n",
      "\n",
      "Starting set 34: folder - generatedTest_2021_11_29_10_51_20\n",
      "Set stats: \n",
      "  --number of fragments: 305\n",
      "  --total adjacent pairs: 1546; total not adjacent pairs: 44814\n",
      "  --dataset for set will contain: 3092 pairs\n",
      "     --> adj: 1546    n-adj: 1546\n",
      "Set completed in: 122.023 seconds: added 3092 pairs --> adj: 1546   n_adj: 1546\n",
      "currently dataset contains 60022 pairs. \n",
      "\n",
      "\n",
      "Starting set 35: folder - generatedTest_2021_11_29_10_53_31\n",
      "Set stats: \n",
      "  --number of fragments: 314\n",
      "  --total adjacent pairs: 1587; total not adjacent pairs: 47554\n",
      "  --dataset for set will contain: 3174 pairs\n",
      "     --> adj: 1587    n-adj: 1587\n",
      "Set completed in: 114.645 seconds: added 3174 pairs --> adj: 1587   n_adj: 1587\n",
      "currently dataset contains 63196 pairs. \n",
      "\n",
      "\n",
      "Starting set 36: folder - generatedTest_2021_11_29_10_54_15\n",
      "Set stats: \n",
      "  --number of fragments: 140\n",
      "  --total adjacent pairs: 666; total not adjacent pairs: 9064\n",
      "  --dataset for set will contain: 1332 pairs\n",
      "     --> adj: 666    n-adj: 666\n",
      "Set completed in: 55.842 seconds: added 1332 pairs --> adj: 666   n_adj: 666\n",
      "currently dataset contains 64528 pairs. \n",
      "\n",
      "\n",
      "Starting set 37: folder - generatedTest_2021_11_29_10_55_01\n",
      "Set stats: \n",
      "  --number of fragments: 135\n",
      "  --total adjacent pairs: 626; total not adjacent pairs: 8419\n",
      "  --dataset for set will contain: 1252 pairs\n",
      "     --> adj: 626    n-adj: 626\n",
      "Set completed in: 47.274 seconds: added 1252 pairs --> adj: 626   n_adj: 626\n",
      "currently dataset contains 65780 pairs. \n",
      "\n",
      "\n",
      "Starting set 38: folder - generatedTest_2021_11_29_10_55_44\n",
      "Set stats: \n",
      "  --number of fragments: 128\n",
      "  --total adjacent pairs: 602; total not adjacent pairs: 7526\n",
      "  --dataset for set will contain: 1204 pairs\n",
      "     --> adj: 602    n-adj: 602\n",
      "Set completed in: 39.258 seconds: added 1204 pairs --> adj: 602   n_adj: 602\n",
      "currently dataset contains 66984 pairs. \n",
      "\n",
      "\n",
      "Starting set 39: folder - generatedTest_2021_11_29_10_57_36\n",
      "Set stats: \n",
      "  --number of fragments: 175\n",
      "  --total adjacent pairs: 854; total not adjacent pairs: 14371\n",
      "  --dataset for set will contain: 1708 pairs\n",
      "     --> adj: 854    n-adj: 854\n",
      "Set completed in: 54.153 seconds: added 1708 pairs --> adj: 854   n_adj: 854\n",
      "currently dataset contains 68692 pairs. \n",
      "\n",
      "\n",
      "Starting set 40: folder - generatedTest_2021_11_29_10_58_19\n",
      "Set stats: \n",
      "  --number of fragments: 135\n",
      "  --total adjacent pairs: 615; total not adjacent pairs: 8430\n",
      "  --dataset for set will contain: 1230 pairs\n",
      "     --> adj: 615    n-adj: 615\n",
      "Set completed in: 43.390 seconds: added 1230 pairs --> adj: 615   n_adj: 615\n",
      "currently dataset contains 69922 pairs. \n",
      "\n",
      "\n",
      "Starting set 41: folder - generatedTest_2021_11_29_10_59_06\n",
      "Set stats: \n",
      "  --number of fragments: 162\n",
      "  --total adjacent pairs: 765; total not adjacent pairs: 12276\n",
      "  --dataset for set will contain: 1530 pairs\n",
      "     --> adj: 765    n-adj: 765\n",
      "Set completed in: 59.681 seconds: added 1530 pairs --> adj: 765   n_adj: 765\n",
      "currently dataset contains 71452 pairs. \n",
      "\n",
      "\n",
      "Starting set 42: folder - generatedTest_2021_11_29_11_02_28\n",
      "Set stats: \n",
      "  --number of fragments: 504\n",
      "  --total adjacent pairs: 2695; total not adjacent pairs: 124061\n",
      "  --dataset for set will contain: 5390 pairs\n",
      "     --> adj: 2695    n-adj: 2695\n",
      "Set completed in: 175.500 seconds: added 5390 pairs --> adj: 2695   n_adj: 2695\n",
      "currently dataset contains 76842 pairs. \n",
      "\n",
      "\n",
      "\n",
      "TEST SET STARTING....\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Starting set 43: folder - generatedTest_2021_11_29_11_05_39\n",
      "Set stats: \n",
      "  --number of fragments: 452\n",
      "  --total adjacent pairs: 2316; total not adjacent pairs: 99610\n",
      "  --dataset for set will contain: 4632 pairs\n",
      "     --> adj: 2316    n-adj: 2316\n",
      "Set completed in: 160.753 seconds: added 4632 pairs --> adj: 2316   n_adj: 2316\n",
      "currently dataset contains 81474 pairs. \n",
      "\n",
      "\n",
      "Starting set 44: folder - generatedTest_2021_11_29_11_27_51\n",
      "Set stats: \n",
      "  --number of fragments: 472\n",
      "  --total adjacent pairs: 2426; total not adjacent pairs: 108730\n",
      "  --dataset for set will contain: 4852 pairs\n",
      "     --> adj: 2426    n-adj: 2426\n",
      "Set completed in: 156.272 seconds: added 4852 pairs --> adj: 2426   n_adj: 2426\n",
      "currently dataset contains 86326 pairs. \n",
      "\n",
      "\n",
      "Starting set 45: folder - generatedTest_2021_11_29_11_30_35\n",
      "Set stats: \n",
      "  --number of fragments: 168\n",
      "  --total adjacent pairs: 796; total not adjacent pairs: 13232\n",
      "  --dataset for set will contain: 1592 pairs\n",
      "     --> adj: 796    n-adj: 796\n",
      "Set completed in: 52.880 seconds: added 1592 pairs --> adj: 796   n_adj: 796\n",
      "currently dataset contains 87918 pairs. \n",
      "\n",
      "\n",
      "Starting set 46: folder - generatedTest_2021_11_29_11_32_07\n",
      "Set stats: \n",
      "  --number of fragments: 154\n",
      "  --total adjacent pairs: 693; total not adjacent pairs: 11088\n",
      "  --dataset for set will contain: 1386 pairs\n",
      "     --> adj: 693    n-adj: 693\n",
      "Set completed in: 54.939 seconds: added 1386 pairs --> adj: 693   n_adj: 693\n",
      "currently dataset contains 89304 pairs. \n",
      "\n",
      "\n",
      "Starting set 47: folder - generatedTest_2021_11_29_11_36_31\n",
      "Set stats: \n",
      "  --number of fragments: 163\n",
      "  --total adjacent pairs: 753; total not adjacent pairs: 12450\n",
      "  --dataset for set will contain: 1506 pairs\n",
      "     --> adj: 753    n-adj: 753\n",
      "Set completed in: 52.258 seconds: added 1506 pairs --> adj: 753   n_adj: 753\n",
      "currently dataset contains 90810 pairs. \n",
      "Dataset contains 9210 fragments --> 1568204 total pairs \n",
      "we consider only:  90810  pairs, of which  45419 are adjacent\n",
      "Train set: 76842,   test set: 13968\n",
      "total time: 3261.451 seconds\n"
     ]
    }
   ],
   "source": [
    "create_dataset(main_path,num_points)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "6305d24f3e56ac2ca2871aacb4eb187216de08a2a8667a1a48c2c574de0d34f3"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 64-bit ('EAI': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}