SatNOGS transmitters.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import urllib.request\n",
    "import pickle\n",
    "import collections\n",
    "import pathlib\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "GR_SATELLITES_PATH = '/home/daniel/debian_testing_chroot/home/daniel/gr-satellites'\n",
    "OUTPUT_DIR = '/tmp/satyamls'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "satellites = json.load(urllib.request.urlopen('https://db.satnogs.org/api/satellites/?format=json'))\n",
    "transmitters = json.load(urllib.request.urlopen('https://db.satnogs.org/api/transmitters/?format=json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "modes = {'AFSK1k2' : ('AFSK', False, 1200), 'FSK19k2' : ('FSK', True, 19200), 'FSK1k2' : None, 'FSK2k4' : None, 'FSK4k8' : ('FSK', True, 4800),\\\n",
    "        'FSK9k6' : ('FSK', True, 9600), 'GFSK19k2' : ('FSK', True, 19200), 'GFSK1k2' : None, 'GFSK2k4' : None, 'GFSK4k8' : ('FSK', True, 4800),\\\n",
    "        'GFSK9k6' : ('FSK', True, 9600), 'GMSK19k2' : ('FSK', True, 19200), 'GMSK1k2' : None, 'GMSK2k4' : None, 'GMSK4k8' : ('FSK', True, 4800),\\\n",
    "        'GMSK9k6' : ('FSK', True, 9600), 'MSK19k2' : ('FSK', True, 19200), 'MSK1k2' : None, 'MSK2k4' : None, 'MSK4k8' : ('FSK', True, 4800),\\\n",
    "        'MSK9k6' : ('FSK', True, 9600),}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "norad_list = set([t['norad_cat_id'] for t in transmitters if t['mode'] in modes.keys() and t['type'] == 'Transmitter'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ax25_headers(norad):\n",
    "    tlm = json.load(urllib.request.urlopen(f'https://db.satnogs.org/api/telemetry/?format=json&satellite={norad}'))\n",
    "    headers = [bytes.fromhex(t['frame'][:16*2]) for t in tlm]\n",
    "    return collections.Counter(headers)\n",
    "\n",
    "def to_address(header):\n",
    "    return bytes([a >> 1 for a in header[:14]])\n",
    "\n",
    "def load_info(file):\n",
    "    with open(file, 'rb') as f:\n",
    "        info = pickle.load(f)\n",
    "    return info\n",
    "\n",
    "def addresses(frames):\n",
    "    return {to_address(a[0]):a[1] for a in frames.items()}\n",
    "\n",
    "def sat_from_name(name):\n",
    "    return [s for s in satellites if s['name'] == name][0]\n",
    "\n",
    "def name(norad):\n",
    "    return [s['name'] for s in satellites if s['norad_cat_id'] == norad][0]\n",
    "\n",
    "def norad(name):\n",
    "    return [s['norad_cat_id'] for s in satellites if s['name'] == name][0]\n",
    "\n",
    "def addresses_look_ok(frame):\n",
    "    if frame is None:\n",
    "        return False\n",
    "    addr = str(to_address(frame), encoding = 'ascii')\n",
    "    return addr[:6].isprintable() and addr[7:7+6].isprintable()\n",
    "\n",
    "def get_most_popular_header(headers):\n",
    "    s = sorted(headers.items(), key = lambda a: a[1], reverse = True)\n",
    "    if len(s) == 1:\n",
    "        return s[0][0]\n",
    "    if len(s) == 0:\n",
    "        return None\n",
    "    # the most popular needs to be 4 times more popular than the rest\n",
    "    if s[0][1] > s[1][1] * 4:\n",
    "        return s[0][0]\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def baudrate_str(baud):\n",
    "    return f'{baud//1000}k{(baud-baud//1000*1000)//100}'\n",
    "\n",
    "def norad_from_satyaml(path):\n",
    "    with open(path) as f:\n",
    "        y = yaml.safe_load(f)\n",
    "    return y['norad']\n",
    "\n",
    "def good_filename(name):\n",
    "    return name.replace('/', '_').replace(' ', '_').replace('.', '_') + '.yml'\n",
    "\n",
    "def satyaml(sat):\n",
    "    names = [s.strip() for s in sat['names'].replace('\\r\\n',',').split(',') if len(s.strip()) > 0]\n",
    "    alternatives = '\\nalternative_names:\\n' + '\\n'.join(['  - ' + s for s in names]) if len(names) else ''\n",
    "    return f\"\"\"name: {sat['name']}{alternatives}\n",
    "norad: {sat['norad_cat_id']}\n",
    "data:\n",
    "  &tlm Telemetry:\n",
    "    telemetry: ax25\n",
    "transmitters:\n",
    "\"\"\" + '\\n'.join([f\"\"\"  {baudrate_str(modes[t['mode']][2])} {modes[t['mode']][0]} downlink:\n",
    "    frequency: {t['downlink_low']*1e-6:.03f}e+6\n",
    "    modulation: {modes[t['mode']][0]}\n",
    "    baudrate: {modes[t['mode']][2]}\n",
    "    framing: AX.25{' G3RUH' if modes[t['mode']][1] else ''}\n",
    "    data:\n",
    "    - *tlm\"\"\" for t in transmitters if t['norad_cat_id'] == sat['norad_cat_id'] and t['mode'] in modes]) + '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for norad in norad_list:\n",
    "#    with open(f'satnogsdb/ax25_addresses/{norad}', 'wb') as f:\n",
    "#        try:\n",
    "#            headers = get_ax25_headers(norad)\n",
    "#        except:\n",
    "#            print(f'{name(norad)} {norad} failed')\n",
    "#        pickle.dump(headers, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = {int(f.name) : load_info(f) for f in pathlib.Path('satnogsdb/ax25_addresses/').glob('*')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {name(f[0]) : get_most_popular_header(f[1]) for f in frames.items()}\n",
    "suspicious_addresses = {h[0] : to_address(h[1]) for h in headers.items() if h[1] is not None and not addresses_look_ok(h[1])}\n",
    "ok_addresses = {h[0] : to_address(h[1]) for h in headers.items() if h[1] is not None and addresses_look_ok(h[1])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AALTO-2': b'%Iz\\x17\\x073\\x178bQ9WiP',\n",
       " 'AOBA VELOX-IV': b'\\x16\\x0b\\x04k:){\\x08!9\\x06c\"Y',\n",
       " 'BEESAT-2': b'\\x01*\\x02\\x00\\x00\\x00\\x00\\x00\\x1f\\x00\"(\\x18!',\n",
       " 'BEESAT-3': b'\\x01*\\x02\\x00\\x00\\x00\\x00\\x00\\x1f\\x00\"(\\x18!',\n",
       " 'BUGSAT-1': b'CQ    0LU7AA\\x00p',\n",
       " 'ENDUROSAT ONE': b'1Z|-b \\rcFV\\tS\\x08\\x00',\n",
       " 'IRVINE-01': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00KK6CVB\\x01',\n",
       " 'InnoSat 2': b'\\x00@SA\\x01-h&\\x1d\\x00\\x00\\x00m\\x00',\n",
       " 'LQSAT': b'P?=Y\\x1a\\x01q\\x17s\\x1c\\x7fY',\n",
       " 'M-CUBED & EXP-1 PRIME': b'\\x199\\x02\\x01%\\x1b&)*\\x16\\x18\\x10\\x04\\x10',\n",
       " 'NUSAT 1': b'K3%\\x047K#\\x07=\\x1eHNzp',\n",
       " 'NUSAT 2': b\"QG'kHsq\\x13R\",\n",
       " 'PEGASUS': b\")''\\x18\\x19 *CC2C\\x004\\x00\",\n",
       " 'QBEE': b'\\'\\'\\x18\\x18)\"\\x00\\'\\'\\x18\\x18)\"\\x00',\n",
       " 'SNUGLITE': b'\")\\x18\"$\\x00\\x00\")\\x18\"$\\x00\\x00',\n",
       " 'TECHNOSAT': b'\\x00>\\x02\\x00\\x00\\x00\\x00\\x00\\x19\\x07\"(\\x18*'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suspicious_addresses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AALTO-1': b'OH2AGS\\x00OH2A1S\\x0b',\n",
       " 'AAUSAT-II': b'UI    0OZ2CUB0',\n",
       " 'ACRUX-1': b'NOCALL0CQ    p',\n",
       " 'ARMADILLO': b'CQ    0BJ1SK 0',\n",
       " 'ATLANTIS': b'CQ    0WD8DOXp',\n",
       " 'Astrocast 0.1': b'CQ    0HB9GSF0',\n",
       " 'Astrocast 0.2': b'CQ    0HB9GSF0',\n",
       " 'BISONSAT': b'WH2XPM0N7SKC p',\n",
       " 'BRICSat-2': b'APOFF 0USNAP11',\n",
       " 'CAS-4A': b'CQ    0BJ1SK 0',\n",
       " 'CAS-4B': b'CQ    0BJ1SL 0',\n",
       " 'CHALLENGER': b'QBUS010CQ    p',\n",
       " 'CHOMPTT': b'WI2XVT0WI2XVT0',\n",
       " 'COLUMBIA': b'CQ    0KD8CJTp',\n",
       " 'CSIM-FD': b'BCT   0CSIM  p',\n",
       " 'CUBEBUG-2': b'CQ    0CUBEB2v',\n",
       " 'CubeBel-1': b'EU1XX 0EU10S 0',\n",
       " 'E-ST@R-II': b'ALLALLpESTAR 1',\n",
       " 'ECAMSAT': b'UNDEF pKE6QLL0',\n",
       " 'ELFIN-A': b'W6YRA90WJ2XNXp',\n",
       " 'ELFIN-B': b'W6YRA90WJ2XOXp',\n",
       " 'Eaglet-I': b'OHBROMpEAGLET0',\n",
       " 'FIREBIRD 3': b'K7MSU 0K7MSU p',\n",
       " 'FIREBIRD 4': b'K7MSU 0K7MSU p',\n",
       " 'GRIFEX': b'KD8SPS0CQ    p',\n",
       " 'INS-1C': b'INDUST:INDUSRz',\n",
       " 'IRAZU': b'TI0TEC0TI0IRA0',\n",
       " 'KrakSat': b'APDST46SR9KRA6',\n",
       " 'LITUANICASAT-2': b'CQ    \\x00LY0LS \\x00',\n",
       " 'LightSail-2': b'N6CP  \\x01KK6HIT\\n',\n",
       " 'M6P': b'CQ    \\x00LY0LS \\x00',\n",
       " 'MCUBED-2': b'CQ    0NOCALLp',\n",
       " 'MINXSS': b'CQ    0MINXSSp',\n",
       " 'MinXSS 2': b'CQ    \\x00LY0LS \\x00',\n",
       " 'NEXUS': b'NEXUS 0JS1YAV0',\n",
       " 'NIUSAT': b'NIUGSR:NIUOBTz',\n",
       " 'NO-84': b'APRSON0PSAT  0',\n",
       " 'NODES 1': b'CQ    0KE6QLL0',\n",
       " 'NODES 2': b'CQ    0KE6QLL0',\n",
       " 'NSIGHT-1': b'ZS1SCSpON02AZ0',\n",
       " 'O/OREOS': b'UNDEF pKF6JBP0',\n",
       " 'PHOENIX': b'NCKUGSpON01TW0',\n",
       " 'PHONESAT 2.4': b'CQ    0KJ6KRW0',\n",
       " 'POLYITAN-1': b'QST   pEM0UKP0',\n",
       " 'RANGE A': b'CQ    \\x00LY0LS \\x00',\n",
       " 'ROBUSTA-1B': b'F4KJE 0FX6FR 0',\n",
       " 'SKCUBE': b'OM3KAApOM9SAT0',\n",
       " 'SiriusSat-1': b'R2ANF 0RS13S 1',\n",
       " 'SiriusSat-2': b'R2ANF 0RS14S 2',\n",
       " 'Swiatowid': b'APDST46SR6SAT6',\n",
       " 'TBEX-A': b'KD8SPS0KF6RFXp',\n",
       " 'TBEX-B': b'KD8SPS0KF6RFXp',\n",
       " 'TIGRISAT': b'CQ    0HNATIGp',\n",
       " 'Tanusha-3': b'ALL   0RS8S  0',\n",
       " 'UBAKUSAT': b'TA2MKApYM1RAS0',\n",
       " 'UCLSAT': b'M3END zON03GB3',\n",
       " 'UNISAT-6': b'CQ    0II0US p',\n",
       " 'URSA MAIOR': b'CQ    0URSAMR0',\n",
       " 'UWE-3': b'DD0UWE0DP0UWGp',\n",
       " 'UWE-4': b'DD0UWE0DP0UWHp',\n",
       " 'X-CUBESAT': b'TLM   \\x00ON01FR\\x00',\n",
       " 'XW-2A': b'CQ    0BJ1SB 0',\n",
       " 'XW-2B': b'CQ    0BJ1SC 0',\n",
       " 'XW-2C': b'CQ    0BJ1SD 0',\n",
       " 'XW-2D': b'CQ    0BJ1SE 0',\n",
       " 'XW-2E': b'CQ    0BJ1SF 0',\n",
       " 'XW-2F': b'CQ    0BJ1SG 0',\n",
       " 'ZACUBE-1': b'012345p1234560',\n",
       " 'al-Farabi-2': b'W6YRA90WJ2XNXp'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ok_addresses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "blacklist = {'RANGE A' # LY0LS is another spacecraft\n",
    "            }\n",
    "suspicious_good = {'BUGSAT-1', 'ENDUROSAT ONE', 'IRVINE-01', 'QBEE', 'SNUGLITE'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "good_addresses = (ok_addresses.keys() - blacklist) | suspicious_good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "gr_satellites_norads = {norad_from_satyaml(f) for f in (pathlib.Path(GR_SATELLITES_PATH) / 'python' / 'satyaml').glob('*.yml')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Astrocast 0.1', 'CHALLENGER', 'Swiatowid'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gr_satellites_names = {n for n in good_addresses if norad(n) in gr_satellites_norads}\n",
    "gr_satellites_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error with INS-1C\n",
      "Error with NIUSAT\n",
      "Error with AAUSAT-II\n"
     ]
    }
   ],
   "source": [
    "for name in good_addresses - gr_satellites_names:\n",
    "    with open(pathlib.Path(OUTPUT_DIR) / good_filename(name), 'w') as f:\n",
    "        try:\n",
    "            y = satyaml(sat_from_name(name))\n",
    "        except:\n",
    "            print('Error with', name)\n",
    "        else:\n",
    "            f.write(y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}