some preparations for the next flare-on write-up

samples/__init__.py

@@ -0,0 +1,50 @@
+from typing import Optional, Dict
+
+import hashlib
+import os
+import urllib.request
+
+from refinery.units.crypto.cipher.aes import aes
+
+
+class SampleStore:
+    cache: Dict[str, bytes]
+
+    def __init__(self):
+        self.cache = {}
+
+    def download(self, sha256hash: str, key: Optional[str] = None):
+        def tobytearray(r):
+            if isinstance(r, bytearray):
+                return r
+            return bytearray(r)
+        key = key or 'REFINERYTESTDATA'
+        key = key.encode('latin1')
+        sha256hash = sha256hash.lower()
+        req = urllib.request.Request(
+            F'https://github.com/binref/refinery-test-data/blob/master/{sha256hash}.enc?raw=true')
+        try:
+            with urllib.request.urlopen(req) as response:
+                encoded_sample = tobytearray(response.read())
+        except Exception:
+            api = os.environ['MALSHARE_API']
+            req = urllib.request.Request(
+                F'https://malshare.com/api.php?api_key={api}&action=getfile&hash={sha256hash}')
+            with urllib.request.urlopen(req) as response:
+                result = tobytearray(response.read())
+        else:
+            result = encoded_sample | aes(mode='CBC', key=key) | bytearray
+            if not result or hashlib.sha256(result).hexdigest().lower() != sha256hash:
+                raise ValueError('sample did not decode correctly')
+        self.cache[sha256hash] = result
+        return result
+
+    def get(self, sha256hash: str, key: Optional[str] = None):
+        for cached, value in self.cache.items():
+            if cached.casefold() == sha256hash.casefold():
+                return value
+        else:
+            return self.download(sha256hash, key)
+
+    def __getitem__(self, sha256hash: str):
+        return self.get(sha256hash)

strip-tutorials.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Strips the Jupyter Notebooks in the Tutorial section of run count information.
+"""
+import json
+import pathlib
+
+for path in pathlib.Path.cwd().glob('./tutorials/*.ipynb'):
+    with path.open('r') as fd:
+        notebook = json.load(fd)
+    for cell in notebook['cells']:
+        cell.pop('execution_count', None)
+    with path.open('w') as fd:
+        json.dump(notebook, fd, indent=1)

test/__init__.py

@@ -1,60 +1,13 @@
-from typing import Optional, Dict
-
-import hashlib
 import logging
-import os
 import random
 import refinery
 import string
 import unittest
-import urllib.request
-
-from refinery.units.crypto.cipher.aes import aes
-
-__all__ = ['refinery', 'TestBase', 'NameUnknownException']
 
+from samples import SampleStore
 
-class SampleStore:
-    cache: Dict[str, bytes]
 
-    def __init__(self):
-        self.cache = {}
-
-    def download(self, sha256hash: str, key: Optional[str] = None):
-        def tobytearray(r):
-            if isinstance(r, bytearray):
-                return r
-            return bytearray(r)
-        key = key or 'REFINERYTESTDATA'
-        key = key.encode('latin1')
-        sha256hash = sha256hash.lower()
-        req = urllib.request.Request(
-            F'https://github.com/binref/refinery-test-data/blob/master/{sha256hash}.enc?raw=true')
-        try:
-            with urllib.request.urlopen(req) as response:
-                encoded_sample = tobytearray(response.read())
-        except Exception:
-            api = os.environ['MALSHARE_API']
-            req = urllib.request.Request(
-                F'https://malshare.com/api.php?api_key={api}&action=getfile&hash={sha256hash}')
-            with urllib.request.urlopen(req) as response:
-                result = tobytearray(response.read())
-        else:
-            result = encoded_sample | aes(mode='CBC', key=key) | bytearray
-            if not result or hashlib.sha256(result).hexdigest().lower() != sha256hash:
-                raise ValueError('sample did not decode correctly')
-        self.cache[sha256hash] = result
-        return result
-
-    def get(self, sha256hash: str, key: Optional[str] = None):
-        for cached, value in self.cache.items():
-            if cached.casefold() == sha256hash.casefold():
-                return value
-        else:
-            return self.download(sha256hash, key)
-
-    def __getitem__(self, sha256hash: str):
-        return self.get(sha256hash)
+__all__ = ['refinery', 'TestBase', 'NameUnknownException']
 
 
 class NameUnknownException(Exception):

tutorials/boilerplate.py

@@ -13,17 +13,19 @@
 import hashlib
 import logging
 import re
+import requests
 import shlex
 import getpass
 
-
-os.environ['REFINERY_TERM_SIZE'] = '120'
-os.environ['REFINERY_COLORLESS'] = '1'
+if True:
+    os.environ['REFINERY_TERM_SIZE'] = '120'
+    os.environ['REFINERY_COLORLESS'] = '1'
 
 from refinery.lib.meta import SizeInt
 from refinery.lib.loader import load_pipeline
 from refinery.units import Executable, Unit
-from test import SampleStore
+
+from samples import SampleStore
 
 logging.disable(logging.CRITICAL)
 Executable.Entry = '__DEMO__'
@@ -203,6 +205,18 @@ def show(line: str):
     return Image(filename=line.strip())
 
 
+@register_cell_magic
+def cat(line: str, cell=None):
+    cat, _, out = line.partition('>')
+    cat, _, eof = cat.partition('<<')
+    out = out.strip()
+    eof = eof.strip()
+    cell = cell or ''
+    cell, _, _ = cell.partition(eof)
+    cell = cell.strip()
+    store.cache[out] = cell.encode('utf8')
+
+
 def store_sample(hash: str, name: Optional[str] = None, key: Optional[str] = None):
     store.download(hash, key=key)
     if name is None:
@@ -212,3 +226,14 @@ def store_sample(hash: str, name: Optional[str] = None, key: Optional[str] = Non
 
 def store_clear():
     store.cache.clear()
+
+
+@register_line_magic
+def flare(line: str):
+    name, _, pattern = line.strip().partition(' ')
+    url = F'https://www.awarenetwork.org/home/outlaw/ctfs/flareon/{name}'
+    store_clear()
+    store.cache[name] = requests.get(url).content
+    emit(F'{name} | xt7z {pattern} [| dump {{path}} ]')
+    rm(name)
+    ls()