Merge pull request #488 from chrismbirmingham/scratch-update

[Semio] Scratch Update/Fixes

.dockerignore

@@ -13,5 +13,7 @@ dependencies/emsdk/.emscripten.old
 dependencies/cpython/.git
 dependencies/cpython/builddir
 dependencies/libwallaby/.git
+dependencies/kipr-scratch/libwallaby-build
+dependencies/kipr-scratch/kipr-scratch
 
 **/.github

.github/workflows/ci.yml

@@ -14,6 +14,12 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: simulator
+        submodules: recursive
+    - name: Install Simulator system dependencies
+      run: sudo apt-get update && sudo apt-get install -y wget git cmake build-essential swig zlib1g-dev doxygen default-jre python2.7
+    - name: Install Simulator dependencies
+      run: yarn run build-deps
+      working-directory: simulator
     - name: Install Simulator modules
       # Specifying an alternative cache folder to work around a race condition issue in yarn,
       # which seems to occur b/c of how ivygate is referenced from GitHub.

.gitignore

@@ -2,6 +2,7 @@
 node_modules
 yarn-error.log
 dist
+.yarncache
 
 # Prerequisites
 *.d

.gitmodules

@@ -7,4 +7,6 @@
 [submodule "dependencies/emsdk"]
 	path = dependencies/emsdk
 	url = https://github.com/emscripten-core/emsdk
-
+[submodule "dependencies/kipr-scratch"]
+	path = dependencies/kipr-scratch
+	url = https://github.com/kipr/kipr-scratch

Dockerfile

@@ -4,7 +4,7 @@ RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 ENV TZ=America/Los_Angeles
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-RUN apt-get update && apt-get install -y wget git cmake build-essential python3.8 swig zlib1g-dev doxygen
+RUN apt-get update && apt-get install -y wget git cmake build-essential python3.8 swig zlib1g-dev doxygen default-jre python2.7
 
 RUN wget https://deb.nodesource.com/setup_18.x && chmod +x ./setup_18.x && ./setup_18.x
 RUN apt-get install -y nodejs
@@ -13,11 +13,12 @@ RUN npm install -g yarn
 
 ADD . /app
 
+EXPOSE 3000
 WORKDIR /app/
-RUN python3.8 dependencies/build.py
+RUN yarn run build-deps
+# RUN python3.8 dependencies/build.py
 
-WORKDIR /app
-EXPOSE 3000
+# WORKDIR /app
 # WORKDIR /app/simulator
 # RUN yarn install --cache-folder ./.yarncache && yarn build; true
 RUN yarn install --cache-folder ./.yarncache

README.md

@@ -16,6 +16,7 @@ Simulates a botball/JBC style demobot with a built in IDE.
 - [CMake](https://cmake.org/)
 - [SWIG 4+](https://swig.org/)
 - [Python 3.7 or newer](https://www.python.org/)
+- [Also python 2.7]
 
 ### Debian/Ubuntu
 ```bash
@@ -59,6 +60,7 @@ Or, if you've already cloned the repository without `--recurse-submodules`, you
 
 ```bash
 git submodule update --init
+cd dependencies/kipr-scratch/libwallaby && git submodule update --init
 ```
 
 ### Optional: Pull large files
@@ -71,11 +73,52 @@ git lfs pull
 
 ```bash
 # Python 3.7+ is required for the build process
-python3 dependencies/build.py
-```
+yarn run build-deps
+# The build-deps command executes:
+"python3 dependencies/build.py && yarn add file:dependencies/kipr-scratch/kipr-scratch --cache-folder ./.yarncache",
 
+```
 Tip: if you are experiencing issues with this step, you may try deleting the repository and follow the steps listed above again.
 
+### Notes on building dependencies
+```python3 dependencies/build.py```
+
+1. Run some setup checks
+2. Build libwallaby (probably only need to do this once)
+3. Delete ''unnecessary blocks' from scratch-blocks/blocks-vertical (renames them with a .old extension)
+    - 'event.js',
+    - 'extensions.js',
+    - 'default_toolbox.js',
+    - 'looks.js',
+    - 'motion.js',
+    - 'sensing.js',
+    - 'sound.js'
+
+
+4. blockify libwallaby
+    - Input is libwallaby-build, output is in scratch-blocks/blocks_vertical
+    - Start by parsing the xml result of the libwallaby build
+      - Essentially getting all the functions from the included files and cdecl nodes
+      - Ends up with a list of modules and their functions
+    - For all functions, needs to generate a blockly block of the module_function with
+      with the correct parameters and return types.
+      - Only does it for analog, digital, wait_for, time, motor, and servo modules
+    - Patches the colors in core/colours.js for the new modules from info in module_hsl.json
+    - Writes a new default_toolbox.js - this is what we have access to pick from
+    - Modifies the category names in the vertical extensions file, removing 'sounds', 'motion', 
+      'looks', 'event', 'sensing', 'pen', but adding our modules
+    - Modifies the messages to include those from our modules and the program start
+    - Modifies the css flyout style
+    - Modifies the field variables to remove non initialization case
+
+
+5. Patch the scratch-blocks package.json to use python2
+
+6. Install the scratch blocks
+
+
+
+
 ## Install JavaScript Dependencies
 
 Navigate to the root directory of this repository, then run:

dependencies/.gitignore

@@ -1,5 +1,5 @@
 libkipr_build_c
 libkipr_build_python
 libkipr_install_c
-ammo_build
+scratch-rt.js
 dependencies.json

dependencies/build.py

@@ -5,6 +5,26 @@
 import pathlib
 import json
 import multiprocessing
+import hashlib
+
+def sha1OfFile(filepath):
+  sha = hashlib.sha1()
+  with open(filepath, 'rb') as f:
+    while True:
+      block = f.read(2**10) # Magic number: one-megabyte blocks.
+      if not block: break
+      sha.update(block)
+    return sha.hexdigest()
+
+def hash_dir(dir_path):
+  hashes = []
+  for path, dirs, files in os.walk(dir_path):
+    for file in sorted(files):
+      hashes.append(sha1OfFile(os.path.join(path, file)))
+    for dir in sorted(dirs):
+      hashes.append(hash_dir(os.path.join(path, dir)))
+    break
+  return str(hash(''.join(hashes)))
 
 def is_tool(name):
     """Check whether `name` is on PATH and marked as executable."""
@@ -68,6 +88,7 @@ def is_tool(name):
 }
 
 libkipr_dir = working_dir / 'libwallaby'
+libkipr_hash = hash_dir(libkipr_dir)
 
 # libkipr (C)
 
@@ -86,6 +107,7 @@ def is_tool(name):
     '-Dwith_python_binding=OFF',
     '-Dwith_documentation=ON',
     '-Dwith_tests=OFF',
+    '-Dwith_graphics=OFF',
     f'-DCMAKE_INSTALL_PREFIX={libkipr_install_c_dir}',
     libkipr_dir
   ],
@@ -200,12 +222,46 @@ def is_tool(name):
 print('Generating JSON documentation...')
 libkipr_c_documentation_json = f'{libkipr_build_c_dir}/documentation/json.json'
 subprocess.run(
-  [ 'python3', 'generate_doxygen_json.py', f'{libkipr_build_c_dir}/documentation/xml', libkipr_c_documentation_json ],
+  [ python, 'generate_doxygen_json.py', f'{libkipr_build_c_dir}/documentation/xml', libkipr_c_documentation_json ],
+  # [ 'python3', 'generate_doxygen_json.py', f'{libkipr_build_c_dir}/documentation/xml', libkipr_c_documentation_json ],
   cwd = working_dir,
   check = True
 )
 
+print('Building kipr-scratch...')
+kipr_scratch_path = working_dir / 'kipr-scratch'
+subprocess.run(
+  [ python, kipr_scratch_path / 'build.py' ],
+  cwd = kipr_scratch_path,
+  check = True
+)
 
+print('Packaging kipr-scratch...')
+subprocess.run(
+  [ python, kipr_scratch_path / 'package.py' ],
+  cwd = kipr_scratch_path,
+  check = True
+)
+
+print('Generating scratch runtime...')
+scratch_runtime_path = working_dir / 'scratch-rt'
+# emcc -s WASM=0 -s INVOKE_RUN=0 -s ASYNCIFY -s EXIT_RUNTIME=1 -s "EXPORTED_FUNCTIONS=['_main', '_simMainWrapper']" -I${config.server.dependencies.libkipr_c}/include -Wl,--whole-archive -L${config.server.dependencies.libkipr_c}/lib -lkipr -o ${path}.js ${path}
+subprocess.run([
+    'emcc',
+    '-sWASM=0',
+    '-sINVOKE_RUN=0',
+    '-sEXIT_RUNTIME=0',
+    '-sERROR_ON_UNDEFINED_SYMBOLS=0',
+    '-sLINKABLE=1',
+    '-sEXPORT_ALL=1',
+    f'-L{libkipr_install_c_dir}/lib',
+    '-Wl,--whole-archive', '-lkipr', '-Wl,--no-whole-archive',
+    f'-o', f'{scratch_runtime_path}.js',
+    f'{scratch_runtime_path}.c'
+  ],
+  env = env,
+  check = True
+)
 
 print('Outputting results...')
 output = json.dumps({
@@ -216,10 +272,13 @@ def is_tool(name):
     'EMSDK': f'{emsdk_dir}',
     'EM_CONFIG': f'{emsdk_dot_emscripten}'
   },
+  'libkipr_hash': libkipr_hash,
   'libkipr_c': f'{libkipr_install_c_dir}',
   'libkipr_python': f'{libkipr_build_python_dir}',
   'cpython': f'{cpython_emscripten_build_dir}',
+  'cpython_hash': hash_dir(cpython_dir),
   "libkipr_c_documentation": libkipr_c_documentation_json,
+  'scratch_rt': f'{scratch_runtime_path}.js',
 })
 
 with open(working_dir / 'dependencies.json', 'w') as f:

dependencies/scratch-rt.c

@@ -0,0 +1 @@
+int main() { return 0; }

express.js

@@ -11,6 +11,7 @@ const sourceDir = 'dist';
 const { get: getConfig } = require('./config');
 const { WebhookClient } = require('discord.js');
 const proxy = require('express-http-proxy');
+const path = require('path');
 
 
 let config;
@@ -214,6 +215,22 @@ app.use('/static', express.static(`${__dirname}/static`, {
   maxAge: config.caching.staticMaxAge,
 }));
 
+
+if (config.server.dependencies.scratch_rt) {
+  console.log('Scratch Runtime is enabled.');
+  app.use('/scratch/rt.js', express.static(`${config.server.dependencies.scratch_rt}`, {
+    maxAge: config.caching.staticMaxAge,
+  }));
+}
+
+app.use('/scratch', express.static(path.resolve(__dirname, 'node_modules', 'kipr-scratch'), {
+  maxAge: config.caching.staticMaxAge,
+}));
+
+app.use('/media', express.static(path.resolve(__dirname, 'node_modules', 'kipr-scratch', 'media'), {
+  maxAge: config.caching.staticMaxAge,
+}));
+
 // Expose cpython artifacts
 if (config.server.dependencies.cpython) {
   console.log('CPython artifacts are enabled.');

package.json

@@ -7,6 +7,7 @@
   "license": "GPL-3.0-only",
   "scripts": {
     "build": "yarn run clean-dist && webpack --config=configs/webpack/prod.js",
+    "build-deps": "python3 dependencies/build.py && yarn add file:dependencies/kipr-scratch/kipr-scratch --cache-folder ./.yarncache",
     "watch": "node --max-old-space-size=8192 node_modules/webpack/bin/webpack.js --watch --config=configs/webpack/dev.js",
     "clean-dist": "rm -f -r -d dist",
     "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
@@ -26,14 +27,15 @@
     "@types/history": "^4.7.2",
     "@types/jest": "^29.4.0",
     "@types/qs": "^6.9.7",
-    "@types/react-dom": "^16.9.5",
+    "@types/react-dom": "17.0.1",
     "@types/react-redux": "^7.1.18",
     "@types/react-router": "^5.0.0",
     "@types/redux": "^3.6.0",
     "@types/sprintf-js": "^1.1.2",
     "@types/styletron-engine-atomic": "^1.1.1",
     "@types/styletron-react": "^5.0.2",
     "@types/uuid": "^8.3.1",
+    "@types/xmldom": "^0.1.31",
     "@typescript-eslint/eslint-plugin": "^4.18.0",
     "@typescript-eslint/parser": "^4.18.0",
     "babel-loader": "^8.1.0",
@@ -62,7 +64,6 @@
     "@fortawesome/free-brands-svg-icons": "^6.2.0",
     "@fortawesome/free-solid-svg-icons": "^6.2.0",
     "@fortawesome/react-fontawesome": "^0.2.0",
-    "@types/react": "^16.9.25",
     "babylonjs-gltf2interface": "6.18.0",
     "body-parser": "^1.19.0",
     "colorjs.io": "^0.4.2",
@@ -79,7 +80,9 @@
     "image-loader": "^0.0.1",
     "image-webpack-loader": "^6.0.0",
     "immer": "^9.0.15",
-    "ivygate": "https://github.com/kipr/ivygate#v0.1.5",
+    "itch": "https://github.com/chrismbirmingham/itch#36",
+    "ivygate": "https://github.com/kipr/ivygate#v0.1.8",
+    "kipr-scratch": "file:dependencies/kipr-scratch/kipr-scratch",
     "morgan": "^1.10.0",
     "prop-types": "^15.8.1",
     "qs": "^6.11.0",
@@ -99,6 +102,7 @@
     "styletron-react": "^6.0.1",
     "symmetry": "^0.6.1",
     "url-loader": "^4.1.1",
-    "uuid": "^8.3.2"
+    "uuid": "^8.3.2",
+    "xmldom": "^0.6.0"
   }
 }

src/components/Challenge/ChallengeMenu.tsx

@@ -56,7 +56,7 @@ export interface MenuProps extends StyleProps, ThemeProps {
   onShowAll: () => void;
   onHideAll: () => void;
 
-  onRunClick: () => void;
+  onRunClick?: () => void;
   onStopClick: () => void;
   onResetChallengeClick: () => void;
 
@@ -247,7 +247,7 @@ class ChallengeMenu extends React.PureComponent<Props, State> {
         <RunItem
           theme={theme}
           onClick={SimulatorState.isStopped(simulatorState) ? onRunClick : undefined}
-          disabled={!SimulatorState.isStopped(simulatorState)}
+          disabled={!onRunClick || !SimulatorState.isStopped(simulatorState)}
           style={{ borderLeft: `1px solid ${theme.borderColor}` }}
         >
           <ItemIcon icon={faPlay} />

src/components/Challenge/SimMenu.tsx

@@ -75,7 +75,7 @@ export interface MenuPublicProps extends StyleProps, ThemeProps {
   onShowAll: () => void;
   onHideAll: () => void;
 
-  onRunClick: () => void;
+  onRunClick?: () => void;
   onStopClick: () => void;
   onResetWorldClick: () => void;
 
@@ -315,7 +315,7 @@ class SimMenu extends React.PureComponent<Props, State> {
         <RunItem
           theme={theme}
           onClick={SimulatorState.isStopped(simulatorState) ? onRunClick : undefined}
-          disabled={!SimulatorState.isStopped(simulatorState)}
+          disabled={!onRunClick || !SimulatorState.isStopped(simulatorState)}
           style={{ borderLeft: `1px solid ${theme.borderColor}` }}
         >
           <ItemIcon icon={faPlay} />