From 3f34519d02e333bf156ed2cf469d48161b95692f Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Sun, 18 Aug 2024 10:30:01 -0600 Subject: [PATCH 01/18] Initial try at converting from cargo-screeps to node/rollup/babel --- .example-screeps.yaml | 50 ++++++++++++++ .gitignore | 11 +++- Cargo.toml | 8 +-- README.md | 31 +++++++-- example-screeps.toml | 44 ------------- javascript/main.js | 69 -------------------- js_src/main.js | 89 +++++++++++++++++++++++++ js_tools/deploy.js | 147 ++++++++++++++++++++++++++++++++++++++++++ package.json | 38 +++++++++++ src/logging.rs | 3 +- 10 files changed, 361 insertions(+), 129 deletions(-) create mode 100644 .example-screeps.yaml delete mode 100644 example-screeps.toml delete mode 100644 javascript/main.js create mode 100644 js_src/main.js create mode 100644 js_tools/deploy.js create mode 100644 package.json diff --git a/.example-screeps.yaml b/.example-screeps.yaml new file mode 100644 index 0000000..0efabba --- /dev/null +++ b/.example-screeps.yaml @@ -0,0 +1,50 @@ +servers: + # Deploy to the main MMO server - note that tokens are + # the only supported auth method for official servers (mmo, season, and ptr) + mmo: + host: screeps.com + secure: true + token: your-auth-token-here + branch: default + # The public test realm can be a good place to test your code + ptr: + host: screeps.com + secure: true + token: your-auth-token-here + path: /ptr + branch: default + # Seasonal server configuration - this environment has unique mechanics each + # season, so it might make sense to have feature flag(s) for different mechanics + season: + host: screeps.com + secure: true + token: your-auth-token-here + path: /season + branch: default + private-server: + host: 127.0.0.1 + port: 21025 + secure: false + username: user + password: password + branch: default +configs: + # Whether to minify generated javascript for each configured server + terser: + # The special '*'' key sets a default for all servers which + # will be **overridden** by an applicable per-server config + '*': false + ptr: false + localhost: false + # Additional options to pass to wasm-pack to customize the build for each server + wasm-pack-options: + # The special '*'' key sets flags applied to all servers, which + # will be **concatenated** with any applicable per-server config + '*': [] + # This setting enables the `mmo` crate feature for these destinations, + # which enables the API functions for intershard communication and pixel + # generation, which are specific to MMO + mmo: ["--features", "mmo"] + ptr: ["--features", "mmo"] + # Other servers can each have their own build flags, including crate features: + #season: ["--features", "my-season-7-feature"] diff --git a/.gitignore b/.gitignore index 387ab4e..a6e8f18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ # Compiled source # ################### target +dist +pkg + +# Dependencies # +################ +node_modules # Packages # ############ @@ -33,4 +39,7 @@ nbproject *.iws *.sublime-project *.sublime-workspace -screeps.toml + +# Config w/ secrets # +##################### +.screeps.yaml diff --git a/Cargo.toml b/Cargo.toml index a74c763..51ecd75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,17 +27,13 @@ opt-level = 3 lto = true [package.metadata.wasm-pack.profile.release] -# Replace the following to enable wasm-opt optimization -# wasm-pack will try to install wasm-opt automatically, but it must be installed by hand on some -# operating systems. -wasm-opt = false # See wasm-opt for full available options; handy examples: # -O4 - optimize aggressively for performance # -Oz - optimize aggressively for code size # -g - leave debug info in place, allowing for more descriptive stack traces on panic -# --disable-sign-ext - prevents opcodes that the screeps servers can't load (see +# --signext-lowering - removes opcodes that the screeps servers can't load (see # https://github.com/rustyscreeps/screeps-game-api/issues/391) -#wasm-opt = ["-O4", "--disable-sign-ext"] +wasm-opt = ["-O4", "--signext-lowering"] [features] default = [] diff --git a/README.md b/README.md index fe2f62a..c9c5296 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,20 @@ apis are broken). Quickstart: ```sh -# Install CLI dependency: -cargo install cargo-screeps +# Install rustup: https://rustup.rs/ + +# Install wasm-pack +cargo install wasm-pack + +# Install wasm-opt +cargo install wasm-opt + +# Install nvm: https://github.com/nvm-sh/nvm +# (Windows: https://github.com/coreybutler/nvm-windows) + +# Install node at version 20 (16 to 21 tested ok, IIRC) +nvm install 20 +nvm use 20 # Clone the starter git clone https://github.com/rustyscreeps/screeps-starter-rust.git @@ -26,15 +38,20 @@ cd screeps-starter-rust # note: if you customize the name of the crate, you'll need to update the MODULE_NAME # variable in the javascript/main.js file with the updated name +# Install node deps +npm install + # Copy the example config, and set up at least one deployment mode. # Configure credentials if you'd like to upload directly, or a directory to copy to # if you'd prefer to use the game client to deploy: -cp example-screeps.toml screeps.toml -nano screeps.toml +cp .example-screeps.yaml .screeps.yaml +nano .screeps.yaml + +# compile for a configured server but don't upload +npm run deploy -- --server ptr --dryrun -# Compile plus deploy to the configured 'upload' mode; any section name you -# set up in your screeps.toml for different environments and servers can be used -cargo screeps deploy -m upload +# compile and deploy to a configured server +npm run deploy -- --server mmo ``` [screeps]: https://screeps.com/ diff --git a/example-screeps.toml b/example-screeps.toml deleted file mode 100644 index 9a91c38..0000000 --- a/example-screeps.toml +++ /dev/null @@ -1,44 +0,0 @@ -default_deploy_mode = "upload" - -[build] -# options to allow building code against rust versions >=1.70 without opcodes -# incompatible with screeps server environments; requires nightly rust. See -# https://github.com/rustyscreeps/screeps-game-api/issues/391 -extra_options = ["--config", "build.rustflags=['-Ctarget-cpu=mvp']", "-Z", "build-std=std,panic_abort"] - -[upload] -auth_token = "your screeps.com auth token" -[upload.build] - # set the MMO feature to allow for MMO-only API endpoints related to multiple shards and - # intershard resources - extra_options = ["--features=mmo"] - -[ptr] -auth_token = "your screeps.com auth token" -prefix = "ptr" -[ptr.build] - extra_options = ["--features=mmo"] - -[season] -auth_token = "your screeps.com auth token" -prefix = "season" - [season.build] - extra_options = ["--features=my-crate-season-1-feature"] - -[copy] -destination = "path to your local code directory from your game client, without branch directory" -branch = "default" -[copy.build] - # set the sim feature to allow for use with the coordinates in the simulation room - extra_options = ["--features=sim"] - -[pserver] -hostname = "192.0.2.1" -port = 21025 -username = "username-or-email" -password = "your-password" -ssl = false -branch = "default" - -# for full syntax and available options, see -# https://github.com/rustyscreeps/cargo-screeps/blob/master/screeps-defaults.toml diff --git a/javascript/main.js b/javascript/main.js deleted file mode 100644 index 532a6ec..0000000 --- a/javascript/main.js +++ /dev/null @@ -1,69 +0,0 @@ -"use strict"; -let wasm_module; - -// replace this with the name of your module -const MODULE_NAME = "screeps-starter-rust"; - -// This provides the function `console.error` that wasm_bindgen sometimes expects to exist, -// especially with type checks in debug mode. An alternative is to have this be `function () {}` -// and let the exception handler log the thrown JS exceptions, but there is some additional -// information that wasm_bindgen only passes here. -// -// There is nothing special about this function and it may also be used by any JS/Rust code as a convenience. -function console_error() { - const processedArgs = _.map(arguments, (arg) => { - if (arg instanceof Error) { - // On this version of Node, the `stack` property of errors contains - // the message as well. - return arg.stack; - } else { - return arg; - } - }).join(' '); - console.log("ERROR:", processedArgs); - Game.notify(processedArgs); -} - -let halt_next_tick = false; - -module.exports.loop = function () { - // need to freshly override the fake console object each tick - console.error = console_error; - if (halt_next_tick === true) { - // we've had an error on the last tick (see error catch); skip execution during the current - // tick, asking to have our IVM immediately destroyed so we get a fresh environment next tick - // to work around https://github.com/rustwasm/wasm-bindgen/issues/3130 - Game.cpu.halt(); - } else { - try { - if (wasm_module) { - wasm_module.loop(); - } else { - // attempt to load the wasm only if there's enough bucket to do a bunch of work this tick - if (Game.cpu.bucket < 750) { - console.log("we are running out of time, pausing compile!" + JSON.stringify(Game.cpu)); - return; - } - // load the wasm module - wasm_module = require(MODULE_NAME); - // load the wasm instance! - wasm_module.initialize_instance(); - // go ahead and run the loop for its first tick - wasm_module.loop(); - } - } catch (error) { - console.error("caught exception:", error); - // we've already logged the more-descriptive stack trace from rust's panic_hook - // if for some reason (like wasm init problems) you're not getting output from that - // and need more information, uncomment the following: - // if (error.stack) { - // console.error("stack trace:", error.stack); - // } - console.error("resetting VM next tick."); - // if we call `Game.cpu.halt();` this tick, console output from the tick (including the - // stack trace) is not shown due to those contents being copied post-tick (and the halt - // function destroying the environment immediately). This delays the halt() until next tick. - halt_next_tick = true; - } - } -} diff --git a/js_src/main.js b/js_src/main.js new file mode 100644 index 0000000..bb05820 --- /dev/null +++ b/js_src/main.js @@ -0,0 +1,89 @@ +"use strict"; +import 'fastestsmallesttextencoderdecoder-encodeinto/EncoderDecoderTogether.min.js'; + +import * as bot from '../pkg/screeps_starter_rust.js'; +// replace this with the name of your module +const MODULE_NAME = "screeps_starter_rust"; + +// This provides the function `console.error` that wasm_bindgen sometimes expects to exist, +// especially with type checks in debug mode. An alternative is to have this be `function () {}` +// and let the exception handler log the thrown JS exceptions, but there is some additional +// information that wasm_bindgen only passes here. +// +// There is nothing special about this function and it may also be used by any JS/Rust code as a convenience. +function console_error() { + const processedArgs = _.map(arguments, (arg) => { + if (arg instanceof Error) { + // On this version of Node, the `stack` property of errors contains + // the message as well. + return arg.stack; + } else { + return arg; + } + }).join(' '); + console.log("ERROR:", processedArgs); + Game.notify(processedArgs); +} + +let halt_next_tick = false; + +const BUCKET_BOOT_THRESHOLD = 1500; + +// track whether running wasm loop for each tick completes, to detect errors or aborted execution +let running = false; + +function loaded_loop() { + // need to freshly override the fake console object each tick + console.error = console_error; + if (running) { + // we've had an error on the last tick; skip execution during the current tick, asking to + // have our IVM immediately destroyed so we get a fresh environment next tick; + // workaround for https://github.com/rustwasm/wasm-bindgen/issues/3130 + Game.cpu.halt(); + } else { + try { + running = true; + bot.loop(); + // if execution doesn't get to this point for any reason (error or out-of-CPU + // cancellation), setting to false won't happen which will cause a halt() next tick + running = false; + } catch (error) { + console.log(`caught exception, will halt next tick: ${error}`); + // not logging stack since we've already logged the stack trace from rust via the panic + // hook and that one is generally better, but if we need it, uncomment: + + // if (error.stack) { + // console.log("js stack:", error.stack); + // } + } + } +} + +// cache for each step of the wasm module's initialization +let wasm_bytes, wasm_module, wasm_instance; + +module.exports.loop = function() { + // need to freshly override the fake console object each tick + console.error = console_error; + // temporarily need to polyfill this too because there's a bug causing the warn + // in initSync to fire in bindgen 0.2.93 + console.warn = console.log; + + // attempt to load the wasm only if there's lots of bucket + if (Game.cpu.bucket < BUCKET_BOOT_THRESHOLD) { + console.log(`startup deferred; ${Game.cpu.bucket} / ${BUCKET_BOOT_THRESHOLD} required bucket`); + return; + } + + // run each step of the load process, saving each result so that this can happen over multiple ticks + if (!wasm_bytes) wasm_bytes = require(MODULE_NAME); + if (!wasm_module) wasm_module = new WebAssembly.Module(wasm_bytes); + if (!wasm_instance) wasm_instance = bot.initSync(wasm_module); + + // remove the bytes from the heap and require cache, we don't need 'em anymore + wasm_bytes = null; + delete require.cache[MODULE_NAME]; + // replace this function with the post-load loop for next tick + module.exports.loop = loaded_loop; + console.log(`loading complete, CPU used: ${Game.cpu.getUsed()}`) +} diff --git a/js_tools/deploy.js b/js_tools/deploy.js new file mode 100644 index 0000000..e884a6f --- /dev/null +++ b/js_tools/deploy.js @@ -0,0 +1,147 @@ +const fs = require('fs'); +const path = require('path'); +const spawnSync = require('child_process').spawnSync; + +const { rollup } = require('rollup'); +const babel = require('@rollup/plugin-babel'); +const commonjs = require('@rollup/plugin-commonjs'); +const copy = require('rollup-plugin-copy'); +const { nodeResolve } = require('@rollup/plugin-node-resolve'); +const terser = require('@rollup/plugin-terser'); + +const { ScreepsAPI } = require('screeps-api'); +const yaml = require('yamljs'); +const argv = require('yargs') + .option('server', { + describe: 'server to connect to; must be defined in .screeps.yaml servers section', + }) + .demandOption('server') + .option('dryrun', { + describe: 'execute a dry run, skipping the upload of the generated code', + type: 'boolean', + default: false, + }) + .argv; + +function load_config() { + const yaml_conf = yaml.parse(fs.readFileSync('.screeps.yaml', { encoding: 'utf8' })); + + const branch = yaml_conf.servers[argv.server].branch || 'default'; + let use_terser = false; + let extra_options = []; + + const configs = yaml_conf.configs || {}; + + const terser_configs = configs.terser || {}; + if (terser_configs['*'] !== undefined) { + use_terser = terser_configs['*']; + } + if (terser_configs[argv.server] !== undefined) { + use_terser = terser_configs[argv.server]; + } + + const wasm_pack_options = configs['wasm-pack-options'] || {}; + if (wasm_pack_options['*']) { + extra_options = extra_options.concat(wasm_pack_options['*']) + } + if (wasm_pack_options[argv.server]) { + extra_options = extra_options.concat(wasm_pack_options[argv.server]) + } + + return { + branch: branch, + use_terser: use_terser, + extra_options: extra_options, + } +} + +function output_clean() { + for (dir of ['dist', 'pkg']) { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true }); + } + } +} + +function run_wasm_pack(extra_options) { + let args = ['run', 'nightly', 'wasm-pack', 'build', '--target', 'web', '--release', '.', ...extra_options]; + spawnSync('rustup', args, { stdio: 'inherit' }) +} + +async function run_rollup(use_terser) { + const bundle = await rollup({ + input: 'js_src/main.js', + plugins: [ + commonjs(), + nodeResolve(), + babel({ + babelHelpers: 'bundled', + presets: ['@babel/preset-env'], + targets: { + "node": 12, + } + }), + copy({ + // todo, should figure out a better way to get the name + targets: [{ src: 'pkg/screeps_starter_rust_bg.wasm', dest: 'dist', rename: 'screeps_starter_rust.wasm' }] + }), + ] + }); + await bundle.write({ + format: 'cjs', + file: 'dist/main.js', + plugins: [use_terser && terser()], + }); +} + +async function upload(server, branch, dryrun) { + let modules = {}; + let used_bytes = 0; + + await fs.readdirSync('dist').map(function (filename) { + if (filename.endsWith('.wasm')) { + const data = fs.readFileSync(path.join('dist', filename), {encoding: 'base64'}); + const filename_stripped = filename.replace(/\.wasm$/, ''); + used_bytes += data.length; + modules[filename_stripped] = { + binary: data, + } + } else { + const data = fs.readFileSync(path.join('dist', filename), {encoding: 'utf8'}); + const filename_stripped = filename.replace(/\.js$/, ''); + used_bytes += data.length; + modules[filename_stripped] = data; + } + }); + + const used_mib = used_bytes / (1024 * 1024); + const used_percent = 100 * used_mib / 5; + + const usage_string = `${used_mib.toFixed(2)} MiB of 5.0 MiB code size limit (${used_percent.toFixed(2)}%)` + if (dryrun) { + console.log(`Not uploading due to --dryrun; would use ${usage_string}`); + } else { + console.log(`Uploading; using ${usage_string}`); + const api = await ScreepsAPI.fromConfig(server); + const response = await api.code.set(branch, modules); + console.log(JSON.stringify(response)); + } +} + +async function run() { + const config = load_config(); + + // clean output (pkg/dist dirs) + output_clean(); + + // run cargo build + run_wasm_pack(config.extra_options); + + // run rollup + await run_rollup(config.use_terser); + + // read resulting files and upload + await upload(argv.server, config.branch, argv.dryrun); +} + +run().catch(console.error) diff --git a/package.json b/package.json new file mode 100644 index 0000000..327d80c --- /dev/null +++ b/package.json @@ -0,0 +1,38 @@ +{ + "name": "screeps-starter-rust", + "version": "0.1.0", + "description": "", + "files": [ + "dist/" + ], + "main": "dist/main.js", + "scripts": { + "deploy": "node js_tools/deploy.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/rustyscreeps/screeps-starter-rust.git" + }, + "author": "Shane Madden", + "license": "MIT", + "bugs": { + "url": "https://github.com/rustyscreeps/screeps-starter-rust/issues" + }, + "homepage": "https://github.com/rustyscreeps/screeps-starter-rust", + "screeps_bot": true, + "dependencies": { + "fastestsmallesttextencoderdecoder-encodeinto": "^1.0.22" + }, + "devDependencies": { + "@babel/preset-env": "^7.25.3", + "@rollup/plugin-babel": "^6.0.4", + "@rollup/plugin-commonjs": "^26.0.1", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-terser": "^0.4.4", + "rollup": "^4.21.0", + "rollup-plugin-copy": "^3.5.0", + "screeps-api": "^1.16.0", + "yamljs": "^0.3.0", + "yargs": "^17.7.2" + } +} diff --git a/src/logging.rs b/src/logging.rs index bc0916e..84114ea 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -1,4 +1,3 @@ -use core::panic::PanicInfo; use std::{fmt::Write, panic}; use js_sys::JsString; @@ -71,7 +70,7 @@ extern "C" { fn stack_trace_limit(size: f32); } -fn panic_hook(info: &PanicInfo) { +fn panic_hook(info: &panic::PanicHookInfo) { // import JS Error API to get backtrace info (backtraces don't work in wasm) // Node 8 does support this API: https://nodejs.org/docs/latest-v8.x/api/errors.html#errors_error_stack From 24a7af0a82467f96fbf04d2e62759a3f9313ff5d Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 19 Aug 2024 09:51:56 -0600 Subject: [PATCH 02/18] first swing at migration howto, plus some other readme updates --- README.md | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c9c5296..24fe8e8 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,16 @@ Starter Rust AI for [Screeps: World][screeps], the JavaScript-based MMO game. This uses the [`screeps-game-api`] bindings from the [rustyscreeps] organization. -While it's possible to compile using [`wasm-pack`] directly using the Node.js target, -some modifications are needed to load the output within the Screep environment, so it's -recommended to use [`cargo-screeps`] for building and deploying your code. +[`wasm-pack`] is used for building the Rust code to WebAssembly. This example uses [Rollup] to +bundle the resulting javascript, [Babel] to transpile generated code for compatibility with older +Node.js versions running on the Screeps servers, and the [`screeps-api`] Node.js package to deploy. -The documentation is currently a bit sparse. API docs which list functions one -can use are located at https://docs.rs/screeps-game-api/. +Documentation for the Rust version of the game APIs is at https://docs.rs/screeps-game-api/. Almost all crates on https://crates.io/ are usable (only things which interact with OS apis are broken). -Quickstart: +## Quickstart: ```sh # Install rustup: https://rustup.rs/ @@ -54,8 +53,33 @@ npm run deploy -- --server ptr --dryrun npm run deploy -- --server mmo ``` +## Migration to 0.22 + +Versions of [`screeps-game-api`] at 0.22 or higher are no longer compatible with the +[`cargo-screeps`] tool for building and deployment; the transpilation step being handled +by Babel is required to generate code that the game servers can load. + +To migrate an existing bot to using the new Javascript translation layer and deploy script: + +- Create a `.screeps.yaml` with the relevant settings from your `screeps.toml` file applied to the + new `.example-screeps.yaml` example file in this repo. Add it to your `.gitignore`, as well as + entries for the `node_modules` directory and the `dist` directory. +- Create a `package.json` copied from the one in this repo and make appropriate customizations. +- Install the node dependencies from the quickstart steps above, then run `npm install` from within + the directory to install the required packages. +- Copy the deploy script over to a `js_tools` directory, fix name in the file + (todo would be nice to not have hardcoded). +- Add `main.js` to a `js_src` directory, either moved from your exist and updated, or freshly + copied. If updating, you'll need to change: + - Import formatting, particularly for the wasm module. + - wasm module initialization has changed, requiring two calls to first compile the module, + then to initialize the instance of the module. +- Run `npm run deploy -- --server ptr --dryrun` to compile for PTR, remove the `--dryrun` to deploy + [screeps]: https://screeps.com/ [`wasm-pack`]: https://rustwasm.github.io/wasm-pack/ -[`cargo-screeps`]: https://github.com/rustyscreeps/cargo-screeps/ +[Rollup]: https://rollupjs.org/ +[Babel]: https://babeljs.io/ [`screeps-game-api`]: https://github.com/rustyscreeps/screeps-game-api/ +[`cargo-screeps`]: https://github.com/rustyscreeps/cargo-screeps/ [rustyscreeps]: https://github.com/rustyscreeps/ From c4638bf346c10cb76516ca8d4c87a9c3311b391b Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 19 Aug 2024 09:57:29 -0600 Subject: [PATCH 03/18] readme tweaks --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 24fe8e8..e4c20bf 100644 --- a/README.md +++ b/README.md @@ -62,15 +62,15 @@ by Babel is required to generate code that the game servers can load. To migrate an existing bot to using the new Javascript translation layer and deploy script: - Create a `.screeps.yaml` with the relevant settings from your `screeps.toml` file applied to the - new `.example-screeps.yaml` example file in this repo. Add it to your `.gitignore`, as well as - entries for the `node_modules` directory and the `dist` directory. + new `.example-screeps.yaml` example file in this repo. +- Add to your `.gitignore`: `.screeps.yaml`, `node_modules`, and `dist` - Create a `package.json` copied from the one in this repo and make appropriate customizations. - Install the node dependencies from the quickstart steps above, then run `npm install` from within - the directory to install the required packages. -- Copy the deploy script over to a `js_tools` directory, fix name in the file + the bot directory to install the required packages. +- Copy the `deploy.js` script over to a `js_tools` directory, fix name in the file (todo would be nice to not have hardcoded). -- Add `main.js` to a `js_src` directory, either moved from your exist and updated, or freshly - copied. If updating, you'll need to change: +- Add `main.js` to a `js_src` directory, either moved from your existing `javascript` dir and updated, + or freshly copied. If updating, you'll need to change: - Import formatting, particularly for the wasm module. - wasm module initialization has changed, requiring two calls to first compile the module, then to initialize the instance of the module. From 6480d868f8d0232ac69977a376d01a35e3372229 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 19 Aug 2024 10:08:27 -0600 Subject: [PATCH 04/18] Fix link --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e4c20bf..13f98ff 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ npm run deploy -- --server mmo Versions of [`screeps-game-api`] at 0.22 or higher are no longer compatible with the [`cargo-screeps`] tool for building and deployment; the transpilation step being handled -by Babel is required to generate code that the game servers can load. +by [Babel] is required to generate code that the game servers can load. To migrate an existing bot to using the new Javascript translation layer and deploy script: @@ -80,6 +80,7 @@ To migrate an existing bot to using the new Javascript translation layer and dep [`wasm-pack`]: https://rustwasm.github.io/wasm-pack/ [Rollup]: https://rollupjs.org/ [Babel]: https://babeljs.io/ +[`screeps-api`]: https://github.com/screepers/node-screeps-api [`screeps-game-api`]: https://github.com/rustyscreeps/screeps-game-api/ [`cargo-screeps`]: https://github.com/rustyscreeps/cargo-screeps/ [rustyscreeps]: https://github.com/rustyscreeps/ From b32c1461057ec20aff6947bf7bc6e57c4eb0c3e2 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 19 Aug 2024 14:37:50 -0600 Subject: [PATCH 05/18] deploy script improvements from private bot --- js_tools/deploy.js | 78 +++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/js_tools/deploy.js b/js_tools/deploy.js index e884a6f..3307b85 100644 --- a/js_tools/deploy.js +++ b/js_tools/deploy.js @@ -1,6 +1,7 @@ const fs = require('fs'); +const fsExtra = require('fs-extra'); const path = require('path'); -const spawnSync = require('child_process').spawnSync; +const { spawnSync } = require('child_process'); const { rollup } = require('rollup'); const babel = require('@rollup/plugin-babel'); @@ -23,15 +24,24 @@ const argv = require('yargs') }) .argv; +// load configuration from .screeps.yaml +// unified config format: +// https://github.com/screepers/screepers-standards/blob/master/SS3-Unified_Credentials_File.md function load_config() { const yaml_conf = yaml.parse(fs.readFileSync('.screeps.yaml', { encoding: 'utf8' })); + const configs = yaml_conf.configs || {}; - const branch = yaml_conf.servers[argv.server].branch || 'default'; - let use_terser = false; - let extra_options = []; + if (!yaml_conf.servers[argv.server]) { + console.log(`no configuration found for server ${argv.server} in .screeps.yaml`); + return + } - const configs = yaml_conf.configs || {}; + const branch = yaml_conf.servers[argv.server].branch || 'default'; + // whether the terser minification step should be called during rollup + // read the default from the '*' key first (if it exists) then override + // with server config (if it exists) + let use_terser = false; const terser_configs = configs.terser || {}; if (terser_configs['*'] !== undefined) { use_terser = terser_configs['*']; @@ -40,6 +50,9 @@ function load_config() { use_terser = terser_configs[argv.server]; } + // extra options to pass to wasm-pack - append the options from the '*' + // key then any server-specific options + let extra_options = []; const wasm_pack_options = configs['wasm-pack-options'] || {}; if (wasm_pack_options['*']) { extra_options = extra_options.concat(wasm_pack_options['*']) @@ -48,26 +61,23 @@ function load_config() { extra_options = extra_options.concat(wasm_pack_options[argv.server]) } - return { - branch: branch, - use_terser: use_terser, - extra_options: extra_options, - } + return { branch, use_terser, extra_options } } -function output_clean() { +// clear the dist and pkg directories of any existing build results +async function output_clean() { for (dir of ['dist', 'pkg']) { - if (fs.existsSync(dir)) { - fs.rmSync(dir, { recursive: true }); - } + await fsExtra.emptyDir(dir); } } +// invoke wasm-pack, compiling the wasm module into the pkg directory function run_wasm_pack(extra_options) { let args = ['run', 'nightly', 'wasm-pack', 'build', '--target', 'web', '--release', '.', ...extra_options]; spawnSync('rustup', args, { stdio: 'inherit' }) } +// run the rollup bundler on the main.js file, outputting the results to the dist directory async function run_rollup(use_terser) { const bundle = await rollup({ input: 'js_src/main.js', @@ -94,20 +104,22 @@ async function run_rollup(use_terser) { }); } -async function upload(server, branch, dryrun) { +// load the built code from the dist directory and craft it into the format the API needs +function load_built_code() { let modules = {}; + // track how much space our code uses, since that's limited to 5 MiB let used_bytes = 0; - await fs.readdirSync('dist').map(function (filename) { + fs.readdirSync('dist').map(filename => { if (filename.endsWith('.wasm')) { - const data = fs.readFileSync(path.join('dist', filename), {encoding: 'base64'}); + const data = fs.readFileSync(path.join('dist', filename), { encoding: 'base64' }); const filename_stripped = filename.replace(/\.wasm$/, ''); used_bytes += data.length; modules[filename_stripped] = { binary: data, } } else { - const data = fs.readFileSync(path.join('dist', filename), {encoding: 'utf8'}); + const data = fs.readFileSync(path.join('dist', filename), { encoding: 'utf8' }); const filename_stripped = filename.replace(/\.js$/, ''); used_bytes += data.length; modules[filename_stripped] = data; @@ -117,31 +129,33 @@ async function upload(server, branch, dryrun) { const used_mib = used_bytes / (1024 * 1024); const used_percent = 100 * used_mib / 5; - const usage_string = `${used_mib.toFixed(2)} MiB of 5.0 MiB code size limit (${used_percent.toFixed(2)}%)` + return { used_mib, used_percent, modules } +} + +// upload the code to the servers using the API (or simulate it without uploading, if +// dryrun is true) +async function upload(code, server, branch, dryrun) { + const usage_string = `${code.used_mib.toFixed(2)} MiB of 5.0 MiB code size limit (${code.used_percent.toFixed(2)}%)` if (dryrun) { console.log(`Not uploading due to --dryrun; would use ${usage_string}`); } else { - console.log(`Uploading; using ${usage_string}`); + console.log(`Uploading to branch ${branch}; using ${usage_string}`); const api = await ScreepsAPI.fromConfig(server); - const response = await api.code.set(branch, modules); + const response = await api.code.set(branch, code.modules); console.log(JSON.stringify(response)); } } async function run() { const config = load_config(); - - // clean output (pkg/dist dirs) - output_clean(); - - // run cargo build - run_wasm_pack(config.extra_options); - - // run rollup + if (!config) { + return + } + await output_clean(); + await run_wasm_pack(config.extra_options); await run_rollup(config.use_terser); - - // read resulting files and upload - await upload(argv.server, config.branch, argv.dryrun); + const code = load_built_code(); + await upload(code, argv.server, config.branch, argv.dryrun); } run().catch(console.error) From 42f74944d70a759c5a331037d5a6522e56465087 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 19 Aug 2024 23:36:02 -0600 Subject: [PATCH 06/18] version update note --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 13f98ff..003aab9 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ To migrate an existing bot to using the new Javascript translation layer and dep - Import formatting, particularly for the wasm module. - wasm module initialization has changed, requiring two calls to first compile the module, then to initialize the instance of the module. +- Update your `Cargo.toml` with version `0.22` for `screeps-game-api` - Run `npm run deploy -- --server ptr --dryrun` to compile for PTR, remove the `--dryrun` to deploy [screeps]: https://screeps.com/ From 8edba04bfd06dcc38294341b0f1256765cf3cb22 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 15:48:59 -0600 Subject: [PATCH 07/18] use process.env.npm_package_name to get package name --- js_tools/deploy.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/js_tools/deploy.js b/js_tools/deploy.js index 3307b85..9e72c29 100644 --- a/js_tools/deploy.js +++ b/js_tools/deploy.js @@ -24,6 +24,8 @@ const argv = require('yargs') }) .argv; +const package_name_underscore = process.env.npm_package_name.replace("-", "_"); + // load configuration from .screeps.yaml // unified config format: // https://github.com/screepers/screepers-standards/blob/master/SS3-Unified_Credentials_File.md @@ -89,11 +91,14 @@ async function run_rollup(use_terser) { presets: ['@babel/preset-env'], targets: { "node": 12, - } + }, }), copy({ - // todo, should figure out a better way to get the name - targets: [{ src: 'pkg/screeps_starter_rust_bg.wasm', dest: 'dist', rename: 'screeps_starter_rust.wasm' }] + targets: [{ + src: `pkg/${package_name_underscore}_bg.wasm`, + dest: 'dist', + rename: `${package_name_underscore}.wasm`, + }] }), ] }); From 25ee6668d3f85dd3d1b05881670fe62824c43b7d Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 15:51:45 -0600 Subject: [PATCH 08/18] update readme --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 003aab9..f9af071 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,9 @@ To migrate an existing bot to using the new Javascript translation layer and dep - Create a `package.json` copied from the one in this repo and make appropriate customizations. - Install the node dependencies from the quickstart steps above, then run `npm install` from within the bot directory to install the required packages. -- Copy the `deploy.js` script over to a `js_tools` directory, fix name in the file - (todo would be nice to not have hardcoded). -- Add `main.js` to a `js_src` directory, either moved from your existing `javascript` dir and updated, - or freshly copied. If updating, you'll need to change: +- Copy the `deploy.js` script over to a new `js_tools` directory. +- Add `main.js` to a new `js_src` directory, either moved from your existing `javascript` dir and + updated, or freshly copied. If updating, you'll need to change: - Import formatting, particularly for the wasm module. - wasm module initialization has changed, requiring two calls to first compile the module, then to initialize the instance of the module. From 50d78bc53ae8b41c74bf39191246ff1aade4a325 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 16:43:49 -0600 Subject: [PATCH 09/18] Fix name replace, remove author --- js_tools/deploy.js | 2 +- package.json | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/js_tools/deploy.js b/js_tools/deploy.js index 9e72c29..bf7ecad 100644 --- a/js_tools/deploy.js +++ b/js_tools/deploy.js @@ -24,7 +24,7 @@ const argv = require('yargs') }) .argv; -const package_name_underscore = process.env.npm_package_name.replace("-", "_"); +const package_name_underscore = process.env.npm_package_name.replace(/\-/g, "_"); // load configuration from .screeps.yaml // unified config format: diff --git a/package.json b/package.json index 327d80c..b4c4731 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,7 @@ "type": "git", "url": "git+https://github.com/rustyscreeps/screeps-starter-rust.git" }, - "author": "Shane Madden", - "license": "MIT", + "author": "", "bugs": { "url": "https://github.com/rustyscreeps/screeps-starter-rust/issues" }, From f2de796ebd1c8e448608fb06e5b3124539ab5b31 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 16:49:00 -0600 Subject: [PATCH 10/18] minor cleanup in main.js --- js_src/main.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/js_src/main.js b/js_src/main.js index bb05820..f5e1541 100644 --- a/js_src/main.js +++ b/js_src/main.js @@ -4,6 +4,7 @@ import 'fastestsmallesttextencoderdecoder-encodeinto/EncoderDecoderTogether.min. import * as bot from '../pkg/screeps_starter_rust.js'; // replace this with the name of your module const MODULE_NAME = "screeps_starter_rust"; +const BUCKET_BOOT_THRESHOLD = 1500; // This provides the function `console.error` that wasm_bindgen sometimes expects to exist, // especially with type checks in debug mode. An alternative is to have this be `function () {}` @@ -25,10 +26,6 @@ function console_error() { Game.notify(processedArgs); } -let halt_next_tick = false; - -const BUCKET_BOOT_THRESHOLD = 1500; - // track whether running wasm loop for each tick completes, to detect errors or aborted execution let running = false; From 29a923817cd0bc54e06cb043c34f0c3eb5fb87b2 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 16:59:17 -0600 Subject: [PATCH 11/18] readme updates --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f9af071..fdfdb6d 100644 --- a/README.md +++ b/README.md @@ -24,18 +24,20 @@ cargo install wasm-pack # Install wasm-opt cargo install wasm-opt -# Install nvm: https://github.com/nvm-sh/nvm -# (Windows: https://github.com/coreybutler/nvm-windows) +# Install Node.js for build steps - versions 16 through 22 have been tested, any should work +# nvm is recommended but not required to manage the install, follow instructions at: +# Mac/Linux: https://github.com/nvm-sh/nvm +# Windows: https://github.com/coreybutler/nvm-windows -# Install node at version 20 (16 to 21 tested ok, IIRC) -nvm install 20 -nvm use 20 +# Installs node at version 22 +nvm install 22 +nvm use 22 # Clone the starter git clone https://github.com/rustyscreeps/screeps-starter-rust.git cd screeps-starter-rust # note: if you customize the name of the crate, you'll need to update the MODULE_NAME -# variable in the javascript/main.js file with the updated name +# variable in the javascript/main.js file and the module import with the updated name # Install node deps npm install @@ -69,7 +71,7 @@ To migrate an existing bot to using the new Javascript translation layer and dep the bot directory to install the required packages. - Copy the `deploy.js` script over to a new `js_tools` directory. - Add `main.js` to a new `js_src` directory, either moved from your existing `javascript` dir and - updated, or freshly copied. If updating, you'll need to change: + updated,or freshly copied. If updating, you'll need to change: - Import formatting, particularly for the wasm module. - wasm module initialization has changed, requiring two calls to first compile the module, then to initialize the instance of the module. From e166f4e5a1e9e135ffd48be5dc1c65eb28093734 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 17:01:34 -0600 Subject: [PATCH 12/18] small additional readme updates --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fdfdb6d..feed792 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ cargo install wasm-opt # Mac/Linux: https://github.com/nvm-sh/nvm # Windows: https://github.com/coreybutler/nvm-windows -# Installs node at version 22 +# Installs Node.js at version 22 nvm install 22 nvm use 22 @@ -39,12 +39,10 @@ cd screeps-starter-rust # note: if you customize the name of the crate, you'll need to update the MODULE_NAME # variable in the javascript/main.js file and the module import with the updated name -# Install node deps +# Install dependencies for JS build npm install # Copy the example config, and set up at least one deployment mode. -# Configure credentials if you'd like to upload directly, or a directory to copy to -# if you'd prefer to use the game client to deploy: cp .example-screeps.yaml .screeps.yaml nano .screeps.yaml From d21dc3a36ce67dafebfa5ece02a2bede32ffc86c Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 17:02:41 -0600 Subject: [PATCH 13/18] formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index feed792..fbc0680 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ To migrate an existing bot to using the new Javascript translation layer and dep the bot directory to install the required packages. - Copy the `deploy.js` script over to a new `js_tools` directory. - Add `main.js` to a new `js_src` directory, either moved from your existing `javascript` dir and - updated,or freshly copied. If updating, you'll need to change: + updated, or freshly copied. If updating, you'll need to change: - Import formatting, particularly for the wasm module. - wasm module initialization has changed, requiring two calls to first compile the module, then to initialize the instance of the module. From fca8909dc8f7dd9825a5f302077b91a731c9bbfe Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 17:06:41 -0600 Subject: [PATCH 14/18] wording tweak --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbc0680..7e62667 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ nano .screeps.yaml # compile for a configured server but don't upload npm run deploy -- --server ptr --dryrun -# compile and deploy to a configured server +# compile and upload to a configured server npm run deploy -- --server mmo ``` From 9d7707f2438ff506563fce91bded4a98c645385e Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 17:12:36 -0600 Subject: [PATCH 15/18] formatting tweaks --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7e62667..ff5aa9f 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ npm run deploy -- --server mmo ## Migration to 0.22 Versions of [`screeps-game-api`] at 0.22 or higher are no longer compatible with the -[`cargo-screeps`] tool for building and deployment; the transpilation step being handled -by [Babel] is required to generate code that the game servers can load. +[`cargo-screeps`] tool for building and deployment; the transpile step being handled by [Babel] is +required to transform the generated JS into code that the game servers can load. To migrate an existing bot to using the new Javascript translation layer and deploy script: From 2488fe4dd65a65cd7aa4d208736feb045378a9fc Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 17:13:36 -0600 Subject: [PATCH 16/18] nitpicks --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ff5aa9f..f822664 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ nvm use 22 git clone https://github.com/rustyscreeps/screeps-starter-rust.git cd screeps-starter-rust # note: if you customize the name of the crate, you'll need to update the MODULE_NAME -# variable in the javascript/main.js file and the module import with the updated name +# variable in the js_src/main.js file and the module import with the updated name # Install dependencies for JS build npm install @@ -59,7 +59,7 @@ Versions of [`screeps-game-api`] at 0.22 or higher are no longer compatible with [`cargo-screeps`] tool for building and deployment; the transpile step being handled by [Babel] is required to transform the generated JS into code that the game servers can load. -To migrate an existing bot to using the new Javascript translation layer and deploy script: +To migrate an existing bot to using the new JavaScript translation layer and deploy script: - Create a `.screeps.yaml` with the relevant settings from your `screeps.toml` file applied to the new `.example-screeps.yaml` example file in this repo. From aa3155bfeee16d9fb0b7941f3b82e6dbb3819c4f Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 17:17:21 -0600 Subject: [PATCH 17/18] formatting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f822664..47b52b8 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,8 @@ To migrate an existing bot to using the new JavaScript translation layer and dep new `.example-screeps.yaml` example file in this repo. - Add to your `.gitignore`: `.screeps.yaml`, `node_modules`, and `dist` - Create a `package.json` copied from the one in this repo and make appropriate customizations. -- Install the node dependencies from the quickstart steps above, then run `npm install` from within - the bot directory to install the required packages. +- Install Node.js (from the quickstart steps above), then run `npm install` from within the bot + directory to install the required packages. - Copy the `deploy.js` script over to a new `js_tools` directory. - Add `main.js` to a new `js_src` directory, either moved from your existing `javascript` dir and updated, or freshly copied. If updating, you'll need to change: From 43c0cc975baf0ebf51d7ca38af6c2a19d990afa5 Mon Sep 17 00:00:00 2001 From: Shane Madden Date: Mon, 26 Aug 2024 17:19:25 -0600 Subject: [PATCH 18/18] package.json name change note --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 47b52b8..0f26fdd 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ nvm use 22 git clone https://github.com/rustyscreeps/screeps-starter-rust.git cd screeps-starter-rust # note: if you customize the name of the crate, you'll need to update the MODULE_NAME -# variable in the js_src/main.js file and the module import with the updated name +# variable in the js_src/main.js file and the module import with the updated name, as well +# as the "name" in the package.json # Install dependencies for JS build npm install