Add support for large preloads and premines (#153)

We're using the Kurtosis Ethereum package to deploy chains in testing.
Some of these chains have large numbers of preloaded contracts and
premine addresses. Since the preloads and premines are loaded into the
entrypoint via env vars, they can sometimes overflow the maximum arg
space available to Bash and cause 'Argument list too long' errors. This
PR removes the preloads/premines from the env file so that the script
can proceed as originally intend, and implements a custom envsubst
script in Python for use with execution layer genesis generation.

apps/el-gen/envsubst.py

@@ -0,0 +1,23 @@
+import os
+import re
+import sys
+
+from dotenv import dotenv_values
+
+config = dotenv_values(os.environ['FULL_ENV_FILE'])
+
+bracket_sub_re = re.compile(r'\${(\w+)}')
+basic_sub_re = re.compile(r'\$([A-Z0-9_]+)')
+
+
+def sub(m):
+    cfg = config.get(m.group(1), None)
+    if cfg is None:
+        raise Exception(f"Missing environment variable {m.group(1)}")
+    return cfg
+
+
+for line in sys.stdin:
+    out = bracket_sub_re.sub(sub, line)
+    out = basic_sub_re.sub(sub, out)
+    sys.stdout.write(out)

apps/el-gen/requirements.txt

@@ -1,3 +1,4 @@
 ruamel.yaml==0.17.16
 web3>=6.15.0
-setuptools>=69.1.0
+setuptools>=69.1.0
+python-dotenv==1.0.1

entrypoint.sh

@@ -1,9 +1,15 @@
 #!/bin/bash -e
+export FULL_ENV_FILE="/defaults/defaults.env"
+
 if [ -f /config/values.env ];
 then
-    source /config/values.env
+    export FULL_ENV_FILE="/config/values.env"
 fi
-source /defaults/defaults.env
+
+# Pull these values out of the env file since they can be very large and cause
+# "arguments list too long" errors in the shell.
+grep -v "ADDITIONAL_PRELOADED_CONTRACTS" $FULL_ENV_FILE | grep -v "EL_PREMINE_ADDRS" > /tmp/values-short.env
+source /tmp/values-short.env
 
 SERVER_ENABLED="${SERVER_ENABLED:-false}"
 SERVER_PORT="${SERVER_PORT:-8000}"
@@ -30,7 +36,7 @@ gen_el_config(){
     if ! [ -f "/data/metadata/genesis.json" ]; then
         tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
         mkdir -p /data/metadata
-        envsubst < /config/el/genesis-config.yaml > $tmp_dir/genesis-config.yaml
+        python3 /apps/el-gen/envsubst.py < /config/el/genesis-config.yaml > $tmp_dir/genesis-config.yaml
         python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml      > /data/metadata/genesis.json
         python3 /apps/el-gen/genesis_chainspec.py $tmp_dir/genesis-config.yaml > /data/metadata/chainspec.json
         python3 /apps/el-gen/genesis_besu.py $tmp_dir/genesis-config.yaml > /data/metadata/besu.json