Osrs env upstream

build.sh

@@ -25,6 +25,7 @@ for arg in "$@"; do
         --debug) DEBUG=1 ;;
         --local) MODE=local ;;
         --fast)  MODE=fast ;;
+        --gprof) MODE=fast; GPROF=1 ;;
         --web)   MODE=web ;;
         --profile) MODE=profile ;;
         --cpu)   MODE=cpu; PRECISION="-DPRECISION_FLOAT" ;;
@@ -55,7 +56,8 @@ PLATFORM="$(uname -s)"
 if [ "$PLATFORM" = "Linux" ]; then
     RAYLIB_NAME='raylib-5.5_linux_amd64'
     OMP_LIB=-lomp5
-    SANITIZE_FLAGS=(-fsanitize=address,undefined,bounds,pointer-overflow,leak -fno-omit-frame-pointer)
+    # SANITIZE_FLAGS=(-fsanitize=address,undefined,bounds,pointer-overflow,leak -fno-omit-frame-pointer)
+    SANITIZE_FLAGS=()
     STANDALONE_LDFLAGS=(-lGL)
     SHARED_LDFLAGS=(-Bsymbolic-functions)
 else
@@ -116,6 +118,18 @@ elif [ "$ENV" = "impulse_wars" ]; then
     download "$BOX2D_NAME" "$BOX2D_URL/$BOX2D_NAME.tar.gz"
     INCLUDES+=(-I./$BOX2D_NAME/include -I./$BOX2D_NAME/src)
     LINK_ARCHIVES+=("./$BOX2D_NAME/libbox2d.a")
+elif [[ "$ENV" == osrs_* ]]; then
+    SRC_DIR="ocean/$ENV"
+    INCLUDES+=(-I./ocean/osrs)
+    # download visual assets to data/ at repo root (where the binary looks).
+    # eval via puffer renders through the same _C.so, so assets must be present
+    # for any osrs build, not just --local.
+    if [ ! -f "data/equipment.models" ]; then
+        echo "Downloading OSRS visual assets..."
+        OSRS_ASSETS_URL="https://github.com/valtterivalo/PufferLib/releases/download/osrs-assets-v8/osrs-assets-v8.tar.gz"
+        mkdir -p data
+        curl -sL "$OSRS_ASSETS_URL" | tar xz --strip-components=1 -C data
+    fi
 elif [ -d "ocean/$ENV" ]; then
     SRC_DIR="ocean/$ENV"
 else
@@ -131,17 +145,28 @@ if [ -n "$DEBUG" ] || [ "$MODE" = "local" ]; then
     LINK_OPT="-g"
 else
     CLANG_OPT=(-O2 -DNDEBUG "${CLANG_WARN[@]}")
+    if [ -n "$GPROF" ]; then
+        CLANG_OPT+=(-pg)
+    fi
     NVCC_OPT="-O2 --threads 0"
     LINK_OPT="-O2"
 fi
 if [ "$MODE" = "local" ] || [ "$MODE" = "fast" ]; then
+    # OSRS envs share a single visual binary
+    if [[ "$ENV" == osrs_* ]]; then
+        VISUAL_SRC="ocean/osrs/osrs_visual.c"
+        VISUAL_DEFS="-DOSRS_VISUAL"
+    else
+        VISUAL_SRC="$SRC_DIR/$ENV.c"
+        VISUAL_DEFS=""
+    fi
     FLAGS=(
         "${INCLUDES[@]}"
-        "$SRC_DIR/$ENV.c" $EXTRA_SRC -o "$OUTPUT_NAME"
+        "$VISUAL_SRC" $EXTRA_SRC -o "$OUTPUT_NAME"
         "${LINK_ARCHIVES[@]}"
         "${STANDALONE_LDFLAGS[@]}"
         -lm -lpthread -fopenmp
-        -DPLATFORM_DESKTOP
+        -DPLATFORM_DESKTOP $VISUAL_DEFS
     )
     echo "Compiling $ENV..."
     ${CC:-clang} "${CLANG_OPT[@]}" "${FLAGS[@]}"
@@ -216,8 +241,9 @@ fi
 
 echo "Compiling static library for $ENV..."
 ${CC:-clang} -c "${CLANG_OPT[@]}" \
+    "${INCLUDES[@]}" \
     -I. -Isrc -I$SRC_DIR -Ivendor \
-    -I./$RAYLIB_NAME/include -I$CUDA_HOME/include \
+    -I$CUDA_HOME/include \
     -DPLATFORM_DESKTOP \
     -fno-semantic-interposition -fvisibility=hidden \
     -fPIC -fopenmp \

config/osrs_inferno.ini

@@ -0,0 +1,184 @@
+# OSRS Inferno encounter.
+# 8 action heads (79 logits), 1058 obs, long episodes (300-8000+ ticks).
+
+[base]
+env_name = osrs_inferno
+score_metric = episode_return
+
+[env]
+start_wave = 69
+damage_reward_coeff = 0.01
+shield_penalty_coeff = 0.01
+tag_reward_coeff = 0.25
+late_start_supply_profile_scale = 1.0
+mask_in_obs = 1.0
+record_best_replay_path = ""
+play_replay_path = ""
+# curriculum: fraction of agents starting at later waves (rest at start_wave)
+curriculum_wave_1 = 20
+curriculum_frac_1 = 0.00
+curriculum_wave_2 = 40
+curriculum_frac_2 = 0.00
+curriculum_wave_3 = 60
+curriculum_frac_3 = 0.00
+
+[vec]
+total_agents = 8192
+num_buffers = 4
+
+[policy]
+hidden_size = 128
+num_layers = 3
+
+[train]
+total_timesteps = 400_000_000
+
+[sweep]
+min_sps = 50000
+max_suggestion_cost = 3600
+metric = episode_return
+metric_distribution = linear
+
+[sweep.train.total_timesteps]
+distribution = log_normal
+min = 500000000
+max = 3200000000
+scale = time
+
+[sweep.train.horizon]
+distribution = uniform_pow2
+min = 32
+max = 256
+scale = auto
+
+[sweep.train.learning_rate]
+distribution = log_normal
+min = 0.0003
+max = 0.01
+scale = 0.5
+
+[sweep.train.ent_coef]
+distribution = log_normal
+min = 0.001
+max = 0.1
+scale = auto
+
+[sweep.train.gamma]
+distribution = logit_normal
+min = 0.99
+max = 0.999999
+scale = auto
+
+[sweep.train.min_lr_ratio]
+distribution = uniform
+min = 0.0
+max = 0.5
+scale = auto
+
+[sweep.train.beta1]
+distribution = uniform
+min = 0.8
+max = 0.99
+scale = auto
+
+[sweep.train.eps]
+distribution = log_normal
+min = 1e-6
+max = 1e-4
+scale = auto
+
+[sweep.train.gae_lambda]
+distribution = logit_normal
+min = 0.5
+max = 0.999
+scale = auto
+
+[sweep.train.vtrace_rho_clip]
+distribution = uniform
+min = 1.0
+max = 3.0
+scale = auto
+
+[sweep.train.vtrace_c_clip]
+distribution = uniform
+min = 1.0
+max = 2.5
+scale = auto
+
+[sweep.train.prio_alpha]
+distribution = logit_normal
+min = 0.0
+max = 0.999
+scale = auto
+
+[sweep.train.prio_beta0]
+distribution = logit_normal
+min = 0.01
+max = 0.8
+scale = auto
+
+[sweep.train.clip_coef]
+distribution = uniform
+min = 0.05
+max = 1.5
+scale = auto
+
+[sweep.train.vf_coef]
+distribution = log_normal
+min = 0.005
+max = 0.5
+scale = auto
+
+[sweep.train.vf_clip_coef]
+distribution = uniform
+min = 0.1
+max = 2.0
+scale = auto
+
+[sweep.train.max_grad_norm]
+distribution = uniform
+min = 0.5
+max = 3.0
+scale = auto
+
+[sweep.train.replay_ratio]
+distribution = uniform
+min = 0.1
+max = 2.0
+scale = auto
+
+[sweep.train.weight_decay]
+distribution = log_normal
+min = 0.001
+max = 1.0
+scale = auto
+
+[sweep.train.minibatch_size]
+distribution = uniform_pow2
+min = 2048
+max = 8192
+scale = auto
+
+[sweep.vec.num_buffers]
+distribution = uniform_pow2
+min = 1
+max = 4
+scale = auto
+
+[sweep.vec.total_agents]
+distribution = uniform_pow2
+min = 128
+max = 4096
+scale = auto
+
+[sweep.policy.hidden_size]
+distribution = uniform_pow2
+min = 128
+max = 1024
+scale = auto
+
+[sweep.policy.num_layers]
+distribution = uniform
+min = 2
+max = 5.0
+scale = auto

config/osrs_pvp.ini

@@ -0,0 +1,37 @@
+# OSRS NH PvP encounter.
+# 7 action heads (39 logits), 334 obs + 39 mask = 373 total, short episodes (~300 ticks).
+
+[base]
+env_name = osrs_pvp
+policy_name = MinGRU
+rnn_name = Recurrent
+score_metric = episode_return
+
+[env]
+mask_in_obs = 1.0
+
+[vec]
+total_agents = 512
+num_buffers = 2
+
+[policy]
+hidden_size = 256
+num_layers = 2
+
+[train]
+total_timesteps = 500000000
+horizon = 32
+learning_rate = 0.003
+beta1 = 0.95
+eps = 0.00001
+ent_coef = 0.01
+gamma = 0.997
+gae_lambda = 0.95
+clip_coef = 0.2
+vf_coef = 0.5
+vf_clip_coef = 0.5
+max_grad_norm = 1.0
+replay_ratio = 0.25
+minibatch_size = 4096
+ns_iters = 5
+weight_decay = 0.001

config/osrs_zulrah.ini

@@ -0,0 +1,38 @@
+# OSRS Zulrah encounter.
+# 6 action heads (41 logits), 81 obs + 41 mask = 122 total, medium episodes (~600 ticks max).
+
+[base]
+env_name = osrs_zulrah
+policy_name = MinGRU
+rnn_name = Recurrent
+score_metric = episode_return
+
+[env]
+mask_in_obs = 1.0
+gear_tier = 2.0
+
+[vec]
+total_agents = 512
+num_buffers = 2
+
+[policy]
+hidden_size = 256
+num_layers = 2
+
+[train]
+total_timesteps = 500000000
+horizon = 32
+learning_rate = 0.003
+beta1 = 0.95
+eps = 0.00001
+ent_coef = 0.01
+gamma = 0.999
+gae_lambda = 0.95
+clip_coef = 0.2
+vf_coef = 0.5
+vf_clip_coef = 0.5
+max_grad_norm = 1.0
+replay_ratio = 0.25
+minibatch_size = 4096
+ns_iters = 5
+weight_decay = 0.001

ocean/osrs/README.md

@@ -0,0 +1,35 @@
+# osrs envs
+
+build the compiled backend with the normal puffer flow:
+
+```bash
+./build.sh osrs_inferno
+./build.sh osrs_zulrah
+./build.sh osrs_pvp
+```
+
+the active python entrypoint is `puffer`, backed by `pufferlib/pufferl.py`.
+
+```bash
+puffer train osrs_inferno
+puffer sweep osrs_inferno
+puffer eval osrs_inferno
+puffer eval osrs_inferno --load-model-path /path/to/checkpoint.bin
+```
+
+env configs live in `config/ocean/<env>.ini`.
+
+inferno best replay recording is opt-in:
+
+```bash
+puffer train osrs_inferno --env.record-best-replay-path checkpoints/osrs_inferno/best.replay
+puffer eval osrs_inferno --env.play-replay-path checkpoints/osrs_inferno/best.replay
+```
+
+the standalone visual binary still exists for direct rendering:
+
+```bash
+cd ocean/osrs
+make visual
+./osrs_visual --encounter inferno --replay /path/to/best.replay
+```