Add missing legal_action_space_mask default methods (#1075)

jeremiahpslewis · web-flow · commit 89a46d973826 · 2024-05-13T12:29:22.000+02:00
* Fix devcontainer

* Add patch for missing legal_action_space_mask defaults, add test to StockTradingEnv
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -11,5 +11,5 @@
       "--privileged"
   ],
   "dockerFile": "Dockerfile",
-  "updateContentCommand": "julia -e 'using Pkg; Pkg.develop(path=\"src/ReinforcementLearningBase\"); Pkg.develop(path=\"src/ReinforcementLearningEnvironments\"); Pkg.develop(path=\"src/ReinforcementLearningCore\"); Pkg.develop(path=\"src/ReinforcementLearningFarm\"); Pkg.develop(path=\"src/ReinforcementLearning\");'"
+  "updateContentCommand": "julia -e 'using Pkg; Pkg.develop(path=\"src/ReinforcementLearningBase\"); Pkg.develop(path=\"src/ReinforcementLearningEnvironments\"); Pkg.develop(path=\"src/ReinforcementLearningCore\"); Pkg.develop(path=\"src/ReinforcementLearningFarm\"); Pkg.develop(path=\".\");'"
 }
diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
@@ -15,7 +15,7 @@ jobs:
         run: which julia
         continue-on-error: true
       - name: Install Julia, but only if it is not already available in the PATH
-        uses: julia-actions/setup-julia@v1
+        uses: julia-actions/setup-julia@v2
         with:
           version: '1'
           arch: ${{ runner.arch }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,11 +34,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 100
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: julia-actions/cache@v1
+      - uses: julia-actions/cache@v2
       - name: Get changed files
         id: RLBase-changed
         uses: tj-actions/changed-files@v42
@@ -75,11 +75,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 100
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: julia-actions/cache@v1
+      - uses: julia-actions/cache@v2
       - name: Get changed files
         id: RLCore-changed
         uses: tj-actions/changed-files@v42
@@ -121,11 +121,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 100
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: julia-actions/cache@v1
+      - uses: julia-actions/cache@v2
       - name: Get changed files
         id: RLFarm-changed
         uses: tj-actions/changed-files@v42
@@ -168,11 +168,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 100
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      # - uses: julia-actions/cache@v1
+      # - uses: julia-actions/cache@v2
       - name: Get changed files
         id: RLEnvironments-changed
         uses: tj-actions/changed-files@v42
@@ -205,7 +205,7 @@ jobs:
         with:
           fetch-depth: 0
       - run: python -m pip install --user matplotlib
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: "1"
       - name: Build homepage
diff --git a/Project.toml b/Project.toml
@@ -4,7 +4,6 @@ authors = ["Johanni Brea <jbrea@users.noreply.github.com>", "Jun Tian <tianjun.c
 version = "0.11.0"
 
 [deps]
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 ReinforcementLearningBase = "e575027e-6cd6-5018-9292-cdc6200d2b44"
 ReinforcementLearningCore = "de1b191a-4ae0-4afa-a27b-92d07f46b2d6"
diff --git a/docs/homepage/blog/a_practical_introduction_to_RL.jl/index.html b/docs/homepage/blog/a_practical_introduction_to_RL.jl/index.html
@@ -15415,7 +15415,7 @@ <h2 id="Environments">Environments<a class="anchor-link" href="#Environments">&#
 <div class="text_cell_render border-box-sizing rendered_html">
 
 <pre><code>RLBase.action_space(env::MultiArmBanditsEnv) = Base.OneTo(length(env.true_values))
-RLBase.state(env::MultiArmBanditsEnv) = 1
+RLBase.state(env::MultiArmBanditsEnv, ::Observation, ::DefaultPlayer) = 1
 RLBase.state_space(env::MultiArmBanditsEnv) = Base.OneTo(1)
 RLBase.is_terminated(env::MultiArmBanditsEnv) = env.is_terminated
 RLBase.reward(env::MultiArmBanditsEnv) = env.reward
diff --git a/docs/src/How_to_write_a_customized_environment.md b/docs/src/How_to_write_a_customized_environment.md
@@ -68,7 +68,7 @@ Here `RLBase` is just an alias for `ReinforcementLearningBase`.
 
 ```@repl customized_env
 RLBase.reward(env::LotteryEnv) = env.reward
-RLBase.state(env::LotteryEnv) = !isnothing(env.reward)
+RLBase.state(env::LotteryEnv, ::Observation, ::DefaultPlayer) = !isnothing(env.reward)
 RLBase.state_space(env::LotteryEnv) = [false, true]
 RLBase.is_terminated(env::LotteryEnv) = !isnothing(env.reward)
 RLBase.reset!(env::LotteryEnv) = env.reward = nothing
@@ -181,7 +181,7 @@ RLCore.forward(p.learner.approximator, false)
 
 OK, now we know where the problem is. But how to fix it?
 
-An initial idea is to rewrite the `RLBase.state(env::LotteryEnv)` function to
+An initial idea is to rewrite the `RLBase.state(env::LotteryEnv, ::Observation, ::DefaultPlayer)` function to
 force it return an `Int`. That's workable. But in some cases, we may be using
 environments written by others and it's not very easy to modify the code
 directly. Fortunatelly, some environment wrappers are provided to help us
diff --git a/src/ReinforcementLearningBase/NEWS.md b/src/ReinforcementLearningBase/NEWS.md
@@ -1,5 +1,13 @@
 ### ReinforcementLearningBase.jl Release Notes
 
+#### v0.13.1
+
+- Don't call `legal_action_space_mask` methods when `ActionStyle` is `MinimalActionSet`
+
+#### v0.13.0
+
+- Breaking release compatible with RL.jl v0.11
+
 #### v0.12.0
 
 - Transition to `RLCore.forward`, `RLBase.act!`, `RLBase.plan!` and `Base.push!` syntax instead of functional objects for hooks, policies and environments
diff --git a/src/ReinforcementLearningBase/Project.toml b/src/ReinforcementLearningBase/Project.toml
@@ -1,7 +1,7 @@
 name = "ReinforcementLearningBase"
 uuid = "e575027e-6cd6-5018-9292-cdc6200d2b44"
 authors = ["Johanni Brea <jbrea@users.noreply.github.com>", "Jun Tian <tianjun.cpp@gmail.com>"]
-version = "0.13.0"
+version = "0.13.1"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
diff --git a/src/ReinforcementLearningBase/src/interface.jl b/src/ReinforcementLearningBase/src/interface.jl
@@ -487,7 +487,7 @@ For environments of [`MINIMAL_ACTION_SET`](@ref), the result is the same with
 @multi_agent_env_api legal_action_space(env::AbstractEnv, player=current_player(env)) =
     legal_action_space(ActionStyle(env), env, player)
 
-legal_action_space(::MinimalActionSet, env, player::AbstractPlayer) = action_space(env)
+legal_action_space(::MinimalActionSet, env::AbstractEnv, player::AbstractPlayer) = action_space(env)
 
 """
     legal_action_space_mask(env, player=current_player(env)) -> AbstractArray{Bool}
@@ -497,6 +497,9 @@ Required for environments of [`FULL_ACTION_SET`](@ref). As a default implementat
      the subset [`legal_action_space`](@ref).
 """
 @multi_agent_env_api legal_action_space_mask(env::AbstractEnv, player=current_player(env)) =
+    legal_action_space_mask(ActionStyle(env), env, player)
+
+legal_action_space_mask(::FullActionSet, env::AbstractEnv, player=current_player(env)) =
     map(action_space(env, player)) do action
         action in legal_action_space(env, player)
     end
diff --git a/src/ReinforcementLearningBase/test/interface.jl b/src/ReinforcementLearningBase/test/interface.jl
@@ -4,7 +4,7 @@ struct TestEnv <: RLBase.AbstractEnv
     state::Int
 end
 
-function RLBase.state(env::TestEnv, ::Observation{Any}, ::DefaultPlayer)
+function RLBase.state(env::TestEnv, ::Observation, ::DefaultPlayer)
     return env.state
 end
 
diff --git a/src/ReinforcementLearningCore/NEWS.md b/src/ReinforcementLearningCore/NEWS.md
@@ -1,5 +1,9 @@
 # ReinforcementLearningCore.jl Release Notes
 
+#### v0.15.3
+
+- Make `FluxApproximator` work with `QBasedPolicy`
+
 #### v0.15.2
 
 - Make QBasedPolicy general for AbstractLearner s (#1069)
diff --git a/src/ReinforcementLearningCore/Project.toml b/src/ReinforcementLearningCore/Project.toml
@@ -1,6 +1,6 @@
 name = "ReinforcementLearningCore"
 uuid = "de1b191a-4ae0-4afa-a27b-92d07f46b2d6"
-version = "0.15.2"
+version = "0.15.3"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
diff --git a/src/ReinforcementLearningCore/src/policies/learners/abstract_learner.jl b/src/ReinforcementLearningCore/src/policies/learners/abstract_learner.jl
@@ -25,12 +25,15 @@ function RLBase.optimise!(::AbstractLearner, ::AbstractStage, ::Trajectory) end
 
 function RLBase.optimise!(::AbstractLearner, ::AbstractStage, ::NamedTuple) end
 
-function RLBase.plan!(explorer::AbstractExplorer, learner::AbstractLearner, env::AbstractEnv)
-    legal_action_space_ = RLBase.legal_action_space_mask(env)
-    RLBase.plan!(explorer, forward(learner, env), legal_action_space_)
+function RLBase.plan!(explorer::AbstractExplorer, learner::AbstractLearner, env::AbstractEnv, player=current_player(env))
+    return RLBase.plan!(ActionStyle(env), explorer, learner, env, player)
 end
 
-function RLBase.plan!(explorer::AbstractExplorer, learner::AbstractLearner, env::AbstractEnv, player::AbstractPlayer)
+function RLBase.plan!(::FullActionSet, explorer::AbstractExplorer, learner::AbstractLearner, env::AbstractEnv, player=current_player(env))
     legal_action_space_ = RLBase.legal_action_space_mask(env, player)
     return RLBase.plan!(explorer, forward(learner, env, player), legal_action_space_)
 end
+
+function RLBase.plan!(::MinimalActionSet, explorer::AbstractExplorer, learner::AbstractLearner, env::AbstractEnv, player=current_player(env))
+    return RLBase.plan!(explorer, forward(learner, env, player))
+end
diff --git a/src/ReinforcementLearningCore/src/policies/learners/flux_approximator.jl b/src/ReinforcementLearningCore/src/policies/learners/flux_approximator.jl
@@ -41,7 +41,7 @@ FluxApproximator(model, optimiser::Flux.Optimise.AbstractOptimiser; use_gpu=fals
 Flux.@layer FluxApproximator trainable=(model,)
 
 forward(A::FluxApproximator, args...; kwargs...) = A.model(args...; kwargs...)
-forward(A::FluxApproximator, env::E) where {E <: AbstractEnv} = env |> state |> (x -> forward(A, x))
+forward(A::FluxApproximator, env::E, player::AbstractPlayer=current_player(env)) where {E <: AbstractEnv} = env |> (x -> state(x, player)) |> (x -> forward(A, x))
 
 RLBase.optimise!(A::FluxApproximator, grad::NamedTuple) =
     Flux.Optimise.update!(A.optimiser_state, A.model, grad.model)
diff --git a/src/ReinforcementLearningCore/src/policies/q_based_policy.jl b/src/ReinforcementLearningCore/src/policies/q_based_policy.jl
@@ -36,7 +36,14 @@ function RLBase.plan!(policy::QBasedPolicy{L,Ex}, env::E, player::Player) where
 end
 
 RLBase.prob(policy::QBasedPolicy{L,Ex}, env::AbstractEnv) where {L<:AbstractLearner,Ex<:AbstractExplorer} =
+    prob(ActionStyle(env), policy, env)
+
+RLBase.prob(::MinimalActionSet, policy::QBasedPolicy{L,Ex}, env::AbstractEnv) where {L<:AbstractLearner,Ex<:AbstractExplorer} =
+    prob(policy.explorer, forward(policy.learner, env))
+
+RLBase.prob(::FullActionSet, policy::QBasedPolicy{L,Ex}, env::AbstractEnv) where {L<:AbstractLearner,Ex<:AbstractExplorer} =
     prob(policy.explorer, forward(policy.learner, env), legal_action_space_mask(env))
 
+
 #the internal learner defines the optimization stage.
 RLBase.optimise!(policy::QBasedPolicy, stage::AbstractStage, trajectory::Trajectory) = RLBase.optimise!(policy.learner, stage, trajectory)
diff --git a/src/ReinforcementLearningCore/src/policies/random_policy.jl b/src/ReinforcementLearningCore/src/policies/random_policy.jl
@@ -24,7 +24,7 @@ RandomPolicy(s = nothing; rng = Random.default_rng()) = RandomPolicy(s, rng)
 
 RLBase.optimise!(::RandomPolicy, x::NamedTuple) = nothing
 
-RLBase.plan!(p::RandomPolicy{S,RNG}, env::AbstractEnv) where {S,RNG<:AbstractRNG} = rand(p.rng, p.action_space)
+RLBase.plan!(p::RandomPolicy{S,RNG}, ::AbstractEnv) where {S,RNG<:AbstractRNG} = rand(p.rng, p.action_space)
 
 function RLBase.plan!(p::RandomPolicy{Nothing,RNG}, env::AbstractEnv) where {RNG<:AbstractRNG}
     legal_action_space_ = RLBase.legal_action_space(env)
@@ -45,7 +45,7 @@ function RLBase.prob(p::RandomPolicy{S,RNG}, s) where {S,RNG<:AbstractRNG}
     Categorical(Fill(1 / n, n); check_args = false)
 end
 
-RLBase.prob(p::RandomPolicy{Nothing,RNG}, x) where {RNG<:AbstractRNG} =
+RLBase.prob(::RandomPolicy{Nothing,RNG}, x) where {RNG<:AbstractRNG} =
     @error "no I really don't know how to calculate the prob from nothing"
 
 #####
@@ -54,7 +54,7 @@ RLBase.prob(p::RandomPolicy{Nothing,RNG}, env::AbstractEnv) where {RNG<:Abstract
     prob(p, env, ChanceStyle(env))
 
 function RLBase.prob(
-    p::RandomPolicy{Nothing,RNG},
+    ::RandomPolicy{Nothing,RNG},
     env::AbstractEnv,
     ::RLBase.AbstractChanceStyle,
 ) where {RNG<:AbstractRNG}
diff --git a/src/ReinforcementLearningCore/test/policies/learners/abstract_learner.jl b/src/ReinforcementLearningCore/test/policies/learners/abstract_learner.jl
@@ -14,7 +14,7 @@ struct MockLearner <: AbstractLearner end
             return [1.0, 2.0]
         end
 
-        RLBase.state(::MockEnv, ::Observation{Any}, ::DefaultPlayer) = 1
+        RLBase.state(::MockEnv, ::Observation, ::DefaultPlayer) = 1
         RLBase.state(::MockEnv, ::Observation{Any}, ::Player) = 1
 
         env = MockEnv()
diff --git a/src/ReinforcementLearningEnvironments/NEWS.md b/src/ReinforcementLearningEnvironments/NEWS.md
@@ -1,5 +1,13 @@
 ### ReinforcementLearningEnvironments.jl Release Notes
 
+#### v0.9.1
+
+- Update `state` calls to use full signature (so compatible with more algorithms)
+
+#### v0.9.0
+
+- Compatible with RL.jl v0.11
+
 #### v0.8
 
 - Transition to `RLCore.forward`, `RLBase.act!`, `RLBase.plan!` and `Base.push!` syntax instead of functional objects for hooks, policies and environments
@@ -63,4 +71,4 @@
 
 #### v0.6.0
 
-- Set `AcrobotEnv` into lazy loading to reduce the dependency of `OrdinaryDiffEq`.
+- Set `AcrobotEnv` into lazy loading to reduce the dependency of `OrdinaryDiffEq`.
diff --git a/src/ReinforcementLearningEnvironments/Project.toml b/src/ReinforcementLearningEnvironments/Project.toml
@@ -34,6 +34,7 @@ ArcadeLearningEnvironment = "b7f77d8d-088d-5e02-8ac0-89aab2acc977"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 DomainSets = "5b8099bc-c8ec-5219-889f-1d9e522a28bf"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 OpenSpiel = "ceb70bd2-fe3f-44f0-b81f-41608acaf2f2"
 OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
@@ -48,6 +49,7 @@ test = [
     "JLD2",
     "Conda",
     "DomainSets",
+    "Flux",
     "OpenSpiel",
     "OrdinaryDiffEq",
     "PyCall",
diff --git a/src/ReinforcementLearningEnvironments/src/environments/3rd_party/AcrobotEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/3rd_party/AcrobotEnv.jl
@@ -85,7 +85,7 @@ RLBase.state_space(env::AcrobotEnv) = ArrayProductDomain(
 )
 
 RLBase.is_terminated(env::AcrobotEnv) = env.done
-RLBase.state(env::AcrobotEnv) = acrobot_observation(env.state)
+RLBase.state(env::AcrobotEnv, ::Observation, ::DefaultPlayer) = acrobot_observation(env.state)
 RLBase.reward(env::AcrobotEnv) = env.reward
 
 function RLBase.reset!(env::AcrobotEnv{T}) where {T<:Number}
diff --git a/src/ReinforcementLearningEnvironments/src/environments/3rd_party/atari.jl b/src/ReinforcementLearningEnvironments/src/environments/3rd_party/atari.jl
@@ -121,7 +121,7 @@ RLBase.nameof(env::AtariEnv) = "AtariEnv($(env.name))"
 RLBase.action_space(env::AtariEnv) = env.action_space
 RLBase.reward(env::AtariEnv) = env.reward
 RLBase.is_terminated(env::AtariEnv) = is_terminal(env)
-RLBase.state(env::AtariEnv) = env.screens[1]
+RLBase.state(env::AtariEnv, ::Observation, ::DefaultPlayer) = env.screens[1]
 RLBase.state_space(env::AtariEnv) = env.observation_space
 
 function Random.seed!(env::AtariEnv, s)
diff --git a/src/ReinforcementLearningEnvironments/src/environments/3rd_party/gym.jl b/src/ReinforcementLearningEnvironments/src/environments/3rd_party/gym.jl
@@ -86,7 +86,7 @@ function RLBase.is_terminated(env::GymEnv{T}) where {T}
     end
 end
 
-function RLBase.state(env::GymEnv{T}) where {T}
+function RLBase.state(env::GymEnv{T}, ::Observation, ::DefaultPlayer) where {T}
     if pyisinstance(env.state, PyCall.@pyglobalobj :PyTuple_Type) && length(env.state) == 4
         obs, reward, isdone, info = convert(Tuple{T,Float64,Bool,PyDict}, env.state)
         obs
diff --git a/src/ReinforcementLearningEnvironments/src/environments/3rd_party/snake.jl b/src/ReinforcementLearningEnvironments/src/environments/3rd_party/snake.jl
@@ -42,7 +42,7 @@ RLBase.act!(env::SnakeGameEnv, action::Int) = env([SNAKE_GAME_ACTIONS[action]])
 RLBase.act!(env::SnakeGameEnv, actions::Vector{Int}) = env(map(a -> SNAKE_GAME_ACTIONS[a], actions))
 
 RLBase.action_space(env::SnakeGameEnv) = Base.OneTo(4)
-RLBase.state(env::SnakeGameEnv) = env.game.board
+RLBase.state(env::SnakeGameEnv, ::Observation, ::DefaultPlayer) = env.game.board
 RLBase.state_space(env::SnakeGameEnv) = ArrayProductDomain(fill(false:true, size(env.game.board)))
 RLBase.reward(env::SnakeGameEnv{<:Any,SINGLE_AGENT}) =
     length(env.game.snakes[]) - env.latest_snakes_length[]
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/BitFlippingEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/BitFlippingEnv.jl
@@ -37,7 +37,7 @@ function RLBase.act!(env::BitFlippingEnv, action::Int)
     end
 end
 
-RLBase.state(env::BitFlippingEnv) = state(env::BitFlippingEnv, Observation{BitArray{1}}())
+RLBase.state(env::BitFlippingEnv, ::Observation, ::DefaultPlayer) = state(env::BitFlippingEnv, Observation{BitArray{1}}())
 RLBase.state(env::BitFlippingEnv, ::Observation) = env.state
 RLBase.state(env::BitFlippingEnv, ::GoalState) = env.goal_state
 RLBase.state_space(env::BitFlippingEnv, ::Observation) = ArrayProductDomain(fill(false:true, env.N))
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/CartPoleEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/CartPoleEnv.jl
@@ -83,7 +83,7 @@ CartPoleEnv{T}(; kwargs...) where {T} = CartPoleEnv(T=T, kwargs...)
 Random.seed!(env::CartPoleEnv, seed) = Random.seed!(env.rng, seed)
 RLBase.reward(env::CartPoleEnv{T}) where {T} = env.done ? zero(T) : one(T)
 RLBase.is_terminated(env::CartPoleEnv) = env.done
-RLBase.state(env::CartPoleEnv) = env.state
+RLBase.state(env::CartPoleEnv, ::Observation, ::DefaultPlayer) = env.state
 
 function RLBase.state_space(env::CartPoleEnv{T}) where {T}
     ((-2 * env.params.xthreshold) .. (2 * env.params.xthreshold)) ×
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/GraphShortestPathEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/GraphShortestPathEnv.jl
@@ -54,7 +54,7 @@ function RLBase.act!(env::GraphShortestPathEnv, action)
     env.reward = env.pos == env.goal ? 0 : -1
 end
 
-RLBase.state(env::GraphShortestPathEnv) = env.pos
+RLBase.state(env::GraphShortestPathEnv, ::Observation, ::DefaultPlayer) = env.pos
 RLBase.state_space(env::GraphShortestPathEnv) = axes(env.graph, 2)
 RLBase.action_space(env::GraphShortestPathEnv) = axes(env.graph, 2)
 RLBase.legal_action_space(env::GraphShortestPathEnv) = (env.graph[:, env.pos]).nzind
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/KuhnPokerEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/KuhnPokerEnv.jl
@@ -107,7 +107,7 @@ RLBase.action_space(env::KuhnPokerEnv, ::ChancePlayer) = Base.OneTo(length(KUHN_
 
 RLBase.legal_action_space(env::KuhnPokerEnv, p::ChancePlayer) = Tuple(x for x in action_space(env, p) if KUHN_POKER_CARDS[x] ∉ env.cards)
 
-function RLBase.legal_action_space_mask(env::KuhnPokerEnv, p::ChancePlayer)
+function RLBase.legal_action_space_mask(env::KuhnPokerEnv, ::ChancePlayer)
     m = fill(true, 3)
     m[env.cards] .= false
     m
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/MontyHallEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/MontyHallEnv.jl
@@ -58,7 +58,7 @@ function RLBase.legal_action_space_mask(env::MontyHallEnv)
     mask
 end
 
-function RLBase.state(env::MontyHallEnv)
+function RLBase.state(env::MontyHallEnv, ::Observation, ::DefaultPlayer)
     if isnothing(env.host_action)
         1
     else
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/MountainCarEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/MountainCarEnv.jl
@@ -94,7 +94,7 @@ RLBase.action_space(::MountainCarEnv{<:AbstractFloat,<:AbstractFloat}) = -1.0 ..
 
 RLBase.reward(env::MountainCarEnv{T}) where {T} = env.done ? zero(T) : -one(T)
 RLBase.is_terminated(env::MountainCarEnv) = env.done
-RLBase.state(env::MountainCarEnv) = env.state
+RLBase.state(env::MountainCarEnv, ::Observation, ::DefaultPlayer) = env.state
 
 function RLBase.reset!(env::MountainCarEnv{T}) where {T}
     env.state[1] = 0.2 * rand(env.rng, T) - 0.6
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/MultiArmBanditsEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/MultiArmBanditsEnv.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/PendulumEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/PendulumEnv.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/RandomWalk1D.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/RandomWalk1D.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/RockPaperScissorsEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/RockPaperScissorsEnv.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/StockTradingEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/StockTradingEnv.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/TicTacToeEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/TicTacToeEnv.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/examples/TigerProblemEnv.jl b/src/ReinforcementLearningEnvironments/src/environments/examples/TigerProblemEnv.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/non_interactive/pendulum.jl b/src/ReinforcementLearningEnvironments/src/environments/non_interactive/pendulum.jl
diff --git a/src/ReinforcementLearningEnvironments/src/environments/wrappers/DefaultStateStyle.jl b/src/ReinforcementLearningEnvironments/src/environments/wrappers/DefaultStateStyle.jl
diff --git a/src/ReinforcementLearningEnvironments/test/environments/examples/rock_paper_scissors.jl b/src/ReinforcementLearningEnvironments/test/environments/examples/rock_paper_scissors.jl
diff --git a/src/ReinforcementLearningEnvironments/test/environments/examples/stock_trading_env.jl b/src/ReinforcementLearningEnvironments/test/environments/examples/stock_trading_env.jl
diff --git a/src/ReinforcementLearningEnvironments/test/environments/wrappers/wrappers.jl b/src/ReinforcementLearningEnvironments/test/environments/wrappers/wrappers.jl

Original file line number	Diff line number	Diff line change
`@@ -11,5 +11,5 @@`
`11`	`11`	`"--privileged"`
`12`	`12`	`],`
`13`	`13`	`"dockerFile": "Dockerfile",`
`14`		`- "updateContentCommand": "julia -e 'using Pkg; Pkg.develop(path=\"src/ReinforcementLearningBase\"); Pkg.develop(path=\"src/ReinforcementLearningEnvironments\"); Pkg.develop(path=\"src/ReinforcementLearningCore\"); Pkg.develop(path=\"src/ReinforcementLearningFarm\"); Pkg.develop(path=\"src/ReinforcementLearning\");'"`
	`14`	`+ "updateContentCommand": "julia -e 'using Pkg; Pkg.develop(path=\"src/ReinforcementLearningBase\"); Pkg.develop(path=\"src/ReinforcementLearningEnvironments\"); Pkg.develop(path=\"src/ReinforcementLearningCore\"); Pkg.develop(path=\"src/ReinforcementLearningFarm\"); Pkg.develop(path=\".\");'"`
`15`	`15`	`}`