71 KiB
71 KiB
Крестики-нолики: https://github.com/nczempin/gym-tic-tac-toe
In [2]:
import gymnasium as gym
from gymnasium import spaces
class TicTacToeEnv(gym.Env):
metadata = {'render.modes': ['human']}
symbols = ['O', ' ', 'X']
def __init__(self):
super().__init__()
self.action_space = spaces.Discrete(9)
self.observation_space = spaces.Discrete(9 * 3 * 2)
self.reset()
def step(self, action):
done = False
reward = 0
p, square = action
board = self.state['board']
proposed = board[square]
om = self.state['on_move']
if proposed != 0:
print(f"Незаконный ход: Квадрат {square} уже занят.")
done = True
reward = -1 * om
if p != om:
print(f"Незаконный ход: игрок {p} не находится в движении")
done = True
reward = -1 * om
else:
board[square] = p
self.state['on_move'] = -p
for i in range(3):
if (board[i * 3] == p and board[i * 3 + 1] == p and board[i * 3 + 2] == p) or \
(board[i] == p and board[i + 3] == p and board[i + 6] == p):
reward = p
done = True
break
if (board[0] == p and board[4] == p and board[8] == p) or \
(board[2] == p and board[4] == p and board[6] == p):
reward = p
done = True
return self.state, reward, done, {}
def reset(self):
self.state = {}
self.state['board'] = [0, 0, 0, 0, 0, 0, 0, 0, 0]
self.state['on_move'] = 1
return self.state, {}
def render(self, close=False):
if close:
return
print("on move: " , self.symbols[self.state['on_move']+1])
for i in range (9):
print (self.symbols[self.state['board'][i]+1], end=" ")
if ((i % 3) == 2):
print()
def move_generator(self):
moves = []
for i in range(9):
if self.state['board'][i] == 0:
p = self.state['on_move']
m = [p, i]
moves.append(m)
return moves
In [3]:
import random
class Agent:
def __init__(self, symbol):
self.symbol = symbol
def get_action(self, moves):
return random.choice(moves)
In [4]:
environment = TicTacToeEnv()
agent = Agent(symbol=1)
num_episodes = 100
collected_rewards = []
oom = 1
for i in range(num_episodes):
state, _ = environment.reset()
total_reward = 0
done = False
om = oom
for j in range(9):
moves = environment.move_generator()
if not moves:
break
if len(moves) == 1:
move = moves[0]
else:
move = agent.get_action(moves)
next_state, reward, done, info = environment.step(move)
total_reward += reward
state = next_state
environment.render()
if done:
break
om = -om
collected_rewards.append(total_reward)
print(f"Episode {i+1}, Total Reward: {total_reward}")
average_reward = sum(collected_rewards) / len(collected_rewards)
print(f"Average Reward: {average_reward}")
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O O
X
X
on move: O
O O
X
X X
on move: X
O O
X
X X O
on move: O
O O
X X
X X O
on move: X
O O O
X X
X X O
Episode 1, Total Reward: -1
Average Reward: -1.0
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
O X
X
O
on move: O
O X
X
X O
on move: X
O O X
X
X O
on move: O
O O X
X X
X O
on move: X
O O X
X X O
X O
on move: O
O O X
X X O
X X O
Episode 2, Total Reward: 1
Average Reward: 0.0
on move: O
X
on move: X
O X
on move: O
O X
X
on move: X
O X
O
X
on move: O
O X
O
X X
on move: X
O X
O O
X X
on move: O
O X
O O
X X X
Episode 3, Total Reward: 1
Average Reward: 0.3333333333333333
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
O O X
X X
on move: X
O O X
O
X X
on move: O
O O X
X O
X X
on move: X
O O X
X O O
X X
on move: O
O O X
X O O
X X X
Episode 4, Total Reward: 1
Average Reward: 0.5
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
X
O
X O
on move: O
X X
O
X O
on move: X
X X
O
X O O
on move: O
X X
X O
X O O
Episode 5, Total Reward: 1
Average Reward: 0.6
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
O X
X O
on move: O
O X
X
X O
on move: X
O X O
X
X O
on move: O
O X O
X X
X O
on move: X
O X O
X O X
X O
on move: O
O X O
X O X
X O X
Episode 6, Total Reward: 0
Average Reward: 0.5
on move: O
X
on move: X
X O
on move: O
X O X
on move: X
O
X O X
on move: O
O
X
X O X
on move: X
O
O X
X O X
on move: O
X O
O X
X O X
on move: X
X O O
O X
X O X
on move: O
X O O
O X X
X O X
Episode 7, Total Reward: 1
Average Reward: 0.5714285714285714
on move: O
X
on move: X
O
X
on move: O
X
O
X
on move: X
X
O
O X
on move: O
X X
O
O X
on move: X
X X O
O
O X
on move: O
X X O
O
O X X
on move: X
X X O
O O
O X X
Episode 8, Total Reward: -1
Average Reward: 0.375
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X
O O
on move: O
X
X X
O O
on move: X
X O
X X
O O
on move: O
X X O
X X
O O
on move: X
X X O
X X
O O O
Episode 9, Total Reward: -1
Average Reward: 0.2222222222222222
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X O
X
O
on move: O
X O
X
O X
on move: X
X O
O X
O X
on move: O
X O
O X
O X X
on move: X
X O O
O X
O X X
on move: O
X O O
O X X
O X X
Episode 10, Total Reward: 1
Average Reward: 0.3
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
X
X O O
on move: O
X
X
X O O
on move: X
O X
X
X O O
on move: O
O X
X X
X O O
on move: X
O X O
X X
X O O
on move: O
O X O
X X X
X O O
Episode 11, Total Reward: 1
Average Reward: 0.36363636363636365
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
O X
O
on move: O
X X
O X
O
on move: X
X X
O O X
O
on move: O
X X X
O O X
O
Episode 12, Total Reward: 1
Average Reward: 0.4166666666666667
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
X O
X
on move: O
O
X O
X X
on move: X
O
X O
O X X
Episode 13, Total Reward: -1
Average Reward: 0.3076923076923077
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O
O
X X
on move: O
X O
O
X X
on move: X
X O O
O
X X
on move: O
X O O
O X
X X
on move: X
X O O
O X
X X O
on move: O
X O O
X O X
X X O
Episode 14, Total Reward: 1
Average Reward: 0.35714285714285715
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
X
O X O
on move: O
X
O X O
X
Episode 15, Total Reward: 1
Average Reward: 0.4
on move: O
X
on move: X
X
O
on move: O
X X
O
on move: X
X X
O
O
on move: O
X X
O X
O
on move: X
X X
O X
O O
on move: O
X X X
O X
O O
Episode 16, Total Reward: 1
Average Reward: 0.4375
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
X O
X O
on move: O
X
X O
X O
on move: X
X
X O
X O O
on move: O
X
X O X
X O O
on move: X
O X
X O X
X O O
Episode 17, Total Reward: -1
Average Reward: 0.35294117647058826
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
X
O
X O
on move: O
X X
O
X O
on move: X
X X
O
X O O
on move: O
X X
X O
X O O
Episode 18, Total Reward: 1
Average Reward: 0.3888888888888889
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X
O O
on move: O
X X
X
O O
on move: X
X X
X
O O O
Episode 19, Total Reward: -1
Average Reward: 0.3157894736842105
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
X
O X
on move: O
X O
X
O X
on move: X
X O
O X
O X
on move: O
X O
O X
O X X
on move: X
X O O
O X
O X X
on move: O
X O O
O X X
O X X
Episode 20, Total Reward: 1
Average Reward: 0.35
on move: O
X
on move: X
O X
on move: O
O X X
on move: X
O
O X X
on move: O
X O
O X X
on move: X
O X O
O X X
on move: O
O X O
O X X
X
on move: X
O X O
O X X
O X
on move: O
O X O
O X X
X O X
Episode 21, Total Reward: 0
Average Reward: 0.3333333333333333
on move: O
X
on move: X
O
X
on move: O
O X
X
on move: X
O X
X
O
on move: O
O X
X X
O
on move: X
O X
X X
O O
on move: O
O X
X X X
O O
Episode 22, Total Reward: 1
Average Reward: 0.36363636363636365
on move: O
X
on move: X
X
O
on move: O
X
O
X
on move: X
O X
O
X
on move: O
O X
O
X X
on move: X
O X
O O
X X
on move: O
O X X
O O
X X
on move: X
O X X
O O O
X X
Episode 23, Total Reward: -1
Average Reward: 0.30434782608695654
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
O X
X O
on move: O
X
O X
X O
on move: X
X O
O X
X O
on move: O
X O
O X
X X O
on move: X
X O O
O X
X X O
on move: O
X O O
X O X
X X O
Episode 24, Total Reward: 1
Average Reward: 0.3333333333333333
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O
X X
O
on move: O
O
X X
O X
on move: X
O O
X X
O X
on move: O
O O X
X X
O X
on move: X
O O X
X X O
O X
on move: O
O O X
X X O
O X X
Episode 25, Total Reward: 0
Average Reward: 0.32
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
X O
X O
on move: O
X X O
X O
on move: X
O
X X O
X O
on move: O
O X
X X O
X O
Episode 26, Total Reward: 1
Average Reward: 0.34615384615384615
on move: O
X
on move: X
O
X
on move: O
X
O
X
on move: X
X
O
O X
on move: O
X
O
O X X
on move: X
X
O O
O X X
on move: O
X X
O O
O X X
on move: X
X X O
O O
O X X
Episode 27, Total Reward: -1
Average Reward: 0.2962962962962963
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X
O O
on move: O
X
X
O X O
on move: X
X
O X
O X O
on move: O
X X
O X
O X O
on move: X
X O X
O X
O X O
on move: O
X O X
O X X
O X O
Episode 28, Total Reward: 0
Average Reward: 0.2857142857142857
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
X O
X O
on move: O
X O
X O
X
on move: X
X O
X O O
X
on move: O
X O
X O O
X X
Episode 29, Total Reward: 1
Average Reward: 0.3103448275862069
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O
X O X
on move: O
O
X O X
X
on move: X
O O
X O X
X
on move: O
O O
X O X
X X
on move: X
O O O
X O X
X X
Episode 30, Total Reward: -1
Average Reward: 0.26666666666666666
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
X
X O
O
on move: O
X
X O
X O
Episode 31, Total Reward: 1
Average Reward: 0.2903225806451613
on move: O
X
on move: X
O
X
on move: O
X
O
X
on move: X
O X
O
X
on move: O
X O X
O
X
on move: X
X O X
O
X O
on move: O
X O X
X O
X O
on move: X
X O X
X O
O X O
on move: O
X O X
X X O
O X O
Episode 32, Total Reward: 0
Average Reward: 0.28125
on move: O
X
on move: X
X
O
on move: O
X X
O
on move: X
X X
O O
on move: O
X X
X
O O
on move: X
X X
O X
O O
on move: O
X X
O X X
O O
on move: X
X X
O X X
O O O
Episode 33, Total Reward: -1
Average Reward: 0.24242424242424243
on move: O
X
on move: X
O
X
on move: O
X
O
X
on move: X
X
O O
X
on move: O
X
O O
X X
on move: X
O X
O O
X X
on move: O
O X
O O
X X X
Episode 34, Total Reward: 1
Average Reward: 0.2647058823529412
on move: O
X
on move: X
O
X
on move: O
X O
X
on move: X
X O
O X
on move: O
X O
O X X
on move: X
O
X O
O X X
on move: O
X O
X O
O X X
Episode 35, Total Reward: 1
Average Reward: 0.2857142857142857
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
O O
X X X
Episode 36, Total Reward: 1
Average Reward: 0.3055555555555556
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
X
X O
O
on move: O
X
X X O
O
on move: X
X
X X O
O O
on move: O
X X
X X O
O O
on move: X
X X
X X O
O O O
Episode 37, Total Reward: -1
Average Reward: 0.2702702702702703
on move: O
X
on move: X
O
X
on move: O
O X
X
on move: X
O X
X O
on move: O
O X X
X O
on move: X
O
O X X
X O
on move: O
O X
O X X
X O
Episode 38, Total Reward: 1
Average Reward: 0.2894736842105263
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
X
O
X O
on move: O
X X
O
X O
on move: X
X X O
O
X O
on move: O
X X O
O
X O X
on move: X
X X O
O O
X O X
on move: O
X X O
X O O
X O X
Episode 39, Total Reward: 1
Average Reward: 0.3076923076923077
on move: O
X
on move: X
X
O
on move: O
X
O X
on move: X
O
X
O X
on move: O
O
X
X O X
on move: X
O O
X
X O X
on move: O
O X O
X
X O X
on move: X
O X O
X O
X O X
on move: O
O X O
X X O
X O X
Episode 40, Total Reward: 0
Average Reward: 0.3
on move: O
X
on move: X
O
X
on move: O
X
O
X
on move: X
X
O
O X
on move: O
X
X O
O X
on move: X
X
X O
O O X
on move: O
X X
X O
O O X
on move: X
O X X
X O
O O X
on move: O
O X X
X X O
O O X
Episode 41, Total Reward: 0
Average Reward: 0.2926829268292683
on move: O
X
on move: X
X
O
on move: O
X X
O
on move: X
X X
O
O
on move: O
X X X
O
O
Episode 42, Total Reward: 1
Average Reward: 0.30952380952380953
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
X
O
X O
on move: O
X X
O
X O
on move: X
X X O
O
X O
on move: O
X X O
O
X X O
on move: X
X X O
O O
X X O
Episode 43, Total Reward: -1
Average Reward: 0.27906976744186046
on move: O
X
on move: X
O X
on move: O
O X
X
on move: X
O X
X
O
on move: O
X O X
X
O
on move: X
X O X
X
O O
on move: O
X O X
X X
O O
on move: X
X O X
X O X
O O
Episode 44, Total Reward: -1
Average Reward: 0.25
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X O
O
on move: O
X
X O
O X
Episode 45, Total Reward: 1
Average Reward: 0.26666666666666666
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
X
O X O
on move: O
X
O X O
X
on move: X
X
O X O
O X
on move: O
X X
O X O
O X
Episode 46, Total Reward: 1
Average Reward: 0.2826086956521739
on move: O
X
on move: X
X
O
on move: O
X
O
X
on move: X
X
O O
X
on move: O
X X
O O
X
on move: X
X X
O O
O X
on move: O
X X
O O
X O X
on move: X
X X
O O O
X O X
Episode 47, Total Reward: -1
Average Reward: 0.2553191489361702
on move: O
X
on move: X
O X
on move: O
X
O X
on move: X
O
X
O X
on move: O
X O
X
O X
on move: X
X O
O X
O X
on move: O
X O
O X
X O X
on move: X
O X O
O X
X O X
on move: O
O X O
X O X
X O X
Episode 48, Total Reward: 0
Average Reward: 0.25
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O
X X
O
on move: O
O
X X
O X
on move: X
O O
X X
O X
on move: O
O O
X X
O X X
on move: X
O O
X X O
O X X
on move: O
O O X
X X O
O X X
Episode 49, Total Reward: 0
Average Reward: 0.24489795918367346
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
X
X O
on move: O
O
X X
X O
on move: X
O O
X X
X O
on move: O
O O
X X X
X O
Episode 50, Total Reward: 1
Average Reward: 0.26
on move: O
X
on move: X
X O
on move: O
X O
X
on move: X
X O
X O
on move: O
X
X O
X O
Episode 51, Total Reward: 1
Average Reward: 0.27450980392156865
on move: O
X
on move: X
X
O
on move: O
X X
O
on move: X
X X
O O
on move: O
X
X X
O O
on move: X
X
X X
O O O
Episode 52, Total Reward: -1
Average Reward: 0.25
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
X
O O
X X
on move: X
O X
O O
X X
on move: O
O X
O O
X X X
Episode 53, Total Reward: 1
Average Reward: 0.2641509433962264
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
O O
X X
X
on move: X
O O
X O X
X
on move: O
O O X
X O X
X
Episode 54, Total Reward: 1
Average Reward: 0.2777777777777778
on move: O
X
on move: X
X
O
on move: O
X
O
X
on move: X
X
O O
X
on move: O
X X
O O
X
on move: X
X X
O O
X O
on move: O
X X X
O O
X O
Episode 55, Total Reward: 1
Average Reward: 0.2909090909090909
on move: O
X
on move: X
O
X
on move: O
X O
X
on move: X
X O
X O
on move: O
X
X O
X O
on move: X
X O
X O
X O
on move: O
X X O
X O
X O
Episode 56, Total Reward: 1
Average Reward: 0.30357142857142855
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
O O X
X X
on move: X
O O X
X X
O
on move: O
O O X
X X X
O
Episode 57, Total Reward: 1
Average Reward: 0.3157894736842105
on move: O
X
on move: X
O
X
on move: O
X
O
X
on move: X
X
O
O X
on move: O
X
X O
O X
Episode 58, Total Reward: 1
Average Reward: 0.3275862068965517
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O O
X
X
on move: O
X O O
X
X
on move: X
X O O
O X
X
on move: O
X O O
O X
X X
Episode 59, Total Reward: 1
Average Reward: 0.3389830508474576
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
X
O O
X X
on move: X
X
O O
X O X
on move: O
X X
O O
X O X
on move: X
X X O
O O
X O X
on move: O
X X O
O X O
X O X
Episode 60, Total Reward: 1
Average Reward: 0.35
on move: O
X
on move: X
O X
on move: O
X
O X
on move: X
O X
O X
on move: O
O X
O X
X
on move: X
O O X
O X
X
on move: O
O O X
O X
X X
Episode 61, Total Reward: 1
Average Reward: 0.36065573770491804
on move: O
X
on move: X
X O
on move: O
X X O
on move: X
O
X X O
on move: O
O
X X O
X
on move: X
O O
X X O
X
on move: O
O O
X X O
X X
on move: X
O O O
X X O
X X
Episode 62, Total Reward: -1
Average Reward: 0.3387096774193548
on move: O
X
on move: X
O
X
on move: O
O X
X
on move: X
O X
O
X
on move: O
O X
O
X X
on move: X
O X
O O
X X
on move: O
O X
X O O
X X
on move: X
O O X
X O O
X X
on move: O
O O X
X O O
X X X
Episode 63, Total Reward: 1
Average Reward: 0.3492063492063492
on move: O
X
on move: X
O
X
on move: O
X O
X
on move: X
X O
X
O
on move: O
X O
X
O X
on move: X
X O
X O
O X
on move: O
X O
X O
X O X
on move: X
O X O
X O
X O X
on move: O
O X O
X O X
X O X
Episode 64, Total Reward: 0
Average Reward: 0.34375
on move: O
X
on move: X
X
O
on move: O
X
O X
on move: X
X
O X
O
on move: O
X
O X
X O
on move: X
O X
O X
X O
on move: O
O X
O X
X X O
on move: X
O O X
O X
X X O
Episode 65, Total Reward: -1
Average Reward: 0.3230769230769231
on move: O
X
on move: X
O
X
on move: O
X
O
X
on move: X
X
O
X O
on move: O
X
O X
X O
on move: X
X
O X
X O O
on move: O
X X
O X
X O O
on move: X
X O X
O X
X O O
Episode 66, Total Reward: -1
Average Reward: 0.30303030303030304
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
O X
X
on move: O
O
O X
X X
on move: X
O O
O X
X X
on move: O
O O
O X
X X X
Episode 67, Total Reward: 1
Average Reward: 0.31343283582089554
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
X O
X
on move: O
O X
X O
X
on move: X
O X
O X O
X
on move: O
O X
O X O
X X
Episode 68, Total Reward: 1
Average Reward: 0.3235294117647059
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
O X
X
on move: O
O
X O X
X
on move: X
O
X O X
X O
on move: O
O
X O X
X O X
on move: X
O O
X O X
X O X
Episode 69, Total Reward: -1
Average Reward: 0.30434782608695654
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X
O O
on move: O
X X
X
O O
on move: X
X X
O X
O O
on move: O
X X X
O X
O O
Episode 70, Total Reward: 1
Average Reward: 0.3142857142857143
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O O
X
X
on move: O
O O
X X
X
on move: X
O O
X X
O X
on move: O
O O
X X
O X X
on move: X
O O O
X X
O X X
Episode 71, Total Reward: -1
Average Reward: 0.29577464788732394
on move: O
X
on move: X
X
O
on move: O
X
O
X
on move: X
X
O O
X
on move: O
X
O O
X X
on move: X
X
O O
X X O
on move: O
X X
O O
X X O
on move: X
X O X
O O
X X O
on move: O
X O X
X O O
X X O
Episode 72, Total Reward: 1
Average Reward: 0.3055555555555556
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
X
X O
on move: O
O
X
X X O
on move: X
O
O X
X X O
on move: O
O X
O X
X X O
on move: X
O O X
O X
X X O
on move: O
O O X
O X X
X X O
Episode 73, Total Reward: 1
Average Reward: 0.3150684931506849
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
O
X
X O
on move: O
O X
X
X O
on move: X
O X
X
X O O
on move: O
O X X
X
X O O
on move: X
O X X
X O
X O O
Episode 74, Total Reward: -1
Average Reward: 0.2972972972972973
on move: O
X
on move: X
O X
on move: O
O X
X
on move: X
O X
X
O
on move: O
O X
X X
O
on move: X
O X
X X
O O
on move: O
X O X
X X
O O
on move: X
X O X
X O X
O O
Episode 75, Total Reward: -1
Average Reward: 0.28
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
X
O
X O
on move: O
X X
O
X O
on move: X
X X
O O
X O
on move: O
X X X
O O
X O
Episode 76, Total Reward: 1
Average Reward: 0.2894736842105263
on move: O
X
on move: X
O
X
on move: O
X O
X
on move: X
X O
O X
on move: O
X O
O X
X
on move: X
X O
O O X
X
on move: O
X O X
O O X
X
on move: X
X O X
O O X
X O
Episode 77, Total Reward: -1
Average Reward: 0.2727272727272727
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
O O
X
X X
on move: X
O O
X
O X X
on move: O
O O X
X
O X X
Episode 78, Total Reward: 1
Average Reward: 0.28205128205128205
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
O O
X
X X
on move: X
O O
X
X X O
on move: O
O O X
X
X X O
on move: X
O O X
X O
X X O
on move: O
O O X
X X O
X X O
Episode 79, Total Reward: 1
Average Reward: 0.2911392405063291
on move: O
X
on move: X
X
O
on move: O
X X
O
on move: X
O
X X
O
on move: O
X O
X X
O
on move: X
X O O
X X
O
on move: O
X O O
X X
X O
Episode 80, Total Reward: 1
Average Reward: 0.3
on move: O
X
on move: X
X
O
on move: O
X X
O
on move: X
O
X X
O
on move: O
O
X X
X O
on move: X
O O
X X
X O
on move: O
O O X
X X
X O
Episode 81, Total Reward: 1
Average Reward: 0.30864197530864196
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X O
O
on move: O
X X
X O
O
on move: X
X X
X O
O O
on move: O
X X
X O
O X O
on move: X
X X
X O O
O X O
on move: O
X X X
X O O
O X O
Episode 82, Total Reward: 1
Average Reward: 0.3170731707317073
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
O X
X
O
on move: O
O X
X
O X
on move: X
O X
X O
O X
on move: O
O X
X O
X O X
on move: X
O O X
X O
X O X
on move: O
O O X
X X O
X O X
Episode 83, Total Reward: 1
Average Reward: 0.3253012048192771
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O
O X X
on move: O
O
O X X
X
on move: X
O
O X X
X O
on move: O
O X
O X X
X O
on move: X
O X
O X X
O X O
Episode 84, Total Reward: -1
Average Reward: 0.30952380952380953
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O
X
X O
on move: O
O X
X
X O
on move: X
O O X
X
X O
on move: O
O O X
X
X X O
on move: X
O O X
O X
X X O
Episode 85, Total Reward: -1
Average Reward: 0.29411764705882354
on move: O
X
on move: X
X
O
on move: O
X
X O
on move: X
O
X
X O
on move: O
O X
X
X O
on move: X
O X
X O
X O
on move: O
X O X
X O
X O
Episode 86, Total Reward: 1
Average Reward: 0.3023255813953488
on move: O
X
on move: X
X
O
on move: O
X X
O
on move: X
X X O
O
on move: O
X X O
O
X
on move: X
X X O
O O
X
on move: O
X X O
O O X
X
on move: X
X X O
O O X
X O
on move: O
X X O
O O X
X X O
Episode 87, Total Reward: 0
Average Reward: 0.2988505747126437
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O
X X
O
on move: O
O
X X
O X
on move: X
O
O X X
O X
Episode 88, Total Reward: -1
Average Reward: 0.2840909090909091
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X
O O
on move: O
X
X X
O O
on move: X
O X
X X
O O
on move: O
O X
X X X
O O
Episode 89, Total Reward: 1
Average Reward: 0.29213483146067415
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O O
X X
on move: O
O O
X X
X
on move: X
O O
X X
O X
on move: O
O O
X X X
O X
Episode 90, Total Reward: 1
Average Reward: 0.3
on move: O
X
on move: X
O
X
on move: O
O
X X
on move: X
O
X X
O
on move: O
O
X X
O X
on move: X
O
X X
O X O
on move: O
X O
X X
O X O
on move: X
X O
X O X
O X O
Episode 91, Total Reward: -1
Average Reward: 0.2857142857142857
on move: O
X
on move: X
O X
on move: O
X
O X
on move: X
X
O X
O
on move: O
X
X O X
O
on move: X
X
X O X
O O
on move: O
X X
X O X
O O
on move: X
X X
X O X
O O O
Episode 92, Total Reward: -1
Average Reward: 0.2717391304347826
on move: O
X
on move: X
O X
on move: O
X
O X
on move: X
X
O
O X
on move: O
X X
O
O X
on move: X
X X
O
O X O
on move: O
X X
X O
O X O
on move: X
X X
X O O
O X O
on move: O
X X X
X O O
O X O
Episode 93, Total Reward: 1
Average Reward: 0.27956989247311825
on move: O
X
on move: X
O X
on move: O
O X X
on move: X
O X X
O
on move: O
X
O X X
O
on move: X
O X
O X X
O
on move: O
X O X
O X X
O
on move: X
X O X
O X X
O O
on move: O
X O X
O X X
O X O
Episode 94, Total Reward: 0
Average Reward: 0.2765957446808511
on move: O
X
on move: X
X
O
on move: O
X
O X
on move: X
O X
O X
on move: O
O X
X
O X
Episode 95, Total Reward: 1
Average Reward: 0.28421052631578947
on move: O
X
on move: X
X O
on move: O
X
X O
on move: X
X
X O
O
on move: O
X X
X O
O
on move: X
X X O
X O
O
on move: O
X X O
X O
O X
Episode 96, Total Reward: 1
Average Reward: 0.2916666666666667
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
X
X
O O
on move: O
X X
X
O O
on move: X
X X
X O
O O
on move: O
X X
X O X
O O
on move: X
X X
X O X
O O O
Episode 97, Total Reward: -1
Average Reward: 0.27835051546391754
on move: O
X
on move: X
X
O
on move: O
X
X
O
on move: X
O X
X
O
on move: O
O X
X X
O
on move: X
O X
X X
O O
on move: O
O X
X X X
O O
Episode 98, Total Reward: 1
Average Reward: 0.2857142857142857
on move: O
X
on move: X
O
X
on move: O
O
X
X
on move: X
O O
X
X
on move: O
O O
X X
X
on move: X
O O O
X X
X
Episode 99, Total Reward: -1
Average Reward: 0.2727272727272727
on move: O
X
on move: X
X
O
on move: O
X
O
X
on move: X
X
O
O X
on move: O
X X
O
O X
on move: X
X O X
O
O X
on move: O
X O X
O X
O X
on move: X
X O X
O X
O X O
on move: O
X O X
X O X
O X O
Episode 100, Total Reward: 0
Average Reward: 0.27