我正在尝试从原始资源Andrej Karpathy Blog中重新创建非常简单的Policy Gradient示例。在该文章中，您将找到带有CartPole和Policy Gradient以及重量和Softmax激活列表的示例。这是我重新创建的非常简单的CartPole政策梯度示例，效果很好。
import gym
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
import copy
NUM_EPISODES = 4000
LEARNING_RATE = 0.000025
GAMMA = 0.99
# noinspection PyMethodMayBeStatic
class Agent:
def __init__(self):
self.poly = PolynomialFeatures(1)
self.w = np.random.rand(5, 2)
def policy(self, state):
z = state.dot(self.w)
exp = np.exp(z)
return exp/np.sum(exp)
def __softmax_grad(self, softmax):
s = softmax.reshape(-1,1)
return np.diagflat(s) - np.dot(s, s.T)
def grad(self, probs, action, state):
dsoftmax = self.__softmax_grad(probs)[action,:]
dlog = dsoftmax / probs[0,action]
grad = state.T.dot(dlog[None,:])
return grad
def update_with(self, grads, rewards):
for i in range(len(grads)):
# Loop through everything that happend in the episode
# and update towards the log policy gradient times **FUTURE** reward
total_grad_effect = 0
for t, r in enumerate(rewards[i:]):
total_grad_effect += r * (GAMMA ** r)
self.w += LEARNING_RATE * grads[i] * total_grad_effect
print("Grads update: " + str(np.sum(grads[i])))
def main(argv):
env = gym.make('CartPole-v0')
np.random.seed(1)
agent = Agent()
complete_scores = []
for e in range(NUM_EPISODES):
state = env.reset()[None, :]
state = agent.poly.fit_transform(state)
rewards = []
grads = []
score = 0
while True:
probs = agent.policy(state)
action_space = env.action_space.n
action = np.random.choice(action_space, p=probs[0])
next_state, reward, done,_ = env.step(action)
next_state = next_state[None,:]
next_state = agent.poly.fit_transform(next_state.reshape(1, 4))
grad = agent.grad(probs, action, state)
grads.append(grad)
rewards.append(reward)
score += reward
state = next_state
if done:
break
agent.update_with(grads, rewards)
complete_scores.append(score)
env.close()
plt.plot(np.arange(NUM_EPISODES),
complete_scores)
plt.savefig('image1.png')
if __name__ == '__main__':
main(None)
。
。
题
我正在尝试做几乎相同的示例，但使用了Sigmoid激活（仅出于简化目的）。这就是我要做的。将模型中的激活从切换softmax到sigmoid。哪个应该可以正常工作（基于以下说明）。但是我的Policy Gradient模型什么也不学，并且保持随机性。有什么建议吗？
import gym
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
NUM_EPISODES = 4000
LEARNING_RATE = 0.000025
GAMMA = 0.99
# noinspection PyMethodMayBeStatic
class Agent:
def __init__(self):
self.poly = PolynomialFeatures(1)
self.w = np.random.rand(5, 1) - 0.5
# Our policy that maps state to action parameterized by w
# noinspection PyShadowingNames
def policy(self, state):
z = np.sum(state.dot(self.w))
return self.sigmoid(z)
def sigmoid(self, x):
s = 1 / (1 + np.exp(-x))
return s
def sigmoid_grad(self, sig_x):
return sig_x * (1 - sig_x)
def grad(self, probs, action, state):
dsoftmax = self.sigmoid_grad(probs)
dlog = dsoftmax / probs
grad = state.T.dot(dlog)
grad = grad.reshape(5, 1)
return grad
def update_with(self, grads, rewards):
if len(grads) < 50:
return
for i in range(len(grads)):
# Loop through everything that happened in the episode
# and update towards the log policy gradient times **FUTURE** reward
total_grad_effect = 0
for t, r in enumerate(rewards[i:]):
total_grad_effect += r * (GAMMA ** r)
self.w += LEARNING_RATE * grads[i] * total_grad_effect
def main(argv):
env = gym.make('CartPole-v0')
np.random.seed(1)
agent = Agent()
complete_scores = []
for e in range(NUM_EPISODES):
state = env.reset()[None, :]
state = agent.poly.fit_transform(state)
rewards = []
grads = []
score = 0
while True:
probs = agent.policy(state)
action_space = env.action_space.n
action = np.random.choice(action_space, p=[1 - probs, probs])
next_state, reward, done, _ = env.step(action)
next_state = next_state[None, :]
next_state = agent.poly.fit_transform(next_state.reshape(1, 4))
grad = agent.grad(probs, action, state)
grads.append(grad)
rewards.append(reward)
score += reward
state = next_state
if done:
break
agent.update_with(grads, rewards)
complete_scores.append(score)
env.close()
plt.plot(np.arange(NUM_EPISODES),
complete_scores)
plt.savefig('image1.png')
if __name__ == '__main__':
main(None)绘制所有学习内容将保持随机性。调整超级参数没有任何帮助。在示例图像下方。
参考文献：
更新
似乎下面的答案可以从图形中完成一些工作。但这不是对数概率，甚至不是策略的梯度。并更改RL渐变政策的整个目的。请检查上面的参考。在图像之后，我们接下来声明。
我需要对“ 策略”的“对数”函数取一个渐变（简单来说就是权重和sigmoid激活）。
4
我建议您将此问题发布到Data Science Stack Exchange上，因为它主要是理论问题（Stack Overflow主要用于编码问题）。您还将接触到更多在该领域知识渊博的人。
—
吉尔斯·菲利普·派瑞
@Gilles-PhilippePaillé我添加了代表问题的代码。我需要做的只是通过激活来修复某些部分。请检查更新后的答案。
—
GensaGames
对于“派生策略梯度”，这是参考文章，其中包含相同类型的安排的工作示例，希望您能详细了解：medium.com/@thechrisyoon/…。
—
穆罕默德·乌斯曼
@MuhammadUsman。感谢您的信息。我把那个来源。现在，上面的例子很清楚，我正在尝试将激活从更改
—
GensaGames
softmax为signmoid。在上面的示例中，这只是我要做的一件事。
@JasonChia乙状结肠输出一定范围内的实数，
—
Pavel Tyshevskyi
[0, 1]该实数可以解释为采取积极行动的概率（例如，在CartPole中向右转）。则负动作的可能性（向左转）为1 - sigmoid。这个概率的总和是1。是的，这是一个标准的极卡环境。
