(六)Value Function Approximation-LSPI code (1)

  1 # -*- coding: utf-8 -*-
  2 """Abstract Base Class for Basis Function and some common implementations."""
  3 
  4 import abc
  5 
  6 import numpy as np
  7 
  8 
  9 class BasisFunction(object):
 10 
 11     r"""ABC for basis functions used by LSPI Policies.
 12 
 13     A basis function is a function that takes in a state vector and an action
 14     index and returns a vector of features. The resulting feature vector is
 15     referred to as :math:`\phi` in the LSPI paper (pg 9 of the PDF referenced
 16     in this package‘s documentation). The :math:`\phi` vector is dotted with
 17     the weight vector of the Policy to calculate the Q-value.
 18 
 19     The dimensions of the state vector are usually smaller than the dimensions
 20     of the :math:`\phi` vector. However, the dimensions of the :math:`\phi`
 21     vector are usually much smaller than the dimensions of an exact
 22     representation of the state which leads to significant savings when
 23     computing and storing a policy.
 24 
 25     """
 26 　　＃该函数输入状态向量，和动作，返回特征向量（feature vector）
　　　　＃也就是说该模块是进行系统状态更新的？也可能是不更新，直接就是通过状态计算特征向量
　　　　＃特征向量和权重矩阵点成得到了Ｑ值，通过点乘得到说明是通过线性方程对Ｑ(S,A)Ｊ进行近似

 27     __metaclass__ = abc.ABCMeta　＃构建类的方法
 28 
 29     @abc.abstractmethod　＃ｐｙｔｈｏｎ的＠修饰符，说明要在子类中必须具体实现，不然报错，这似乎是ａｂｃ库的常规用法
 30     def size(self):　＃返回phi也就是特征向量的长度
 31         r"""Return the vector size of the basis function.
 32 
 33         Returns
 34         -------
 35         int
 36             The size of the :math:`\phi` vector.
 37             (Referred to as k in the paper).
 38 
 39         """
 40         pass  # pragma: no cover
 41 
 42     @abc.abstractmethod
 43     def evaluate(self, state, action):　＃该函数通过状态动作对儿计算phi矩阵，完全依赖于子类的实现方法
 44         r"""Calculate the :math:`\phi` matrix for the given state-action pair.
 45 
 46         The way this value is calculated depends entirely on the concrete
 47         implementation of BasisFunction.
 48 
 49         Parameters
 50         ----------
 51         state : numpy.array
 52             The state to get the features for.
 53             When calculating Q(s, a) this is the s.
 54         action : int
 55             The action index to get the features for.
 56             When calculating Q(s, a) this is the a.
 57 
 58 
 59         Returns
 60         -------
 61         numpy.array
 62             The :math:`\phi` vector. Used by Policy to compute Q-value.
 63 
 64         """
 65         pass  # pragma: no cover
 66 
 67     @abc.abstractproperty　＃修饰符，必须实现？
 68     def num_actions(self):　＃返回可能的动作action的数目
 69         """Return number of possible actions.
 70 
 71         Returns
 72         -------
 73         int
 74             Number of possible actions.
 75         """
 76         pass  # pragma: no cover
 77 
 78     @staticmethod　＃静态方法
 79     def _validate_num_actions(num_actions):　＃确认动作的数目是否满足要求
 80         """Return num_actions if valid. Otherwise raise ValueError.
 81 
 82         Return
 83         ------
 84         int
 85             Number of possible actions.
 86 
 87         Raises
 88         ------
 89         ValueError
 90             If num_actions < 1
 91 
 92         """
 93         if num_actions < 1:
 94             raise ValueError(‘num_actions must be >= 1‘)　＃动作数目一定要大于１
 95         return num_actions
 96 
 97 
 98 class FakeBasis(BasisFunction):　＃基于上面的类构建一个新类
 99 
100     r"""Basis that ignores all input. Useful for random sampling.
101 
102     When creating a purely random Policy a basis function is still required.
103     This basis function just returns a :math:`\phi` equal to [1.] for all
104     inputs. It will however, still throw exceptions for impossible values like
105     negative action indexes.
106 
107     """
108 　　＃产生随机的策略时返回的特征向量phi只有一个值？就是１？
109     def __init__(self, num_actions):＃在初始化的时候调用父函数对动作数目进行验证
110         """Initialize FakeBasis."""
111         self.__num_actions = BasisFunction._validate_num_actions(num_actions)
112 
113     def size(self):＃返回特征向量的长度为１
114         r"""Return size of 1.
115 
116         Returns
117         -------
118         int
119             Size of :math:`phi` which is always 1 for FakeBasis
120 
121         Example
122         -------
123 
124         >>> FakeBasis().size()
125         1
126 
127         """
128         return 1
129 
130     def evaluate(self, state, action):　＃返回特征向量１
131         r"""Return :math:`\phi` equal to [1.].
132 
133         Parameters　＃参数
134         ----------
135         state : numpy.array＃状态
136             The state to get the features for.
137             When calculating Q(s, a) this is the s. FakeBasis ignores these
138             values.
139         action : int＃所采取的动作
140             The action index to get the features for.
141             When calculating Q(s, a) this is the a. FakeBasis ignores these
142             values.
143 
144         Returns
145         -------
146         numpy.array＃返回phi矩阵
147             :math:`\phi` vector equal to [1.].
148 
149         Raises
150         ------
151         IndexError
152             If action index is < 0
153 
154         Example
155         -------
156 
157         >>> FakeBasis().evaluate(np.arange(10), 0)
158         array([ 1.])
159 
160         """
161         if action < 0:
162             raise IndexError(‘action index must be >= 0‘)
163         if action >= self.num_actions:
164             raise IndexError(‘action must be < num_actions‘)
165         return np.array([1.])＃此处返回的phi矩阵就是１
166 
167     @property
168     def num_actions(self):＃返回动作的数目
169         """Return number of possible actions."""
170         return self.__num_actions
171 
172     @num_actions.setter　＃该修饰符表示将num_action转化为一个属性而不是一个方法
173     def num_actions(self, value):
174         """Set the number of possible actions.
175 
176         Parameters
177         ----------
178         value: int
179             Number of possible actions. Must be >= 1.
180 
181         Raises
182         ------
183         ValueError
184             If value < 1.
185 
186         """
187         if value < 1:
188             raise ValueError(‘num_actions must be at least 1.‘)
189         self.__num_actions = value
190 
191 
192 class OneDimensionalPolynomialBasis(BasisFunction):　＃一维多项式基
193 
194     """Polynomial features for a state with one dimension.
195 　　＃一维的多项式特征
196     Takes the value of the state and constructs a vector proportional
197     to the specified degree and number of actions. The polynomial is first
198     constructed as [..., 1, value, value^2, ..., value^k, ...]＃这里ｋ是自由度的数目
199     where k is the degree. The rest of the vector is 0.
200 　　＃输入是状态的值构建一个和动作数目成比例的向量，构建的方法如上，就是一个多项式
201     Parameters
202     ----------
203     degree : int＃多项式的自由度
204         The polynomial degree.
205     num_actions: int　＃动作的数量
206         The total number of possible actions
207 
208     Raises
209     ------
210     ValueError
211         If degree is less than 0
212     ValueError
213         If num_actions is less than 1
214 
215     """
216 
217     def __init__(self, degree, num_actions):＃初始化，输入量：自由度，动作数目
218         """Initialize polynomial basis function."""
219         self.__num_actions = BasisFunction._validate_num_actions(num_actions)＃通过父类检测函数检测动作数目
220 
221         if degree < 0:＃检测自由度数目
222             raise ValueError(‘Degree must be >= 0‘)
223         self.degree = degree
224 
225     def size(self):＃返回基函数（特征函数的）长度
226         """Calculate the size of the basis function.
227 　　　　
228         The base size will be degree + 1. This basic matrix is then
229         duplicated once for every action. Therefore the size is equal to
230         (degree + 1) * number of actions
231 　　　　＃基的长度是自由度长度＋１（因为由零次方）
　　　　　　＃对于每一个action都复制一个基向量，因此，总长度为：(degree + 1) * number of action

232 
233         Returns＃返回
234         -------
235         int　＃基矩阵（基向量／ｐｈｉ）的长度，是一个整数
236             The size of the phi matrix that will be returned from evaluate.
237 
238 
239         Example＃例子
240         -------
241 
242         >>> basis = OneDimensionalPolynomialBasis(2, 2)＃基是一维多项式，自由度２，动作数２
243         >>> basis.size()＃总长度：３＊２
244         6
245 
246         """
247         return (self.degree + 1) * self.num_actions
248 
249     def evaluate(self, state, action):＃评估函数
250         r"""Calculate :math:`\phi` matrix for given state action pair.
251 　　　　　　＃计算给出状态动作对儿后的ｐｈｉ矩阵
252         The :math:`\phi` matrix is used to calculate the Q function for the
253         given policy.
254 　　　　　　＃输出的phi矩阵用来计算Ｑ值
255         Parameters　＃输入参数
256         ----------
257         state : numpy.array＃状态
258             The state to get the features for.
259             When calculating Q(s, a) this is the s.
260         action : int　＃动作
261             The action index to get the features for.
262             When calculating Q(s, a) this is the a.
263 
264         Returns　＃返回
265         -------
266         numpy.array　＃phi
267             The :math:`\phi` vector. Used by Policy to compute Q-value.
268 
269         Raises　
270         ------＃报错的情况，对动作的数目检查，对状态向量的维度进行检查
271         IndexError
272             If :math:`0 \le action < num\_actions` then IndexError is raised.
273         ValueError
274             If the state vector has any number of dimensions other than 1 a
275             ValueError is raised.
276 
277         Example＃一个例子，没看懂
278         -------
279 
280         >>> basis = OneDimensionalPolynomialBasis(2, 2)
281         >>> basis.evaluate(np.array([2]), 0)
282         array([ 1.,  2.,  4.,  0.,  0.,  0.])
283 
284         """
285         if action < 0 or action >= self.num_actions:　＃如果动作的序号不对，也就是不是合理的动作
286             raise IndexError(‘Action index out of bounds‘)＃报错
287 
288         if state.shape != (1, ):＃如果状态向量的维度不对
289             raise ValueError(‘This class only supports one dimensional states‘)＃报错
290 
291         phi = np.zeros((self.size(), ))＃初始化phi值
292 
293         offset = (self.size()/self.num_actions)*action　＃偏置：（自由度的数目＋１）＊动做的序号？这一步是跳转到对应的动作的那一组的序号
294 
295         value = state[0]＃状态向量的第一个值
296 
297         phi[offset:offset + self.degree + 1] = \更改对应动作那一组的phi值
298             np.array([pow(value, i) for i in range(self.degree+1)])更改成一维多项式向量
299 
300         return phi＃返回phi
301 
302     @property
303     def num_actions(self):＃返回动作的数目
304         """Return number of possible actions."""
305         return self.__num_actions
306 
307     @num_actions.setter
308     def num_actions(self, value):
309         """Set the number of possible actions.
310 
311         Parameters
312         ----------
313         value: int
314             Number of possible actions. Must be >= 1.
315 
316         Raises
317         ------
318         ValueError
319             If value < 1.
320 
321         """
322         if value < 1:
323             raise ValueError(‘num_actions must be at least 1.‘)
324         self.__num_actions = value
325 
326 
327 class RadialBasisFunction(BasisFunction):＃径向基方程
328 
329     r"""Gaussian Multidimensional Radial Basis Function (RBF).
330 　　　　＃高斯多维径向基函数
331     Given a set of k means :math:`(\mu_1 , \ldots, \mu_k)` produce a feature
332     vector :math:`(1, e^{-\gamma || s - \mu_1 ||^2}, \cdots,
333     e^{-\gamma || s - \mu_k ||^2})` where `s` is the state vector and
334     :math:`\gamma` is a free parameter. This vector will be padded with
335     0‘s on both sides proportional to the number of possible actions
336     specified.
337 　　　　＃给出一组均值，求出一组高斯径向基特征
338     Parameters＃输入
339     ----------
340     means: list(numpy.array)＃一组均值
341         List of numpy arrays representing :math:`(\mu_1, \ldots, \mu_k)`.
342         Each :math:`\mu` is a numpy array with dimensions matching the state
343         vector this basis function will be used with. If the dimensions of each
344         vector are not equal than an exception will be raised. If no means are
345         specified then a ValueError will be raised
346     gamma: float　＃超参数，大于０
347         Free parameter which controls the size/spread of the Gaussian "bumps".
348         This parameter is best selected via tuning through cross validation.
349         gamma must be > 0.
350     num_actions: int　＃动作数目
351         Number of actions. Must be in range [1, :math:`\infty`] otherwise
352         an exception will be raised.
353 
354     Raises　＃一些错误的定义
355     ------
356     ValueError
357         If means list is empty
358     ValueError
359         If dimensions of each mean vector do not match.
360     ValueError
361         If gamma is <= 0.
362     ValueError
363         If num_actions is less than 1.
364 
365     Note
366     ----
367 
368     The numpy arrays specifying the means are not copied.
369 
370     """
371 
372     def __init__(self, means, gamma, num_actions):＃初始化
373         """Initialize RBF instance."""
374         self.__num_actions = BasisFunction._validate_num_actions(num_actions)　＃检查动作的数目是否满足
375 
376         if len(means) == 0:＃检查均值向量的长度是否满足
377             raise ValueError(‘You must specify at least one mean‘)
378 
379         if reduce(RadialBasisFunction.__check_mean_size, means) is None:＃各种检查
380             raise ValueError(‘All mean vectors must have the same dimensions‘)
381 
382         self.means = means
383 
384         if gamma <= 0:
385             raise ValueError(‘gamma must be > 0‘)
386 
387         self.gamma = gamma
388 
389     @staticmethod
390     def __check_mean_size(left, right):＃检查ｍｅａｎ矩阵的维度问题
391         """Apply f if the value is not None.
392 
393         This method is meant to be used with reduce. It will return either the
394         right most numpy array or None if any of the array‘s had
395         differing sizes. I wanted to use a Maybe monad here,
396         but Python doesn‘t support that out of the box.
397 
398         Return
399         ------
400         None or numpy.array
401             None values will propogate through the reduce automatically.
402 
403         """
404         if left is None or right is None:
405             return None
406         else:
407             if left.shape != right.shape:
408                 return None
409         return right
410 
411     def size(self):
412         r"""Calculate size of the :math:`\phi` matrix.
413 
414         The size is equal to the number of means + 1 times the number of
415         number actions.
416 
417         Returns
418         -------
419         int
420             The size of the phi matrix that will be returned from evaluate.
421 
422         """
423         return (len(self.means) + 1) * self.num_actions＃为什么要在ｍｅａｎ矩阵的长度上加１？？，因为打头的是１！！
424 
425     def evaluate(self, state, action):＃计算phi
426         r"""Calculate the :math:`\phi` matrix.
427 
428         Matrix will have the following form:
　　　　　　　　＃矩阵会有如下形式
429 
430         :math:`[\cdots, 1, e^{-\gamma || s - \mu_1 ||^2}, \cdots,
431         e^{-\gamma || s - \mu_k ||^2}, \cdots]`
432 
433         where the matrix will be padded with 0‘s on either side depending
434         on the specified action index and the number of possible actions.
435 
436         Returns
437         -------
438         numpy.array＃返回phi
439             The :math:`\phi` vector. Used by Policy to compute Q-value.
440 
441         Raises
442         ------
443         IndexError＃一些错误的定义
444             If :math:`0 \le action < num\_actions` then IndexError is raised.
445         ValueError
446             If the state vector has any number of dimensions other than 1 a
447             ValueError is raised.
448 
449         """
450         if action < 0 or action >= self.num_actions:＃检查action的编号是否合格
451             raise IndexError(‘Action index out of bounds‘)
452 
453         if state.shape != self.means[0].shape:＃检查状态和均值矩阵的维度问题
454             raise ValueError(‘Dimensions of state must match ‘
455                              ‘dimensions of means‘)
456 
457         phi = np.zeros((self.size(), ))＃初始化
458         offset = (len(self.means[0])+1)*action＃确定action要更改的位置
459 
460         rbf = [RadialBasisFunction.__calc_basis_component(state,
461                                                           mean,
462                                                           self.gamma)
463                for mean in self.means]＃构建径向基向
464         phi[offset] = 1.＃开头的是１　　
465         phi[offset+1:offset+1+len(rbf)] = rbf　＃后面的就是刚才计算的径向基项
466 
467         return phi
468 
469     @staticmethod
470     def __calc_basis_component(state, mean, gamma):＃计算径向基项的方法
471         mean_diff = state - mean＃偏差
472         return np.exp(-gamma*np.sum(mean_diff*mean_diff))＃径向基计算公式
473 
474     @property
475     def num_actions(self):＃返回动作的数目
476         """Return number of possible actions."""
477         return self.__num_actions
478 
479     @num_actions.setter
480     def num_actions(self, value):
481         """Set the number of possible actions.
482 
483         Parameters
484         ----------
485         value: int
486             Number of possible actions. Must be >= 1.
487 
488         Raises
489         ------
490         ValueError
491             If value < 1.
492 
493         """
494         if value < 1:
495             raise ValueError(‘num_actions must be at least 1.‘)
496         self.__num_actions = value
497 
498 
499 class ExactBasis(BasisFunction):
500 
501     """Basis function with no functional approximation.
502 　　　　＃计算准确的状态基，没有经过方程近似的
503     This can only be used in domains with finite, discrete state-spaces. For
504     example the Chain domain from the LSPI paper would work with this basis,
505     but the inverted pendulum domain would not.
506 　　　　＃这种方法用在有有限离散状态的情况下，在比如倒立摆等问题中不适用
507     Parameters
508     ----------＃参数
509     num_states: list＃状态的数目，这个解释略奇怪，这个列表包含了一些整数，这些整数代表了每个状态变量可能的value的数目？什么鬼
510         A list containing integers representing the number of possible values
511         for each state variable.
512     num_actions: int＃动作的数目
513         Number of possible actions.
514     """
515 
516     def __init__(self, num_states, num_actions):初始化
517         """Initialize ExactBasis."""
518         if len(np.where(num_states <= 0)[0]) != 0:＃num_state必须大于０
519             raise ValueError(‘num_states value\‘s must be > 0‘)
520 
521         self.__num_actions = BasisFunction._validate_num_actions(num_actions)＃检查动作数目　　
522         self._num_states = num_states＃给自身变量_num_state赋值
523 
524         self._offsets = [1]＃偏差量，用来存放要更新的phi的位置
525         for i in range(1, len(num_states)):
526             self._offsets.append(self._offsets[-1]*num_states[i-1])＃由于可能的状态数目不一致，所以要记录每一次变化情况，后面求phi的时候可以用
527 
528     def size(self):
529         r"""Return the vector size of the basis function.
530 　　　　　　＃返回Phi的长度
531         Returns
532         -------
533         int
534             The size of the :math:`\phi` vector.
535             (Referred to as k in the paper).
536         """＃先计算可能的状态数目，用了ｌａｍｂｄａ函数，再乘以动作数目
537         return reduce(lambda x, y: x*y, self._num_states, 1)*self.__num_actions
538 
539     def get_state_action_index(self, state, action):
540         """Return the non-zero index of the basis.
541 　　　　　　＃获得state-action对儿对应的位置
542         Parameters
543         ----------＃输入
544         state: numpy.array＃状态
545             The state to get the index for.
546         action: int＃动作
547             The state to get the index for.
548 
549         Returns
550         -------
551         int
552             The non-zero index of the basis
553 
554         Raises
555         ------
556         IndexError
557             If action index < 0 or action index > num_actions
558         """
559         if action < 0:＃动作的编号的合理化检测
560             raise IndexError(‘action index must be >= 0‘)
561         if action >= self.num_actions:
562             raise IndexError(‘action must be < num_actions‘)
563 
564         base = action * int(self.size() / self.__num_actions)＃动作数乘以状态数
565 
566         offset = 0＃偏置置零
567         for i, value in enumerate(state):
568             offset += self._offsets[i] * state[i]
569 
570         return base + offset＃找到要更新的phi的起始位置
571 
572     def evaluate(self, state, action):
573         r"""Return a :math:`\phi` vector that has a single non-zero value.
574 
575         Parameters
576         ----------
577         state: numpy.array
578             The state to get the features for. When calculating Q(s, a) this is
579             the s.
580         action: int
581             The action index to get the features for.
582             When calculating Q(s, a) this is the a.
583 
584         Returns
585         -------
586         numpy.array
587             :math:`\phi` vector
588 
589         Raises
590         ------
591         IndexError
592             If action index < 0 or action index > num_actions
593         ValueError
594             If the size of the state does not match the the size of the
595             num_states list used during construction.
596         ValueError
597             If any of the state variables are < 0 or >= the corresponding
598             value in the num_states list used during construction.
599         """
600         if len(state) != len(self._num_states):＃各种检查是否合格
601             raise ValueError(‘Number of state variables must match ‘
602                              + ‘size of num_states.‘)
603         if len(np.where(state < 0)[0]) != 0:
604             raise ValueError(‘state cannot contain negative values.‘)
605         for state_var, num_state_values in zip(state, self._num_states):
606             if state_var >= num_state_values:
607                 raise ValueError(‘state values must be <= corresponding ‘
608                                  + ‘num_states value.‘)
609 
610         phi = np.zeros(self.size())＃初始化phi
611         phi[self.get_state_action_index(state, action)] = 1＃更新对应位置的phi值．
612 
613         return phi
614 
615     @property
616     def num_actions(self):
617         """Return number of possible actions."""
618         return self.__num_actions
619 
620     @num_actions.setter＃动作数目的计算函数
621     def num_actions(self, value):
622         """Set the number of possible actions.
623 
624         Parameters
625         ----------
626         value: int
627             Number of possible actions. Must be >= 1.
628 
629         Raises
630         ------
631         ValueError
632             if value < 1.
633         """
634         if value < 1:
635             raise ValueError(‘num_actions must be at least 1.‘)
636         self.__num_actions = value