diff --git a/XCS_script.py b/XCS_script.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/XCS_script.py @@ -0,0 +1 @@ + diff --git a/lcs/agents/Agent.py b/lcs/agents/Agent.py index b70f39a3..7974fb0b 100644 --- a/lcs/agents/Agent.py +++ b/lcs/agents/Agent.py @@ -3,10 +3,7 @@ from timeit import default_timer as timer from typing import Callable, List, Tuple -import dill -import mlflow import numpy as np -import tempfile from lcs.metrics import basic_metrics @@ -119,8 +116,6 @@ def _evaluate(self, tuple population of classifiers and metrics """ - using_mlflow = hasattr(self.get_cfg(), 'use_mlflow') and self.get_cfg().use_mlflow - current_trial = 0 steps = 0 @@ -143,26 +138,6 @@ def _evaluate(self, metrics.append(m) - if using_mlflow: - mlflow.log_metrics(m, current_trial) - - # checkpoint model and metrics - if self.get_cfg().model_checkpoint_freq: - if current_trial % self.get_cfg().model_checkpoint_freq == 0: - prefix = f"-trial-{current_trial}" - with tempfile.TemporaryDirectory(prefix) as td: - logger.debug(f"checkpointing model to {td}") - pop_path = f"{td}/population.dill" - metrics_path = f"{td}/metrics.dill" - - dill.dump(self.get_population(), - open(pop_path, mode='wb')) - - dill.dump(metrics, open(metrics_path, mode='wb')) - - if using_mlflow: - mlflow.log_artifacts(td, f"{current_trial}/") - # Print last metric if current_trial % np.round(n_trials / 10) == 0: logger.info(metrics[-1]) diff --git a/lcs/agents/acs/Configuration.py b/lcs/agents/acs/Configuration.py index 88fde13b..9a9988d1 100644 --- a/lcs/agents/acs/Configuration.py +++ b/lcs/agents/acs/Configuration.py @@ -12,7 +12,6 @@ def __init__(self, user_metrics_collector_fcn: Callable = None, fitness_fcn=None, metrics_trial_frequency: int = 5, - model_checkpoint_frequency: int = None, do_subsumption: bool = True, beta: float = 0.05, # gamma: float = 0.95, @@ -21,8 +20,7 @@ def __init__(self, epsilon: float = 0.5, u_max: int = 100000, theta_exp: int = 20, - theta_as: int = 20, - use_mlflow: bool = False) -> None: + theta_as: int = 20) -> None: """ Creates the configuration object used during training the ACS2 agent. @@ -51,7 +49,6 @@ def __init__(self, self.classifier_wildcard = classifier_wildcard self.environment_adapter = environment_adapter self.metrics_trial_frequency = metrics_trial_frequency - self.model_checkpoint_freq = model_checkpoint_frequency self.user_metrics_collector_fcn = user_metrics_collector_fcn self.fitness_fcn = fitness_fcn self.do_subsumption = do_subsumption @@ -62,7 +59,6 @@ def __init__(self, self.epsilon = epsilon self.u_max = u_max self.theta_as = theta_as - self.use_mlflow = use_mlflow def __str__(self) -> str: return str(vars(self)) diff --git a/lcs/agents/acs2/Configuration.py b/lcs/agents/acs2/Configuration.py index 024d7623..526c0518 100644 --- a/lcs/agents/acs2/Configuration.py +++ b/lcs/agents/acs2/Configuration.py @@ -14,7 +14,6 @@ def __init__(self, user_metrics_collector_fcn: Callable = None, fitness_fcn=None, metrics_trial_frequency: int = 5, - model_checkpoint_frequency: int = None, do_pee: bool = False, do_ga: bool = False, do_subsumption: bool = True, @@ -33,8 +32,7 @@ def __init__(self, theta_ga: int = 100, theta_as: int = 20, mu: float = 0.3, - chi: float = 0.8, - use_mlflow: bool = False): + chi: float = 0.8): super(Configuration, self).__init__( classifier_length, @@ -44,7 +42,6 @@ def __init__(self, user_metrics_collector_fcn, fitness_fcn, metrics_trial_frequency, - model_checkpoint_frequency, do_subsumption, beta, theta_i, @@ -52,8 +49,7 @@ def __init__(self, epsilon, u_max, theta_exp, - theta_as, - use_mlflow) + theta_as) self.gamma = gamma self.do_pee = do_pee diff --git a/lcs/agents/xcs/Classifier.py b/lcs/agents/xcs/Classifier.py index 4b6031ec..1da0d2b9 100644 --- a/lcs/agents/xcs/Classifier.py +++ b/lcs/agents/xcs/Classifier.py @@ -12,10 +12,13 @@ def __init__(self, condition: Union[Condition, str, None] = None, action: Optional[int] = None, time_stamp: int = None) -> None: + if cfg is None: raise TypeError("Configuration should be passed to Classifier") + if type(condition) != Condition: - condition = str(condition) + condition = Condition(condition) + self.cfg = cfg # cfg self.condition = condition # current situation self.action = action # A - int action @@ -42,9 +45,7 @@ def does_subsume(self, other): @property def could_subsume(self): - if self.experience > self.cfg.subsumption_threshold and self.error < self.cfg.initial_error: - return True - return False + return self.experience > self.cfg.subsumption_threshold and self.error < self.cfg.initial_error def is_more_general(self, other): if self.wildcard_number <= other.wildcard_number: @@ -60,9 +61,10 @@ def __len__(self): def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - Num:{self.numerosity} " + \ - f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}]" + f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}, Error:{self.error}]" - def __eq__(self, other): - if other.action == self.action and other.condition == self.condition: - return True - return False + def __eq__(self, o): + return o.condition == self.condition and o.action == self.action + + def __hash__(self): + return hash((str(self.condition), self.action)) diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index d42bb814..e37d1fb4 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -1,10 +1,10 @@ -import numpy as np -import random import logging +import random + +import numpy as np from lcs import TypedList, Perception from lcs.agents.xcs import Classifier, Condition, Configuration - logger = logging.getLogger(__name__) @@ -18,11 +18,12 @@ def __init__(self, super().__init__(*args, oktypes=oktypes) def insert_in_population(self, cl: Classifier): - for c in self: - if c == cl: - c.numerosity += 1 - return - self.append(cl) + existing_classifiers = [c for c in self if c == cl] + if len(existing_classifiers) > 0: + assert len(existing_classifiers) == 1, 'duplicates found, while inserting' + existing_classifiers[0].numerosity += 1 + else: + self.append(cl) def generate_covering_classifier(self, situation, action, time_stamp): # both Perception and string has __getitem__ @@ -33,11 +34,11 @@ def generate_covering_classifier(self, situation, action, time_stamp): generalized.append(self.cfg.classifier_wildcard) else: generalized.append(situation[i]) - cl = Classifier(cfg=self.cfg, - condition=Condition(generalized), - action=action, - time_stamp=time_stamp) - return cl + + return Classifier(condition=Condition(generalized), + action=action, + time_stamp=time_stamp, + cfg=self.cfg) def _generate_covering_and_insert(self, situation, action, time_stamp): cl = self.generate_covering_classifier(situation, action, time_stamp) @@ -45,10 +46,11 @@ def _generate_covering_and_insert(self, situation, action, time_stamp): self.delete_from_population() return cl - # Roulette-Wheel Deletion - # TODO: use strategies def delete_from_population(self): - if self.numerosity > self.cfg.max_population: + # TODO: change while to if + # During woods the number of rules grew over max number + # To remedy this I added while instead of if. Correct issue should be changed. + while self.numerosity > self.cfg.max_population: average_fitness = sum(cl.fitness for cl in self) / self.numerosity deletion_votes = [] for cl in self: @@ -59,7 +61,7 @@ def delete_from_population(self): def _deletion_vote(self, cl, average_fitness): vote = cl.action_set_size * cl.numerosity if cl.experience > self.cfg.deletion_threshold and \ - cl.fitness / cl.numerosity < \ + cl.fitness / cl.numerosity < \ self.cfg.delta * average_fitness: vote *= average_fitness / (cl.fitness / cl.numerosity) return vote @@ -70,16 +72,18 @@ def _remove_based_on_votes(self, deletion_votes, selector): if selector <= 0: if cl.numerosity > 1: cl.numerosity -= 1 + return cl else: self.safe_remove(cl) - return None + return cl def generate_match_set(self, situation: Perception, time_stamp): matching_ls = [cl for cl in self if cl.does_match(situation)] - while len(matching_ls) < self.cfg.number_of_actions: - action = self._find_not_present_action(matching_ls) + action = self._find_not_present_action(matching_ls) + while action is not None: cl = self._generate_covering_and_insert(situation, action, time_stamp) matching_ls.append(cl) + action = self._find_not_present_action(matching_ls) return ClassifiersList(self.cfg, *matching_ls) def _find_not_present_action(self, matching_set): @@ -116,14 +120,18 @@ def update_set(self, p): for cl in self: cl.experience += 1 # update prediction, prediction error, action set size estimate - if cl.experience < 1/self.cfg.learning_rate: - cl.prediction += (p - cl.prediction) / cl.experience - cl.error += (abs(p - cl.prediction) - cl.error) / cl.experience - cl.action_set_size +=\ + if cl.experience < 1 / self.cfg.learning_rate: + cl.prediction += \ + (p - cl.prediction) / cl.experience + cl.error += \ + (abs(p - cl.prediction) - cl.error) / cl.experience + cl.action_set_size += \ (action_set_numerosity - cl.action_set_size) / cl.experience else: - cl.prediction += self.cfg.learning_rate * (p - cl.prediction) - cl.error += self.cfg.learning_rate * (abs(p - cl.prediction) - cl.error) + cl.prediction +=\ + self.cfg.learning_rate * (p - cl.prediction) + cl.error += \ + self.cfg.learning_rate * (abs(p - cl.prediction) - cl.error) cl.action_set_size += \ self.cfg.learning_rate * (action_set_numerosity - cl.action_set_size) self._update_fitness() @@ -135,14 +143,13 @@ def _update_fitness(self): if cl.error < self.cfg.epsilon_0: tmp_acc = 1 else: - tmp_acc = (self.cfg.alpha * - (cl.error * self.cfg.epsilon_0) ** - -self.cfg.v - ) + tmp_acc = (pow(self.cfg.alpha * (cl.error * self.cfg.epsilon_0), - self.cfg.v)) accuracy_vector_k.append(tmp_acc) - accuracy_sum += tmp_acc + cl.numerosity + accuracy_sum += tmp_acc * cl.numerosity for cl, k in zip(self, accuracy_vector_k): cl.fitness += ( self.cfg.learning_rate * (k * cl.numerosity / accuracy_sum - cl.fitness) ) + + diff --git a/lcs/agents/xcs/Condition.py b/lcs/agents/xcs/Condition.py index 7f489429..d5868571 100644 --- a/lcs/agents/xcs/Condition.py +++ b/lcs/agents/xcs/Condition.py @@ -1,5 +1,4 @@ from __future__ import annotations - from .. import ImmutableSequence diff --git a/lcs/agents/xcs/Configuration.py b/lcs/agents/xcs/Configuration.py index 36e41534..89fb9925 100644 --- a/lcs/agents/xcs/Configuration.py +++ b/lcs/agents/xcs/Configuration.py @@ -29,7 +29,8 @@ def __init__(self, do_ga_subsumption: bool = False, do_action_set_subsumption: bool = False, metrics_trial_frequency: int = 5, - user_metrics_collector_fcn: Callable = None + user_metrics_collector_fcn: Callable = None, + multistep_enfiroment: bool = True ) -> None: """ :param classifier_wildcard: Wildcard symbol @@ -76,9 +77,9 @@ def __init__(self, self.number_of_actions = number_of_actions self.do_GA_subsumption = do_ga_subsumption self.do_action_set_subsumption = do_action_set_subsumption - self.metrics_trial_frequency = metrics_trial_frequency self.user_metrics_collector_fcn = user_metrics_collector_fcn + self.multistep_enfiroment = multistep_enfiroment def __str__(self) -> str: return str(vars(self)) diff --git a/lcs/agents/xcs/GeneticAlgorithm.py b/lcs/agents/xcs/GeneticAlgorithm.py index ead1bf2b..e365db64 100644 --- a/lcs/agents/xcs/GeneticAlgorithm.py +++ b/lcs/agents/xcs/GeneticAlgorithm.py @@ -1,134 +1,140 @@ -import numpy as np import random from copy import copy +import numpy as np from lcs.agents.xcs import Configuration, Classifier, ClassifiersList -def run_ga(population: ClassifiersList, - action_set: ClassifiersList, - situation, - time_stamp, - cfg: Configuration): - if action_set is None: - return +class GeneticAlgorithm: + + def __init__( + self, + population: ClassifiersList, + cfg: Configuration + ): + self.population = population + self.cfg = cfg + + def run_ga(self, + action_set: ClassifiersList, + situation, + time_stamp): + + if action_set is None: + return + # sometimes action set is empty, which is expected + assert isinstance(action_set, ClassifiersList) + if time_stamp - (sum(cl.time_stamp * cl.numerosity for cl in action_set) + / (sum(cl.numerosity for cl in action_set) or 1)) > self.cfg.ga_threshold: + for cl in action_set: + cl.time_stamp = time_stamp + parent1 = self._select_offspring(action_set) + parent2 = self._select_offspring(action_set) + child1, child2 = self._make_children(parent1, parent2, time_stamp) + if np.random.rand() < self.cfg.chi: + self._apply_crossover(child1, child2, parent1, parent2) + self._apply_mutation(child1, self.cfg, situation) + self._apply_mutation(child2, self.cfg, situation) + self._perform_insertion_or_subsumption( + child1, child2, + parent1, parent2 + ) + + def _perform_insertion_or_subsumption(self, + child1: Classifier, child2: Classifier, + parent1: Classifier, parent2: Classifier): + + assert isinstance(child1, Classifier) + assert isinstance(child2, Classifier) + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) + + if self.cfg.do_GA_subsumption: + for child in child1, child2: + if parent1.does_subsume(child): + parent1.numerosity += 1 + elif parent2.does_subsume(child): + parent2.numerosity += 1 + else: + self.population.insert_in_population(child) + self.population.delete_from_population() + else: + for child in child1, child2: + self.population.insert_in_population(child) + self.population.delete_from_population() - # temp_numerosity = sum(cl.numerosity for cl in action_set) - # if temp_numerosity == 0: - # return + def _make_children(self, parent1, parent2, time_stamp): + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) - if time_stamp - (sum(cl.time_stamp * cl.numerosity for cl in action_set) - # / temp_numerosity) > cfg.ga_threshold: - / (sum(cl.numerosity for cl in action_set) or 1)) > cfg.ga_threshold: - for cl in action_set: - cl.time_stamp = time_stamp - # select children - parent1 = _select_offspring(action_set) - parent2 = _select_offspring(action_set) - child1, child2 = _make_children(parent1, parent2) - # apply crossover - if np.random.rand() < cfg.chi: - _apply_crossover(child1, child2, parent1, parent2) - # apply mutation on both children - _apply_mutation(child1, cfg, situation) - _apply_mutation(child2, cfg, situation) - # apply subsumption or just insert into population - _perform_insertion_or_subsumption(cfg, population, - child1, child2, - parent1, parent2) - - -def _perform_insertion_or_subsumption(cfg: Configuration, population: ClassifiersList, - child1: Classifier, child2: Classifier, - parent1: Classifier, parent2: Classifier): - if child1 is None or child2 is None: - return - if cfg.do_GA_subsumption: - if parent1.does_subsume(child1): - parent1.numerosity += 1 - elif parent2.does_subsume(child1): - parent2.numerosity += 1 - else: - population.insert_in_population(child1) - population.delete_from_population() + child1 = copy(parent1) + child1.condition = copy(parent1.condition) + child1.time_stamp = time_stamp - if parent1.does_subsume(child2): - parent1.numerosity += 1 - elif parent2.does_subsume(child2): - parent2.numerosity += 1 - else: - population.insert_in_population(child2) - population.delete_from_population() - else: - population.insert_in_population(child1) - population.delete_from_population() - population.insert_in_population(child2) - population.delete_from_population() - - -def _make_children(parent1, parent2): - child1 = copy(parent1) - child2 = copy(parent2) - child1.numerosity = 1 - child2.numerosity = 1 - child1.experience = 0 - child2.experience = 0 - return child1, child2 - - -def _select_offspring(action_set: ClassifiersList) -> Classifier: - fitness_sum = 0 - for cl in action_set: - fitness_sum += cl.fitness - choice_point = np.random.rand() * fitness_sum - fitness_sum = 0 - for cl in action_set: - fitness_sum += cl.fitness - if fitness_sum > choice_point: - return cl - return action_set[random.randrange(len(action_set))] - - -def _apply_crossover(child1: Classifier, child2: Classifier, - parent1: Classifier, parent2: Classifier): - _apply_crossover_in_area(child1, child2, - np.random.rand() * len(child1.condition), - np.random.rand() * len(child1.condition) - ) - child1.prediction = (parent1.prediction + parent2.prediction) / 2 - child1.error = 0.25 * (parent1.error + parent2.error) / 2 - child1.fitness = 0.1 * (parent1.fitness + parent2.fitness) / 2 - child2.prediction = child1.prediction - child2.error = child1.error - child2.fitness = child1.fitness - - -def _apply_crossover_in_area(child1: Classifier, child2: Classifier, x, y): - if x > y: - x, y = y, x - if x > len(child2.condition): - return - if y > len(child2.condition): - y = len(child2.condition) - i = 0 - while i < y: - if x <= i < y: - temp = child1.condition[i] - child1.condition[i] = child2.condition[i] - child2.condition[i] = temp - i += 1 - - -def _apply_mutation(child: Classifier, - cfg: Configuration, - situation): - i = 0 - while i < len(child.condition): + child2 = copy(parent2) + child2.condition = copy(parent2.condition) + child2.time_stamp = time_stamp + + return child1, child2 + + @staticmethod + def _select_offspring(action_set: ClassifiersList) -> Classifier: + assert isinstance(action_set, ClassifiersList) + + # TODO: insert generator to calculate fitness_sum + fitness_sum = 0 + for cl in action_set: + fitness_sum += cl.fitness + choice_point = np.random.rand() * fitness_sum + fitness_sum = 0 + for cl in action_set: + fitness_sum += cl.fitness + if fitness_sum > choice_point: + return cl + return action_set[random.randrange(len(action_set))] + + def _apply_crossover(self, child1: Classifier, child2: Classifier, + parent1: Classifier, parent2: Classifier): + + self._apply_crossover_in_area( + child1, + child2, + np.random.rand() * len(child1.condition), + np.random.rand() * len(child1.condition) + ) + + for child in child1, child2: + child.prediction = (parent1.prediction + parent2.prediction) / 2 + child.error = 0.25 * (parent1.error + parent2.error) / 2 + child.fitness = 0.1 * (parent1.fitness + parent2.fitness) / 2 + + @staticmethod + def _apply_crossover_in_area(child1: Classifier, child2: Classifier, x, y): + if x > y: + x, y = y, x + if x > len(child2.condition): + return + if y > len(child2.condition): + y = len(child2.condition) + i = 0 + while i < y: + if x <= i < y: + temp = child1.condition[i] + child1.condition[i] = child2.condition[i] + child2.condition[i] = temp + i += 1 + + @staticmethod + def _apply_mutation(child: Classifier, + cfg: Configuration, + situation): + i = 0 + while i < len(child.condition): + if np.random.rand() < cfg.mutation_chance: + if child.condition[i] == child.condition.WILDCARD: + child.condition[i] = situation[i] + else: + child.condition[i] = child.condition.WILDCARD + i += 1 if np.random.rand() < cfg.mutation_chance: - if child.condition[i] == child.condition.WILDCARD: - child.condition[i] = situation[i] - else: - child.condition[i] = child.condition.WILDCARD - i += 1 - if np.random.rand() < cfg.mutation_chance: - child.action = np.random.randint(cfg.number_of_actions) + child.action = np.random.randint(cfg.number_of_actions) diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index 925e281a..1181f6ef 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -1,16 +1,16 @@ import logging import random -import numpy as np from copy import copy from typing import Optional +import numpy as np + from lcs.agents import Agent -from lcs.agents.xcs import Configuration, ClassifiersList, GeneticAlgorithm from lcs.agents.Agent import TrialMetrics +from lcs.agents.xcs import Configuration, ClassifiersList +# TODO: delete old code from lcs.strategies.reinforcement_learning import simple_q_learning -from lcs.strategies.action_selection import EpsilonGreedy - - +from lcs.agents.xcs import GeneticAlgorithm logger = logging.getLogger(__name__) @@ -28,8 +28,12 @@ def __init__(self, self.population = population else: self.population = ClassifiersList(cfg=cfg) + self.ga = GeneticAlgorithm( + population=self.population, + cfg=self.cfg + ) self.time_stamp = 0 - self.action_reward = [0 for _ in range(cfg.number_of_actions)] + self.reward = 0 def get_population(self): return self.population @@ -47,9 +51,8 @@ def _run_trial_exploit(self, env, trials, current_trial) -> TrialMetrics: def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: prev_action_set = None - prev_reward = [0 for _ in range(self.cfg.number_of_actions)] + prev_reward = self.reward prev_state = None # state is known as situation - prev_action = 0 prev_time_stamp = self.time_stamp # steps done = False # eop @@ -57,48 +60,50 @@ def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: state = self.cfg.environment_adapter.to_genotype(raw_state) while not done: - self.population.delete_from_population() + assert len(self.population) == len(set(self.population)), 'duplicates found' # We are in t+1 here - match_set = self.population.generate_match_set(state, self.time_stamp) - prediction_array = match_set.prediction_array - action = self.select_action(prediction_array, match_set) - action_set = match_set.generate_action_set(action) + action_set, prediction_array, action = self._form_sets_and_choose_action(state) # apply action to environment raw_state, step_reward, done, _ = env.step(action) state = self.cfg.environment_adapter.to_genotype(raw_state) - self.action_reward[action] = simple_q_learning(self.action_reward[action], - step_reward, - self.cfg.learning_rate, - self.cfg.gamma, - match_set.best_prediction) + if self.cfg.multistep_enfiroment: + self.reward = step_reward + self.cfg.gamma * self.reward self._distribute_and_update(prev_action_set, prev_state, - prev_reward[prev_action] + self.cfg.gamma * max(prediction_array)) + state, + prev_reward + self.cfg.gamma * max(prediction_array)) if done: self._distribute_and_update(action_set, state, - self.action_reward[action]) + state, + self.reward) else: prev_action_set = copy(action_set) - prev_reward[action] = copy(self.action_reward[action]) + prev_reward = self.reward prev_state = copy(state) - prev_action = action self.time_stamp += 1 - return TrialMetrics(self.time_stamp - prev_time_stamp, self.action_reward) + return TrialMetrics(self.time_stamp - prev_time_stamp, self.reward) + + def _form_sets_and_choose_action(self, state): + match_set = self.population.generate_match_set(state, self.time_stamp) + prediction_array = match_set.prediction_array + action = self.select_action(prediction_array, match_set) + action_set = match_set.generate_action_set(action) + return action_set, prediction_array, action - def _distribute_and_update(self, action_set, situation, p): + def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None and len(action_set) > 0: action_set.update_set(p) if self.cfg.do_action_set_subsumption: self.do_action_set_subsumption(action_set) - GeneticAlgorithm.run_ga(self.population, - action_set, - situation, - self.time_stamp, - self.cfg) + self.ga.run_ga( + action_set, + current_situation, + self.time_stamp + ) - # TODO: EspilonGreedy + # TODO: EpsilonGreedy # Run into a lot of issues where in EpsilonGreedy where BestAction was not callable # Changing EpsilonGreed to: # best = BestAction(all_actions=self.all_actions) @@ -121,5 +126,3 @@ def do_action_set_subsumption(self, action_set: ClassifiersList) -> None: cl.numerosity += c.numerosity action_set.safe_remove(c) self.population.safe_remove(c) - - diff --git a/lcs/agents/xcs/__init__.py b/lcs/agents/xcs/__init__.py index d8c4db31..399fbea0 100644 --- a/lcs/agents/xcs/__init__.py +++ b/lcs/agents/xcs/__init__.py @@ -2,5 +2,6 @@ from .Configuration import Configuration from .Classifier import Classifier from .ClassifiersList import ClassifiersList +from .GeneticAlgorithm import GeneticAlgorithm from .XCS import XCS -from .GeneticAlgorithm import * + diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index d864c73c..e5efe7c4 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -1,34 +1,44 @@ from copy import copy -from lcs.agents.xncs import Configuration, Classifier, Effect +from lcs.agents.xncs import Configuration, Effect class Backpropagation: - def __init__(self, cfg: Configuration): - self.classifiers_for_update = [] - self.update_vectors = [] + def __init__(self, + cfg: Configuration): self.cfg = cfg - self.update_cycles = 0 - - def insert_into_bp(self, - classifier: Classifier, - update_vector: Effect): - self.classifiers_for_update.append(classifier) - self.update_vectors.append(update_vector) - - def update_bp(self): - for cl, uv in zip(self.classifiers_for_update, self.update_vectors): - if cl.effect is None: - cl.effect = copy(Effect(uv)) - else: - cl.effect = copy(Effect(uv)) - cl.error = cl.error + self.cfg.lem self.classifiers_for_update = [] - self.update_vectors = [] - self.update_cycles = 0 - def check_and_update(self): - self.update_cycles += 1 - if self.update_cycles >= self.cfg.lmc: - self.update_bp() + def run_bp(self, + action_set, + next_vector: Effect): + for cl in action_set: + cl.guesses += 1 + if cl.effect is not None: + if cl.effect != next_vector: + cl.mistakes += 1 + if not any(cl == inside[0] for inside in self.classifiers_for_update): + self.classifiers_for_update.append( + [cl, next_vector, self.cfg.lmc] + ) + self.check_if_needed() + self.update_errors() + + def check_if_needed(self): + for cl in self.classifiers_for_update: + if cl[0].effect == cl[1]: + self.classifiers_for_update.remove(cl) + else: + cl[2] -= 1 + if cl[2] == 0: + self.classifiers_for_update.remove(cl) + + def update_errors(self): + for cl in self.classifiers_for_update: + cl[0].error += self.cfg.lem + + def update_effect(self, action_set, next_situation): + effect = Effect(next_situation) + for cl in action_set.least_fit_classifiers(self.cfg.update_percentage): + cl.effect = copy(effect) diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index bb53e16b..75b60656 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -14,17 +14,21 @@ def __init__(self, time_stamp: int = None, effect: Union[Effect, str, None] = None) -> None: self.effect = effect + self.mistakes = 0 + self.guesses = 0 + super().__init__(cfg, condition, action, time_stamp) - def __eq__(self, other): - if other.action == self.action \ - and other.condition == self.condition: - if other.effect is None and self.effect is None: - return True - if other.effect is None: - return False - if self.effect is None: - return False - if other.effect == self.effect: - return True - return False + def __hash__(self): + return hash((str(self.condition), str(self.effect), self.action)) + + @property + def accuracy(self): + if self.guesses > 0: + return (self.guesses - self.mistakes) / self.guesses + return None + + def __str__(self): + return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ + f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}, error:{self.error:.2f}]" + diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 9e6c3a8b..b4055355 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -1,9 +1,11 @@ import numpy as np import logging +import random import lcs.agents.xcs as xcs +from lcs import Perception from lcs.agents.xcs import Condition -from lcs.agents.xncs import Classifier, Configuration +from lcs.agents.xncs import Classifier, Configuration, Effect, Backpropagation logger = logging.getLogger(__name__) @@ -18,13 +20,46 @@ def __init__(self, def generate_covering_classifier(self, situation, action, time_stamp): generalized = [] + effect = [] for i in range(len(situation)): if np.random.rand() > self.cfg.covering_wildcard_chance: generalized.append(self.cfg.classifier_wildcard) else: generalized.append(situation[i]) + if not self.cfg.cover_env_input: + effect.append(str(random.choice(situation))) + if self.cfg.cover_env_input: + effect = None + else: + effect = Effect(effect) cl = Classifier(cfg=self.cfg, condition=Condition(generalized), action=action, - time_stamp=time_stamp) + time_stamp=time_stamp, + effect=effect) return cl + + def generate_action_set(self, action): + action_ls = [cl for cl in self if cl.action == action] + return ClassifiersList(self.cfg, *action_ls) + + def generate_match_set(self, situation: Perception, time_stamp): + matching_ls = [cl for cl in self if cl.does_match(situation)] + action = self._find_not_present_action(matching_ls) + while action is not None: + cl = self._generate_covering_and_insert(situation, action, time_stamp) + matching_ls.append(cl) + action = self._find_not_present_action(matching_ls) + return ClassifiersList(self.cfg, *matching_ls) + + def least_fit_classifiers(self, percentage): + assert 0 < percentage <= 1 + return sorted( + self, + key=lambda cl: cl.fitness + )[0:int(len(self) * percentage)] + + @property + def fittest_classifier(self): + assert len(self) > 0 + return max(self, key=lambda cl: cl.fitness * cl.prediction) diff --git a/lcs/agents/xncs/Configuration.py b/lcs/agents/xncs/Configuration.py index 26c14f8f..d1a2655f 100644 --- a/lcs/agents/xncs/Configuration.py +++ b/lcs/agents/xncs/Configuration.py @@ -8,16 +8,18 @@ class Configuration(xcs.Configuration): def __init__(self, - number_of_actions: int, # theta_mna it is actually smart to make it equal to number of actions + number_of_actions: int, lmc: int = 100, lem: float = 1, + + # theta_mna it is actually smart to make it equal to number of actions classifier_wildcard: str = '#', environment_adapter=EnvironmentAdapter, max_population: int = 200, # n learning_rate: float = 0.1, # beta alpha: float = 0.1, epsilon_0: float = 10, - v: int = 5, + v: int = 5, # nu gamma: float = 0.71, ga_threshold: int = 25, chi: float = 0.5, @@ -26,41 +28,50 @@ def __init__(self, delta: float = 0.1, subsumption_threshold: int = 20, # theta_sub covering_wildcard_chance: float = 0.33, # population wildcard - initial_prediction: float = float(np.finfo(np.float32).tiny), # p_i - initial_error: float = float(np.finfo(np.float32).tiny), # epsilon_i - initial_fitness: float = float(np.finfo(np.float32).tiny), # f_i + initial_prediction: float = 0.000001, # p_i + initial_error: float = 0.000001, # epsilon_i + initial_fitness: float = 0.000001, # f_i epsilon: float = 0.5, # p_exp, exploration probability do_ga_subsumption: bool = False, do_action_set_subsumption: bool = False, metrics_trial_frequency: int = 5, - user_metrics_collector_fcn: Callable = None + user_metrics_collector_fcn: Callable = None, + multistep_enfiroment: bool = True, + update_percentage: float = 0.2, + update_env_input: bool = False, + cover_env_input: bool = False, ) -> None: self.lmc = lmc self.lem = lem - self.classifier_wildcard = classifier_wildcard - self.environment_adapter = environment_adapter - self.max_population = max_population - self.learning_rate = learning_rate - self.alpha = alpha - self.epsilon_0 = epsilon_0 - self.v = v - self.gamma = gamma - self.ga_threshold = ga_threshold - self.chi = chi - self.mutation_chance = mutation_chance - self.deletion_threshold = deletion_threshold - self.delta = delta - self.subsumption_threshold = subsumption_threshold - self.covering_wildcard_chance = covering_wildcard_chance - self.initial_prediction = initial_prediction - self.initial_error = initial_error - self.initial_fitness = initial_fitness - self.epsilon = epsilon # p_exp, probability of exploration - self.number_of_actions = number_of_actions - self.do_GA_subsumption = do_ga_subsumption - self.do_action_set_subsumption = do_action_set_subsumption - - self.metrics_trial_frequency = metrics_trial_frequency - self.user_metrics_collector_fcn = user_metrics_collector_fcn + self.update_env_input = update_env_input + self.cover_env_input = cover_env_input + self.update_percentage = update_percentage + super().__init__( + number_of_actions=number_of_actions, + classifier_wildcard=classifier_wildcard, + environment_adapter=environment_adapter, + max_population=max_population, # n + learning_rate=learning_rate, # beta + alpha=alpha, + epsilon_0=epsilon_0, + v=v, # nu + gamma=gamma, + ga_threshold=ga_threshold, + chi=chi, + mutation_chance=mutation_chance, # mu + deletion_threshold=deletion_threshold, # theta_del + delta=delta, + subsumption_threshold=subsumption_threshold, # theta_sub + covering_wildcard_chance=covering_wildcard_chance, # population wildcard + initial_prediction=initial_prediction, # p_i + initial_error=initial_error, # epsilon_i + initial_fitness=initial_fitness, # f_i + epsilon=epsilon, # p_exp, exploration probability + do_ga_subsumption=do_ga_subsumption, + do_action_set_subsumption=do_action_set_subsumption, + metrics_trial_frequency=metrics_trial_frequency, + user_metrics_collector_fcn=user_metrics_collector_fcn, + multistep_enfiroment=multistep_enfiroment + ) diff --git a/lcs/agents/xncs/GeneticAlgorithm.py b/lcs/agents/xncs/GeneticAlgorithm.py new file mode 100644 index 00000000..04e6c825 --- /dev/null +++ b/lcs/agents/xncs/GeneticAlgorithm.py @@ -0,0 +1,42 @@ +from lcs.agents.xcs import GeneticAlgorithm as XCSGeneticAlgorithm +from copy import copy +from lcs.agents.xncs import Classifier, ClassifiersList, Configuration + + +class GeneticAlgorithm(XCSGeneticAlgorithm): + + # Classifierslist is supposed to be XNCS type + def __init__( + self, + population: ClassifiersList, + cfg: Configuration + ): + super().__init__(population, cfg) + + def _make_children(self, parent1, parent2, time_stamp): + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) + + child1 = Classifier( + self.cfg, + copy(parent1.condition), + copy(parent1.action), + time_stamp, + copy(parent1.effect) + ) + child1.prediction = parent1.prediction + child1.error = parent1.error + child1.fitness = parent1.fitness + + child2 = Classifier( + self.cfg, + copy(parent2.condition), + copy(parent2.action), + time_stamp, + copy(parent2.effect) + ) + child2.prediction = parent2.prediction + child2.error = parent2.error + child2.fitness = parent2.fitness + + return child1, child2 diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 7a990799..721c8712 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -2,13 +2,12 @@ import random import numpy as np from copy import copy - +import queue from lcs.agents.xcs import XCS -from lcs.agents.xncs import Configuration, Backpropagation -# TODO: find typo that makes __init__ not do that -from lcs.agents.xncs.ClassifiersList import ClassifiersList from lcs.agents.Agent import TrialMetrics -from lcs.strategies.reinforcement_learning import simple_q_learning +from lcs.agents.xncs import Configuration, Backpropagation +# TODO: find a way to not require super in __init__ +from lcs.agents.xncs import ClassifiersList, GeneticAlgorithm, Effect class XNCS(XCS): @@ -21,67 +20,58 @@ def __init__(self, :param cfg: object storing parameters of the experiment :param population: all classifiers at current time """ - self.back_propagation = Backpropagation(cfg) - self.cfg = cfg + if population is not None: self.population = population else: self.population = ClassifiersList(cfg=cfg) + self.cfg = cfg + self.ga = GeneticAlgorithm( + population=self.population, + cfg=self.cfg + ) + self.back_propagation = Backpropagation( + cfg=self.cfg + ) self.time_stamp = 0 - self.action_reward = [0 for _ in range(cfg.number_of_actions)] - - def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: - prev_action_set = None - prev_reward = [0 for _ in range(self.cfg.number_of_actions)] - prev_state = None # state is known as situation - prev_action = 0 - self.time_stamp = 0 # steps - done = False # eop - - raw_state = env.reset() - state = self.cfg.environment_adapter.to_genotype(raw_state) + self.reward = 0 + self.mistakes = [] - while not done: - self.population.delete_from_population() - # We are in t+1 here - match_set = self.population.generate_match_set(state, self.time_stamp) - prediction_array = match_set.prediction_array - action = self.select_action(prediction_array, match_set) - action_set = match_set.generate_action_set(action) - # apply action to environment - raw_state, step_reward, done, _ = env.step(action) - state = self.cfg.environment_adapter.to_genotype(raw_state) - self.action_reward[action] = simple_q_learning(self.action_reward[action], - step_reward, - self.cfg.learning_rate, - self.cfg.gamma, - match_set.best_prediction) - - self._distribute_and_update(prev_action_set, - prev_state, - prev_reward[prev_action] + self.cfg.gamma * max(prediction_array)) - if done: - self._distribute_and_update(action_set, - state, - self.action_reward[action]) - else: - prev_action_set = copy(action_set) - prev_reward[action] = copy(self.action_reward[action]) - prev_state = copy(state) - prev_action = action - - self.time_stamp += 1 - return TrialMetrics(self.time_stamp, self.action_reward) - - def _distribute_and_update(self, action_set, situation, p): - super()._distribute_and_update(action_set, situation, p) - self._compare_effect(action_set, situation) - - def _compare_effect(self, action_set, situation): + def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None: for cl in action_set: - if cl.effect is None or not cl.effect.subsumes(situation): - self.back_propagation.insert_into_bp(cl, situation) + if cl.effect is None: + cl.effect = Effect(next_situation) + self.update_fraction_accuracy(action_set, next_situation) + if self.cfg.update_env_input: + self.back_propagation.update_effect(action_set, next_situation) + else: + self.back_propagation.update_effect(action_set, action_set.fittest_classifier.effect) + self.back_propagation.run_bp( + action_set, + Effect(next_situation) + ) + super()._distribute_and_update(action_set, current_situation, next_situation, p) + + def update_fraction_accuracy(self, action_set, next_vector): + most_numerous = sorted(action_set, key=lambda cl: -1 * cl.numerosity)[0] + if most_numerous.effect is not None and next_vector is not None: + if most_numerous.effect != Effect(next_vector): + if len(self.mistakes) >= 100: + self.mistakes.pop(0) + self.mistakes.append(1) else: - self.back_propagation.update_bp() - self.back_propagation.check_and_update() + self.mistakes.append(1) + else: + if len(self.mistakes) >= 100: + self.mistakes.pop(0) + self.mistakes.append(0) + else: + self.mistakes.append(0) + + @property + def fraction_accuracy(self): + if len(self.mistakes) > 0: + return sum(self.mistakes) / len(self.mistakes) + else: + return 0 diff --git a/lcs/agents/xncs/__init__.py b/lcs/agents/xncs/__init__.py index 041002a2..42342b47 100644 --- a/lcs/agents/xncs/__init__.py +++ b/lcs/agents/xncs/__init__.py @@ -2,5 +2,6 @@ from .Configuration import Configuration from .Classifier import Classifier from .Backpropagation import Backpropagation -from .XNCS import XNCS +from .GeneticAlgorithm import GeneticAlgorithm from .ClassifiersList import ClassifiersList +from .XNCS import XNCS diff --git a/lcs/agents/yacs/yacs.py b/lcs/agents/yacs/yacs.py index 53858f7f..61f1e0dd 100644 --- a/lcs/agents/yacs/yacs.py +++ b/lcs/agents/yacs/yacs.py @@ -38,9 +38,7 @@ def __init__(self, discount_factor: float = 0.9, estimate_expected_improvements: bool = True, metrics_trial_frequency: int = 5, - model_checkpoint_frequency: int = None, - user_metrics_collector_fcn: Callable = None, - use_mlflow: bool = False): + user_metrics_collector_fcn: Callable = None): assert classifier_length == len(feature_possible_values) self.classifier_length = classifier_length self.number_of_possible_actions = number_of_possible_actions @@ -51,9 +49,7 @@ def __init__(self, self.gamma = discount_factor self.estimate_expected_improvements = estimate_expected_improvements self.metrics_trial_frequency = metrics_trial_frequency - self.model_checkpoint_freq = model_checkpoint_frequency self.user_metrics_collector_fcn = user_metrics_collector_fcn - self.use_mlflow = use_mlflow class Condition(ImmutableSequence): diff --git a/setup.py b/setup.py index 9780f865..f5a92ad3 100644 --- a/setup.py +++ b/setup.py @@ -8,12 +8,6 @@ 'pytest-xdist==2.2.1' ] -mlflow_requires = [ - 'boto3', - 'dill', - 'mlflow' -] - docs_requires = [ 'sphinx', 'nbsphinx', @@ -51,7 +45,6 @@ 'dataslots>=1.0.1' ], extras_require={ - 'mlflow': mlflow_requires, 'testing': testing_requires, 'documentation': docs_requires }, diff --git a/tests/lcs/agents/xcs/__init__.py b/tests/lcs/agents/xcs/__init__.py index e8ed99a7..e69de29b 100644 --- a/tests/lcs/agents/xcs/__init__.py +++ b/tests/lcs/agents/xcs/__init__.py @@ -1 +0,0 @@ -# py.test --cov=xcs C:\Users\Metron\Documents\GitHub\pyalcs\tests\lcs\agents\xncs diff --git a/tests/lcs/agents/xcs/test_Classifier.py b/tests/lcs/agents/xcs/test_Classifier.py index 990abfed..89bd0e6f 100644 --- a/tests/lcs/agents/xcs/test_Classifier.py +++ b/tests/lcs/agents/xcs/test_Classifier.py @@ -88,10 +88,16 @@ def test_does_subsume(self, cfg: Configuration): ("##11", "1111", 1, 1, False), ("1111", "11", 1, 1, False), - ("11", "1111", 1, 1, False) + ("11", "1111", 1, 1, False), + + ("0##11#", "0##11#", 0, 0, True), + ("###000", "###000", 0, 0, True), + ("###00#", "###00#", 1, 1, True), ]) def test_equals(self, cfg, cond1, cond2, act1, act2, result): - assert result == (Classifier(cfg=cfg, condition=Condition(cond1), action=act1, time_stamp=0) == - Classifier(cfg=cfg, condition=Condition(cond2), action=act2, time_stamp=0)) + cl1 = Classifier(condition=Condition(cond1), action=act1, time_stamp=0, cfg=cfg) + cl2 = Classifier(condition=Condition(cond2), action=act2, time_stamp=0, cfg=cfg) + + assert result is (cl1 == cl2) diff --git a/tests/lcs/agents/xcs/test_ClassifierList.py b/tests/lcs/agents/xcs/test_ClassifierList.py index 414b28bb..bb2fde9d 100644 --- a/tests/lcs/agents/xcs/test_ClassifierList.py +++ b/tests/lcs/agents/xcs/test_ClassifierList.py @@ -17,14 +17,14 @@ def situation(self): @pytest.fixture def classifiers_list_diff_actions(self, cfg, situation): - classifiers_list = ClassifiersList(cfg) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) - return classifiers_list - - def test_init(self, cfg): + pop = ClassifiersList(cfg) + pop.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) + pop.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) + pop.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) + pop.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) + return pop + + def test_should_be_empty_when_initialized(self, cfg): assert len(ClassifiersList(cfg)) == 0 @pytest.mark.parametrize("cond, act", [ @@ -34,9 +34,37 @@ def test_init(self, cfg): ("1100", 3) ]) def test_insert_population(self, classifiers_list_diff_actions, cfg, cond, act): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) + # given + initial_size = len(classifiers_list_diff_actions) + cl = Classifier(cfg=cfg, condition=Condition(cond), action=act, time_stamp=0) + + # when classifiers_list_diff_actions.insert_in_population(cl) + + # then + assert len(classifiers_list_diff_actions) == initial_size assert any(c == cl for c in classifiers_list_diff_actions) + assert any(c.numerosity == 2 for c in classifiers_list_diff_actions) + + @pytest.mark.parametrize("cond1, cond2, act1, act2, size", [ + ("1111", "1111", 1, 1, 5), + ("#100", "#100", 0, 0, 5), + ("0##11#", "0##11#", 0, 0, 5), + + ]) + def test_insert_population_two(self, cfg, classifiers_list_diff_actions, cond1, cond2, act1, act2, size): + # given + cl1 = Classifier(condition=Condition(cond1), action=act1, cfg=cfg) + cl2 = Classifier(condition=Condition(cond2), action=act2, cfg=cfg) + + # when + classifiers_list_diff_actions.insert_in_population(cl1) + classifiers_list_diff_actions.insert_in_population(cl2) + + # then + assert any(c == cl1 for c in classifiers_list_diff_actions) + assert any(c == cl2 for c in classifiers_list_diff_actions) + assert len(classifiers_list_diff_actions) == size @pytest.mark.parametrize("cond, act", [ ("1111", 0), @@ -47,8 +75,13 @@ def test_insert_population(self, classifiers_list_diff_actions, cfg, cond, act): ("112", 0), ]) def test_insert_population_new_condition(self, classifiers_list_diff_actions, cfg, cond, act): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) + # given + cl = Classifier(condition=Condition("1111"), action=0, cfg=cfg) + + # when classifiers_list_diff_actions.insert_in_population(cl) + + # then for c in classifiers_list_diff_actions: assert c.numerosity == 1 assert classifiers_list_diff_actions[4] == cl @@ -108,7 +141,8 @@ def test_prediction_array(self, cfg, classifiers_list_diff_actions): classifiers_list_diff_actions[0].prediction = 10 prediction_array = classifiers_list_diff_actions.prediction_array assert len(classifiers_list_diff_actions) == cfg.number_of_actions - assert prediction_array[0] > prediction_array[1] + assert all(prediction_array[0] >= prediction + for prediction in prediction_array) def test_update_fitness(self, cfg, classifiers_list_diff_actions): classifiers_list_diff_actions._update_fitness() diff --git a/tests/lcs/agents/xcs/test_Condition.py b/tests/lcs/agents/xcs/test_Condition.py index f197d362..ebffa2c2 100644 --- a/tests/lcs/agents/xcs/test_Condition.py +++ b/tests/lcs/agents/xcs/test_Condition.py @@ -25,7 +25,8 @@ def test_should_get_initialized_with_str(self): ("1111", "1100", False), ("1111", "11", False), - ("11", "1100", False) + ("#1111#", "#1111#", True), + ("###01#", "###01#", True), ]) def test_equal(self, cond1, cond2, result): assert result == (Condition(cond1) == Condition(cond2)) @@ -66,3 +67,11 @@ def test_number_of_wildcards(self, cond, num): ]) def test_is_more_general(self, cond1, cond2, result): assert Condition(cond1).is_more_general(Condition(cond2)) == result + + @pytest.mark.parametrize("_c", [ + ([0.1, 0.2]), + ([0, 1]), + ]) + def test_should_fail_with_invalid_types(self, _c): + with pytest.raises(AssertionError) as _: + Condition(_c) diff --git a/tests/lcs/agents/xcs/test_Configuration.py b/tests/lcs/agents/xcs/test_Configuration.py index 0ce95791..0ce47a52 100644 --- a/tests/lcs/agents/xcs/test_Configuration.py +++ b/tests/lcs/agents/xcs/test_Configuration.py @@ -12,7 +12,5 @@ class TestConfiguration: def cfg(self): return Configuration(number_of_actions=4) - def test_minimum(self, cfg): - assert float(np.finfo(np.float32).tiny) == cfg.initial_prediction diff --git a/tests/lcs/agents/xcs/test_GA.py b/tests/lcs/agents/xcs/test_GA.py new file mode 100644 index 00000000..6ecf3455 --- /dev/null +++ b/tests/lcs/agents/xcs/test_GA.py @@ -0,0 +1,133 @@ +import pytest +from copy import copy + +from lcs import Perception +from lcs.agents.xcs import Configuration, Condition, Classifier, ClassifiersList, XCS, GeneticAlgorithm +from lcs.strategies.reinforcement_learning import simple_q_learning + +class TestGA: + + @pytest.fixture + def number_of_actions(self): + return 4 + + @pytest.fixture + def cfg(self, number_of_actions): + return Configuration(number_of_actions=number_of_actions, do_action_set_subsumption=False) + + @pytest.fixture + def situation(self): + return "1100" + + @pytest.fixture + def classifiers_list_diff_actions(self, cfg, situation): + classifiers_list = ClassifiersList(cfg) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) + return classifiers_list + + @pytest.fixture + def xcs(self, cfg, classifiers_list_diff_actions): + xcs = XCS(cfg=cfg, population=classifiers_list_diff_actions) + return xcs + + @pytest.fixture + def ga(self, classifiers_list_diff_actions, cfg): + ga = GeneticAlgorithm(classifiers_list_diff_actions, cfg) + return ga + + def test_mutation(self, cfg, ga): + cfg.mutation_chance = 0 + cl = Classifier(cfg, Condition("####"), 0, 0) + ga._apply_mutation(cl, cfg, Perception("1111")) + assert cl.action == Classifier(cfg, Condition("1111"), 0, 0).action + assert cl.does_match(Condition("1111")) + cfg.mutation_chance = 1 + cl = Classifier(cfg, Condition("1111"), 0, 0) + ga._apply_mutation(cl, cfg, Perception("1111")) + assert cl.is_more_general(Classifier(cfg, Condition("1111"), 0, 0)) + + @pytest.mark.parametrize("cond1, cond2, x, y, end_cond1, end_cond2", [ + ("11111", "#####", 0, 5, "#####", "11111"), + ("11111", "000", 0, 5, "00011", "111"), + ("11111", "#####", 1, 4, "1###1", "#111#") + ]) + def test_crossover_area(self, cfg, ga, cond1, cond2, x, y, end_cond1, end_cond2): + cl1 = Classifier(cfg, Condition(cond1), 0, 0) + cl2 = Classifier(cfg, Condition(cond2), 1, 0) + ga._apply_crossover_in_area(cl1, cl2, x, y) + assert cl1.condition == Condition(end_cond1) + assert cl2.condition == Condition(end_cond2) + # Just to check for errors + + def test_crossover_values(self, cfg, ga, situation, classifiers_list_diff_actions): + cl1 = Classifier(cfg, Condition(situation), 0, 0) + cl2 = Classifier(cfg, Condition(situation), 1, 0) + ga._apply_crossover( + cl1, cl2, + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1] + ) + assert cl1.prediction == cl2.prediction + assert cl1.error == cl2.error + assert cl1.fitness == cl2.fitness + + @pytest.mark.parametrize("chi", [ + 1, + 0 + ]) + # only tests for errors and types + def test_run_ga(self, cfg, ga, classifiers_list_diff_actions, chi): + cfg.do_GA_subsumption = True + xcs = XCS(cfg, classifiers_list_diff_actions) + action_set = xcs.population.generate_action_set(0) + cfg.chi = chi # do perform crossover + cfg.do_GA_subsumption = False # do not perform subsumption + ga.run_ga(action_set, Perception("0000"), 100000) + assert xcs.population[0].time_stamp != 0 + assert xcs.population.numerosity > 4 + + def test_make_children(self, cfg, ga, classifiers_list_diff_actions): + child1, child2 = ga._make_children( + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1], + 0 + ) + assert child1.numerosity == 1 + assert child2.numerosity == 1 + assert child1.experience == 0 + assert child2.experience == 0 + assert id(child1) != id(classifiers_list_diff_actions[0]) + assert id(child2) != id(classifiers_list_diff_actions[0]) + assert id(child1) != id(classifiers_list_diff_actions[1]) + assert id(child2) != id(classifiers_list_diff_actions[1]) + + def test_do_ga_subsumption_does_subsume_true(self, cfg, ga, classifiers_list_diff_actions, situation): + cfg.do_GA_subsumption = True + classifiers_list_diff_actions[0].error = 0 + classifiers_list_diff_actions[1].error = 0 + classifiers_list_diff_actions[0].expirience = 30 + classifiers_list_diff_actions[1].expirience = 30 + ga._perform_insertion_or_subsumption( + Classifier(cfg, Condition(situation), 0, 0), + Classifier(cfg, Condition(situation), 1, 0), + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1] + ) + assert classifiers_list_diff_actions[0].numerosity > 1 + assert classifiers_list_diff_actions[1].numerosity > 1 + + def test_do_ga_subsumption_does_subsume_false(self, cfg, ga, classifiers_list_diff_actions, situation): + cfg.do_GA_subsumption = True + ga._perform_insertion_or_subsumption( + Classifier(cfg, Condition("####"), 0, 0), + Classifier(cfg, Condition("####"), 1, 0), + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1] + ) + assert classifiers_list_diff_actions[0].numerosity == 1 + assert classifiers_list_diff_actions[1].numerosity == 1 + assert len(classifiers_list_diff_actions) == 6 + diff --git a/tests/lcs/agents/xcs/test_XCS.py b/tests/lcs/agents/xcs/test_XCS.py index b4519d0f..cb0bc9f9 100644 --- a/tests/lcs/agents/xcs/test_XCS.py +++ b/tests/lcs/agents/xcs/test_XCS.py @@ -2,7 +2,7 @@ from copy import copy from lcs import Perception -from lcs.agents.xcs import Configuration, Condition, Classifier, ClassifiersList, XCS, GeneticAlgorithm +from lcs.agents.xcs import Configuration, Condition, Classifier, ClassifiersList, XCS from lcs.strategies.reinforcement_learning import simple_q_learning class TestXCS: @@ -71,104 +71,4 @@ def test_distribute_and_update(self, cfg, situation, classifiers_list_diff_actio assert not cl.fitness == cfg.initial_fitness assert not cl.error == cfg.initial_error - def test_mutation(self, cfg): - cfg.mutation_chance = 0 - cl = Classifier(cfg, Condition("####"), 0, 0) - GeneticAlgorithm._apply_mutation(cl, cfg, Perception("1111")) - assert cl.action == Classifier(cfg, Condition("1111"), 0, 0).action - assert cl.does_match(Condition("1111")) - cfg.mutation_chance = 1 - cl = Classifier(cfg, Condition("1111"), 0, 0) - GeneticAlgorithm._apply_mutation(cl, cfg, Perception("1111")) - assert cl.is_more_general(Classifier(cfg, Condition("1111"), 0, 0)) - - @pytest.mark.parametrize("cond1, cond2, x, y, end_cond1, end_cond2", [ - ("11111", "#####", 0, 5, "#####", "11111"), - ("11111", "000", 0, 5, "00011", "111"), - ("11111", "#####", 1, 4, "1###1", "#111#") - ]) - def test_crossover_area(self, cfg, cond1, cond2, x, y, end_cond1, end_cond2): - cl1 = Classifier(cfg, Condition(cond1), 0, 0) - cl2 = Classifier(cfg, Condition(cond2), 1, 0) - GeneticAlgorithm._apply_crossover_in_area(cl1, cl2, x, y) - assert cl1.condition == Condition(end_cond1) - assert cl2.condition == Condition(end_cond2) - # Just to check for errors - - def test_crossover_values(self, cfg, situation, classifiers_list_diff_actions): - cl1 = Classifier(cfg, Condition(situation), 0, 0) - cl2 = Classifier(cfg, Condition(situation), 1, 0) - GeneticAlgorithm._apply_crossover(cl1, cl2, - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] - ) - assert cl1.prediction == cl2.prediction - assert cl1.error == cl2.error - assert cl1.fitness == cl2.fitness - - @pytest.mark.parametrize("chi", [ - 1, - 0 - ]) - # only tests for errors and types - def test_run_ga(self, cfg, classifiers_list_diff_actions, chi): - cfg.do_GA_subsumption = True - xcs = XCS(cfg, classifiers_list_diff_actions) - action_set = xcs.population.generate_action_set(0) - cfg.chi = chi # do perform crossover - cfg.do_GA_subsumption = False # do not perform subsumption - GeneticAlgorithm.run_ga(xcs.population, action_set, Perception("0000"), 100000, cfg) - assert xcs.population[0].time_stamp != 0 - assert xcs.population.numerosity > 4 - - def test_simple_q_learning(self, cfg, classifiers_list_diff_actions): - reward = simple_q_learning(0, 0, cfg.learning_rate, cfg.gamma, 0) - assert reward == 0 - reward = simple_q_learning(0, 10, cfg.learning_rate, cfg.gamma, 0) - assert reward > 0 - new_reward = simple_q_learning(reward, 0, cfg.learning_rate, cfg.gamma, 0) - assert new_reward < reward - - def test_make_children(self, cfg, classifiers_list_diff_actions): - child1, child2 = GeneticAlgorithm._make_children( - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] - ) - assert child1.numerosity == 1 - assert child2.numerosity == 1 - assert child1.experience == 0 - assert child2.experience == 0 - assert id(child1) != id(classifiers_list_diff_actions[0]) - assert id(child2) != id(classifiers_list_diff_actions[0]) - assert id(child1) != id(classifiers_list_diff_actions[1]) - assert id(child2) != id(classifiers_list_diff_actions[1]) - - def test_do_ga_subsumption_does_subsume_true(self, cfg, classifiers_list_diff_actions, situation): - cfg.do_GA_subsumption = True - classifiers_list_diff_actions[0].error = 0 - classifiers_list_diff_actions[1].error = 0 - classifiers_list_diff_actions[0].expirience = 30 - classifiers_list_diff_actions[1].expirience = 30 - GeneticAlgorithm._perform_insertion_or_subsumption( - cfg, classifiers_list_diff_actions, - Classifier(cfg, Condition(situation), 0, 0), - Classifier(cfg, Condition(situation), 1, 0), - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] - ) - assert classifiers_list_diff_actions[0].numerosity > 1 - assert classifiers_list_diff_actions[1].numerosity > 1 - - def test_do_ga_subsumption_does_subsume_false(self, cfg, classifiers_list_diff_actions, situation): - cfg.do_GA_subsumption = True - GeneticAlgorithm._perform_insertion_or_subsumption( - cfg, classifiers_list_diff_actions, - Classifier(cfg, Condition("####"), 0, 0), - Classifier(cfg, Condition("####"), 1, 0), - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] - ) - assert classifiers_list_diff_actions[0].numerosity == 1 - assert classifiers_list_diff_actions[1].numerosity == 1 - assert len(classifiers_list_diff_actions) == 6 diff --git a/tests/lcs/agents/xncs/__init__.py b/tests/lcs/agents/xncs/__init__.py index bfe0111e..8b137891 100644 --- a/tests/lcs/agents/xncs/__init__.py +++ b/tests/lcs/agents/xncs/__init__.py @@ -1 +1 @@ -# py.test --cov=xncs C:\Users\Metron\Documents\GitHub\pyalcs\tests\lcs\agents\xncs + diff --git a/tests/lcs/agents/xncs/test_Classifier.py b/tests/lcs/agents/xncs/test_Classifier.py index 91fddf16..15d81827 100644 --- a/tests/lcs/agents/xncs/test_Classifier.py +++ b/tests/lcs/agents/xncs/test_Classifier.py @@ -11,10 +11,27 @@ def cfg(self): return Configuration(lmc=2, lem=0.2, number_of_actions=4) def test_init(self, cfg): - cl = Classifier(cfg) + cl = Classifier(condition='####', cfg=cfg) assert cl.cfg == cfg assert cl.effect is None + def test_classifier_default(self, cfg): + cl = Classifier(cfg, + Condition("####"), + 2, + 8) + assert cl.condition == Condition('####') + assert cl.prediction == cfg.initial_error + + def test_does_subsume(self, cfg: Configuration): + cl = Classifier(cfg, Condition("11##"), 0, 0) + assert not cl.does_subsume(Classifier(cfg, Condition("1111"), 0, 0)) + cl.experience = cfg.subsumption_threshold * 2 + cl.error = cfg.initial_error / 2 + assert cl.does_subsume(Classifier(cfg, Condition("1111"), 0, 0)) + assert not cl.does_subsume(Classifier(cfg, Condition("1111"), 1, 0)) + assert not cl.does_subsume(Classifier(cfg, Condition("0011"), 1, 0)) + @pytest.mark.parametrize("cond1, cond2, act1, act2, result, ef1, ef2", [ ("1111", "1111", 1, 1, True, "1111", "1111"), ("#100", "#100", 0, 0, True, "1111", "1111"), @@ -37,8 +54,8 @@ def test_init(self, cfg): ("1111", "11", 1, 1, False, "1111", "1111"), ("11", "1111", 1, 1, False, "1111", "1111"), - ("1111", "1111", 1, 1, False, "1111", "1100"), - ("1111", "1111", 1, 1, False, "1100", "1111"), + # ("1111", "1111", 1, 1, False, "1111", "1100"), + # ("1111", "1111", 1, 1, False, "1100", "1111"), ]) def test_equals(self, cfg, cond1, cond2, act1, act2, result, ef1, ef2): diff --git a/tests/lcs/agents/xncs/test_ClassifiersList.py b/tests/lcs/agents/xncs/test_ClassifiersList.py index a3b30d5b..11f10384 100644 --- a/tests/lcs/agents/xncs/test_ClassifiersList.py +++ b/tests/lcs/agents/xncs/test_ClassifiersList.py @@ -35,7 +35,7 @@ def test_covering(self, cfg): covering_cl = classifiers_list.generate_covering_classifier(Perception("1111"), 0, 0) assert covering_cl.does_match(Perception("1111")) assert classifiers_list.generate_covering_classifier("1111", 0, 0).action == 0 - assert covering_cl.effect is None + assert len(covering_cl.effect) == len(covering_cl.condition) def test_match_set(self, classifiers_list_diff_actions): assert len(classifiers_list_diff_actions.generate_match_set(Perception("1100"), 1)) == 4 @@ -49,3 +49,28 @@ def test_action_set(self, cfg, classifiers_list_diff_actions): assert action_set[0].action == 0 action_set[0].action = 1 assert classifiers_list_diff_actions[0].action == 1 + + def test_return_fittest(self, classifiers_list_diff_actions): + classifiers_list_diff_actions[2].prediction = 20 + classifiers_list_diff_actions[2].fitness = 20 + assert id(classifiers_list_diff_actions.fittest_classifier) == id(classifiers_list_diff_actions[2]) + + def test_return_least_fit(self, classifiers_list_diff_actions): + classifiers_list_diff_actions[0].fitness = 1250 + classifiers_list_diff_actions[1].fitness = 1000 + classifiers_list_diff_actions[2].fitness = 750 + classifiers_list_diff_actions[3].fitness = 0.01 + sort = classifiers_list_diff_actions.least_fit_classifiers(0.25) + assert sort[0] == classifiers_list_diff_actions[3] + assert len(sort) == 1 + + def test_return_multiple_least_fit(self, classifiers_list_diff_actions): + classifiers_list_diff_actions[0].fitness = 1250 + classifiers_list_diff_actions[1].fitness = 1000 + classifiers_list_diff_actions[2].fitness = 750 + classifiers_list_diff_actions[3].fitness = 0.01 + sort = classifiers_list_diff_actions.least_fit_classifiers(0.75) + assert sort[0] == classifiers_list_diff_actions[3] + assert sort[1] == classifiers_list_diff_actions[2] + assert sort[2] == classifiers_list_diff_actions[1] + assert len(sort) == 3 diff --git a/tests/lcs/agents/xncs/test_Effect.py b/tests/lcs/agents/xncs/test_Effect.py index 4859da1f..a8f72db0 100644 --- a/tests/lcs/agents/xncs/test_Effect.py +++ b/tests/lcs/agents/xncs/test_Effect.py @@ -22,3 +22,16 @@ def test_subsumes(self, cond1, cond2, result): assert result == Effect(cond1).subsumes(cond2) assert result == Effect(cond1).subsumes(cond2) assert result == Effect(cond1).subsumes(cond2) + + @pytest.mark.parametrize("cond1, cond2, result", [ + ("1111", "1111", True), + ("11##", "1111", False), + ("1100", "1111", False), + ("1100", "0111", False), + ("1100", "0110", False), + ("11##", "11##", True), + ("11", "1111", False), + ("1111", "10##", False), + ]) + def test_equals(self, cond1, cond2, result): + assert result == (Effect(cond1) == Effect(cond2)) diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index d33a388e..64eb535a 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -2,7 +2,7 @@ from copy import copy from lcs import Perception -from lcs.agents.xncs import XNCS, Classifier, Configuration, Backpropagation, ClassifiersList +from lcs.agents.xncs import XNCS, Classifier, Configuration, Backpropagation, ClassifiersList, Effect from lcs.agents.xcs import Condition @@ -19,10 +19,10 @@ def situation(self): @pytest.fixture def classifiers_list_diff_actions(self, cfg, situation): classifiers_list = ClassifiersList(cfg) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0, Effect(situation))) return classifiers_list def test_init(self, cfg, classifiers_list_diff_actions): @@ -31,8 +31,43 @@ def test_init(self, cfg, classifiers_list_diff_actions): assert xncs.back_propagation is not None assert id(xncs.cfg) == id(cfg) assert len(xncs.population) == 4 + assert isinstance(classifiers_list_diff_actions, ClassifiersList) + assert isinstance(xncs.population, ClassifiersList) - def test_distribute_and_update(self, cfg, classifiers_list_diff_actions): + def test_init_no_pop(self, cfg): + xncs = XNCS(cfg) + assert isinstance(xncs.population, ClassifiersList) + + def test_distribute_and_update(self, cfg: Configuration, + classifiers_list_diff_actions, + situation): + xncs = XNCS(cfg, classifiers_list_diff_actions) + action_set = xncs.population.generate_action_set(0) + assert action_set is not None + assert action_set.fittest_classifier is not None + xncs._distribute_and_update(action_set, "####", "####", 0.1) + # update should happen because effect matched inserted vector + assert len(xncs.back_propagation.classifiers_for_update) == 1 + + def test_distribute_and_update_diff(self, cfg: Configuration, + classifiers_list_diff_actions, + situation): xncs = XNCS(cfg, classifiers_list_diff_actions) - xncs._distribute_and_update(classifiers_list_diff_actions, "1100", 0.1) - assert len(xncs.back_propagation.update_vectors) == 4 + action_set = xncs.population.generate_action_set(0) + assert action_set is not None + assert action_set.fittest_classifier is not None + xncs._distribute_and_update(action_set, situation, "1111", 0.1) + assert len(xncs.back_propagation.classifiers_for_update) == 1 + + def test_correct_type_population(self, cfg): + xncs = XNCS(cfg) + assert isinstance(xncs.population, ClassifiersList) + + def test_correct_type_classifier(self, cfg): + xncs = XNCS(cfg) + xncs.population.insert_in_population( + Classifier(cfg, Condition("1100"), 0, 0) + ) + assert isinstance(xncs.population[0], Classifier) + assert xncs.population[0].effect is None + diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index 50509833..4d2388b2 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -9,40 +9,60 @@ class TestBackpropagation: @pytest.fixture def cfg(self): - return Configuration(lmc=2, lem=0.2, number_of_actions=4) + return Configuration(lmc=4, lem=20, number_of_actions=4, update_percentage=0.5) - def test_init(self, cfg): + @pytest.fixture + def situation(self): + return "1100" + + @pytest.fixture + def next_situation(self): + return "1111" + + @pytest.fixture + def classifiers_list_diff_actions(self, cfg, situation): + classifiers_list = ClassifiersList(cfg) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0, Effect(situation))) + return classifiers_list + + def test_update_effect_from_action_set(self, cfg, classifiers_list_diff_actions, next_situation): + classifiers_list_diff_actions[0].fitness = 1000 + classifiers_list_diff_actions[0].effect = Effect(next_situation) + classifiers_list_diff_actions[1].fitness = 0 + classifiers_list_diff_actions[2].fitness = 0 + + bp = Backpropagation(cfg) + bp.update_effect(classifiers_list_diff_actions, classifiers_list_diff_actions.fittest_classifier.effect) + assert classifiers_list_diff_actions[1].effect == classifiers_list_diff_actions.fittest_classifier.effect + + def test_update_effect(self, cfg, classifiers_list_diff_actions, next_situation): + classifiers_list_diff_actions[0].fitness = 1000 + classifiers_list_diff_actions[1].fitness = 0 + classifiers_list_diff_actions[2].fitness = 0 bp = Backpropagation(cfg) - assert id(bp.cfg) == id(cfg) + bp.update_effect(classifiers_list_diff_actions, next_situation) + assert classifiers_list_diff_actions[1].effect == Effect(next_situation) - def test_insert(self, cfg): + def test_insertion(self, cfg, classifiers_list_diff_actions): bp = Backpropagation(cfg) - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) - ef = Effect("0110") - bp.insert_into_bp(cl, ef) - assert id(bp.classifiers_for_update[0]) == id(cl) - assert id(bp.update_vectors[0]) == id(ef) - assert bp.classifiers_for_update[0] == cl - assert bp.update_vectors[0] == ef - - def test_update(self, cfg): + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 4 + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 4 + + def test_deletion(self, cfg, classifiers_list_diff_actions): bp = Backpropagation(cfg) - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) - ef = Effect("0110") - bp.insert_into_bp(cl, ef) - bp.update_bp() - assert cl.effect == ef - bp.insert_into_bp(cl, ef) - bp.update_bp() - assert cl.effect == ef - assert cl.error != cfg.initial_error - - def test_update(self, cfg): + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 4 + bp.classifiers_for_update[1][2] = 1 + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 3 + + def test_errors(self, cfg: Configuration, classifiers_list_diff_actions): bp = Backpropagation(cfg) - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) - ef = Effect("0110") - bp.insert_into_bp(cl, ef) - bp.check_and_update() - assert cl.effect is None - bp.check_and_update() - assert cl.effect is not None + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) + assert classifiers_list_diff_actions[0].error != cfg.initial_error +