Elizaveta Lutsenko · @LizLutsenko · 80741f45 · 44fce4d8 · 44fce4d8 · d3725374
--- a/fedot/api/api_utils/api_data_analyser.py
+++ b/fedot/api/api_utils/api_data_analyser.py
+       All methods are inplace to prevent unnecessary copy of large datasets
+       It functionality is:
+       1) Cut large datasets to prevent memory stackoverflow
+       2) Use label encoder with tree models instead OneHot when summary cardinality of categorical features is high
+   """
+   def __init__(self, safe_mode, preprocessor: ApiDataProcessor):
+       self.safe_mode = safe_mode
+       self.max_size = 600000
+       self.max_cat_cardinality = 1000
+       self.data_preprocessor = preprocessor
+
+   def safe_preprocess(self, input_data: InputData):
+       """ Preforms preprocessing to preventing crash due of memory stackoverflow if safe_mode on
+           :param input_data - data for preprocessing
+
+       """
--- a/fedot/preprocessing/preprocessing.py
+++ b/fedot/preprocessing/preprocessing.py
       Encode categorical features to numerical. In additional,
       save encoders to use later for prediction data.

       :param data: data to transform
+       :param is_label: is we use a specific label encoder
       :return encoder: operation for preprocessing categorical features
       """

-       transformed, encoder = self._create_encoder(data)
+       transformed, encoder = self._create_encoder(data, is_label)
       data.features = transformed
-       # Store encoder to make prediction in th future
+       # Store encoder to make prediction in the future
       self.features_encoder = encoder

+   def cut_dataset(self, data: InputData, border):
--- a/fedot/api/api_utils/params.py
+++ b/fedot/api/api_utils/params.py

+   def accept_recommendations(self, input_data: InputData, recommendations: Dict):
+       if 'label' in recommendations:
+           self.log.info("Change preset due of label encoding")
+           return self.change_preset_for_labeled_data(input_data.task)
+       else:
+           param_dict = {
+               'task': self.task,
+               'logger': self.log,
+               'metric_name': self.metric_name,
+               'composer_metric': self.metric_to_compose
+           }
+           return {**param_dict, **self.api_params}
+
+   def change_preset_for_labeled_data(self, task: Task):
+       preset_name = 'tree_reg' if task.task_type == 'regression' else 'tree_class'
--- a/fedot/api/main.py
+++ b/fedot/api/main.py
       """

       self.target = target
-       self.train_data = self.data_processor.define_data(features=features, target=target, is_predict=False)
+       self.train_data = deepcopy(self.data_processor.define_data(features=features, target=target, is_predict=False))
+       full_train_not_preprocessed = deepcopy(self.train_data)
--- a/fedot/core/repository/data/model_repository.json
+++ b/fedot/core/repository/data/model_repository.json
     ]
   },
   "rf": {
-     "meta": "sklearn_class"
+     "meta": "sklearn_class",
+     "tags": ["tree"]
--- a/test/unit/api/test_api_safety.py
+++ b/test/unit/api/test_api_safety.py
+   assert data.features[0, 0] == 'a'
+
+
+def test_api_fit_predict_with_pseudo_large_dataset_with_label_correct():
+   model = Fedot(problem="classification",
+                 composer_params=composer_params)
+   model.data_analyser.max_cat_cardinality = 5
+   model.data_analyser.max_size = 18
+   data = get_small_cat_data()
+   model.fit(features=data)
+   model.predict(features=data)
+   assert len(model.api_params['available_operations']) == 13
+   assert 'logit' not in model.api_params['available_operations']
+
+
+def test_api_fit_predict_with_pseudo_large_dataset_with_onehot_correct():
--- a/fedot/api/main.py
+++ b/fedot/api/main.py
       """

       self.target = target
-       self.train_data = self.data_processor.define_data(features=features, target=target, is_predict=False)
+       self.train_data = deepcopy(self.data_processor.define_data(features=features, target=target, is_predict=False))
+       full_train_not_preprocessed = deepcopy(self.train_data)
--- a/fedot/api/api_utils/params.py
+++ b/fedot/api/api_utils/params.py

+   def accept_recommendations(self, input_data: InputData, recommendations: Dict):
+       if 'label' in recommendations:
+           self.log.info("Change preset due of label encoding")
+           return self.change_preset_for_labeled_data(input_data.task)
+       else:
+           param_dict = {
+               'task': self.task,
+               'logger': self.log,
+               'metric_name': self.metric_name,
+               'composer_metric': self.metric_to_compose
+           }
+           return {**param_dict, **self.api_params}
+
+   def change_preset_for_labeled_data(self, task: Task):
+       preset_name = 'tree_reg' if task.task_type == 'regression' else 'tree_class'
--- a/fedot/api/main.py
+++ b/fedot/api/main.py
       """

       self.target = target
-       self.train_data = self.data_processor.define_data(features=features, target=target, is_predict=False)
+       self.train_data = deepcopy(self.data_processor.define_data(features=features, target=target, is_predict=False))
+       full_train_not_preprocessed = deepcopy(self.train_data)
--- a/fedot/core/repository/data/model_repository.json
+++ b/fedot/core/repository/data/model_repository.json
     ]
   },
   "rf": {
-     "meta": "sklearn_class"
+     "meta": "sklearn_class",
+     "tags": ["tree"]
--- a/test/unit/api/test_api_safety.py
+++ b/test/unit/api/test_api_safety.py
+   assert data.features[0, 0] == 'a'
+
+
+def test_api_fit_predict_with_pseudo_large_dataset_with_label_correct():
+   model = Fedot(problem="classification",
+                 composer_params=composer_params)
+   model.data_analyser.max_cat_cardinality = 5
+   model.data_analyser.max_size = 18
+   data = get_small_cat_data()
+   model.fit(features=data)
+   model.predict(features=data)
+   assert len(model.api_params['available_operations']) == 13
+   assert 'logit' not in model.api_params['available_operations']
+
+
+def test_api_fit_predict_with_pseudo_large_dataset_with_onehot_correct():