Skip to content

GitLab

Unverified Commit e6e15f67 authored by Rosneft rosneft's avatar Rosneft rosneft Committed by GitHub
Browse files

Deps ups 0.7.3.1 (#122) 

* Deps ups
* Full tree feature
parent 2e7d8078
master
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 20485 additions and 164 deletions
+20485 -164
app/api/composer/controller.py
View file @ e6e15f67
......@@ -7,6 +7,7 @@ from flask_restx import Namespace, Resource
from .history_convert_utils import history_to_graph
from .schema import ComposingHistoryGraphSchema, ComposingStartSchema
from .service import composer_history_for_case
from utils import clean_case_id
from ..pipelines.service import pipeline_by_uid
from ..showcase.service import create_new_case_async
from ..showcase.showcase_utils import showcase_item_from_db
......@@ -24,7 +25,9 @@ class ComposerHistoryResource(Resource):
@responds(schema=ComposingHistoryGraphSchema, many=False)
def get(self, case_id: str) -> Dict[str, Any]:
"""Get history of the composer for the specific dataset"""
graph_dict = history_to_graph(composer_history_for_case(case_id))
show_full = '_full' in case_id
case_id = clean_case_id(case_id)
graph_dict = history_to_graph(composer_history_for_case(case_id), show_full)
return graph_dict
......@@ -43,6 +46,7 @@ class ComposerRestartResource(Resource):
async def start_async():
data = request.get_json()
case_id = data['case_id']
case_id = clean_case_id(case_id)
initial_uid = data['initial_uid']
gen_index = data.get('gen_index', None)
original_uid = data.get('original_uid', None)
......
app/api/composer/history_convert_utils.py
View file @ e6e15f67
......@@ -6,8 +6,8 @@ from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
from matplotlib import pyplot as plt
def history_to_graph(history: OptHistory) -> Dict[str, Any]:
all_nodes = _create_operators_and_nodes(history)
def history_to_graph(history: OptHistory, show_all: bool = False) -> Dict[str, Any]:
all_nodes = _create_operators_and_nodes(history, show_all)
all_nodes, edges = _create_edges(all_nodes)
all_nodes = _clear_tmp_fields(all_nodes)
......@@ -61,10 +61,10 @@ def _process_operator(all_nodes, operator, individual, o_id, gen_id, prev_operat
return all_nodes
def _create_all_individuals_for_population(history, all_nodes, gen_id, order_id, uids_to_show):
def _create_all_individuals_for_population(history, all_nodes, gen_id, order_id, uids_to_show, show_all):
for ind_id in range(len(history.individuals[gen_id])):
individual = history.individuals[gen_id][ind_id]
if individual.uid not in uids_to_show or gen_id > uids_to_show[individual.uid]:
if not show_all and (individual.uid not in uids_to_show or gen_id > uids_to_show[individual.uid]):
continue
# add pipelines as node
......@@ -90,7 +90,7 @@ def _create_all_individuals_for_population(history, all_nodes, gen_id, order_id,
return all_nodes, order_id
def _create_operators_and_nodes(history):
def _create_operators_and_nodes(history, show_all):
all_nodes = []
current_order_id = 0
if hasattr(history, 'final_choices') and history.final_choices:
......@@ -121,13 +121,14 @@ def _create_operators_and_nodes(history):
o_id = 0
all_nodes, current_order_id = _create_all_individuals_for_population(history, all_nodes, gen_id,
current_order_id,
uid_to_last_generation_map)
uid_to_last_generation_map, show_all)
if gen_id == 0:
continue
for ind_id, individual in enumerate(generation):
if individual.native_generation != gen_id:
continue
if individual.uid not in uid_to_last_generation_map or gen_id > uid_to_last_generation_map[individual.uid]:
if not show_all and (individual.uid not in uid_to_last_generation_map or
gen_id > uid_to_last_generation_map[individual.uid]):
continue
# add evo operators as nodes
......
app/api/composer/service.py
View file @ e6e15f67
import datetime
import itertools
import json
import multiprocessing
import sys
from functools import lru_cache
from os import PathLike
from pathlib import Path
from typing import Optional
......@@ -11,22 +9,21 @@ from typing import Optional
from bson import json_util
from fedot.api.main import Fedot
from fedot.core.pipelines.adapters import PipelineAdapter
from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
from fedot.core.pipelines.pipeline import Pipeline
from fedot.core.pipelines.template import PipelineTemplate
from fedot.core.repository.tasks import TsForecastingParams
from flask import current_app
from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
from app.api.data.service import get_input_data
from app.api.pipelines.service import create_pipeline, is_pipeline_exists
from app.api.showcase.showcase_utils import showcase_item_from_db
from app.singletons.db_service import DBServiceSingleton
from utils import project_root, threading_lock
from utils import project_root, clean_case_id
# @threading_lock
# @lru_cache(maxsize=128) #CHEC
def composer_history_for_case(case_id: str, validate_history: bool = False) -> OptHistory:
case_id = clean_case_id(case_id)
case = showcase_item_from_db(case_id)
if case is None:
raise ValueError(f'Showcase item for case_id={case_id} is None but should exist')
......
app/api/data/service.py
View file @ e6e15f67
......@@ -10,7 +10,23 @@ from fedot.core.data.data import DataTypesEnum, InputData
from fedot.core.repository.tasks import Task, TaskParams, TsForecastingParams
from utils import project_root
datasets = {}
datasets = {
'scoring': {
'train': 'data/scoring/scoring_train.csv',
'test': 'data/scoring/scoring_test.csv',
'data_type': DataTypesEnum.table
},
'metocean': {
'train': 'data/metocean/metocean_train.csv',
'test': 'data/metocean/metocean_test.csv',
'data_type': DataTypesEnum.ts
},
'oil': {
'train': 'data/oil/oil_train.csv',
'test': 'data/oil/oil_test.csv',
'data_type': DataTypesEnum.table
}
}
data_types = {
'ts': DataTypesEnum.ts,
......
app/api/meta/service.py
View file @ e6e15f67
......@@ -2,7 +2,7 @@ from typing import List
from fedot.core.repository.operation_types_repository import \
OperationTypesRepository
from fedot.core.repository.quality_metrics_repository import (
from fedot.core.repository.metrics_repository import (
ClassificationMetricsEnum, ClusteringMetricsEnum, RegressionMetricsEnum)
from fedot.core.repository.tasks import TaskTypesEnum
......
app/api/showcase/service.py
View file @ e6e15f67
......@@ -11,9 +11,12 @@ from init.init_cases import add_case_to_db
from .models import ShowcaseItem, Metadata
from .showcase_utils import prepare_icon_path, showcase_item_from_db
from ..analytics.pipeline_analytics import get_metrics_for_pipeline, get_metrics_for_golem_individual
from utils import clean_case_id
def showcase_full_item_by_uid(case_id: str) -> Optional[ShowcaseItem]:
case_id = clean_case_id(case_id)
dumped_item: Optional[Dict[str, Any]] = DBServiceSingleton().try_find_one('cases', {'case_id': case_id})
if dumped_item is None:
return None
......@@ -83,6 +86,8 @@ def all_showcase_items_ids(with_custom: bool = False) -> List[str]:
def create_new_case(case_id, case_meta_json, opt_history_json, initial_pipeline: Pipeline = None, original_history=None,
modifed_generation_index=None, original_uid=None, is_golem_history=False):
from init.init_history import _init_composer_history_for_case
case_id = clean_case_id(case_id)
case = ShowcaseItem(
case_id=case_id,
title=case_id,
......
app/api/showcase/showcase_utils.py
View file @ e6e15f67
......@@ -5,9 +5,12 @@ from app.singletons.db_service import DBServiceSingleton
from flask import url_for
from .models import ShowcaseItem
from utils import clean_case_id
def showcase_item_from_db(case_id: str) -> Optional[ShowcaseItem]:
case_id = clean_case_id(case_id)
dumped_item = DBServiceSingleton().try_find_one('cases', {'case_id': case_id})
if dumped_item is None:
return None
......
data/metocean/metocean_ts_forecasting.json
View file @ e6e15f67
......@@ -7,7 +7,7 @@
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.quality_metrics_repository/ComplexityMetricsEnum"
"_class_path": "fedot.core.repository.metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.objective/Objective"
......
data/oil/oil_regression.json
View file @ e6e15f67
......@@ -7,7 +7,7 @@
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.quality_metrics_repository/ComplexityMetricsEnum"
"_class_path": "fedot.core.repository.metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.objective/Objective"
......
data/scoring/scoring_classification.json
View file @ e6e15f67
......@@ -7,7 +7,7 @@
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.quality_metrics_repository/ComplexityMetricsEnum"
"_class_path": "fedot.core.repository.metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.objective/Objective"
......
frontend/src/API/composer/composerInterface.ts
View file @ e6e15f67
......@@ -8,7 +8,7 @@ export interface IHistoryEdge {
target: string;
}
export interface IHistoryNodeIndividual {
export interface IHistoryNodeIndividual{
uid: string;
type: string;
individual_id: any;
......
requirements.txt
View file @ e6e15f67
# FEDOT
fedot==0.7.2
fedot==0.7.3.1
# Unique requirements
flask-socketio==5.3.2
......
scripts/ind_history.json 0 → 100644
View file @ e6e15f67
This diff is collapsed.
scripts/upload_custom_history.py
View file @ e6e15f67
......@@ -6,8 +6,8 @@ from uuid import uuid4
DOMAIN = 'https://fedot.onti.actcognitive.org'
# DOMAIN = 'http://127.0.0.1:5000'
BASE_PATH = Path(r"...")
FILE_NAME = '... .json'
BASE_PATH = Path(r".")
FILE_NAME = 'ind_history.json'
if __name__ == '__main__':
case_id = FILE_NAME.replace('.json', '') + str(uuid4())
......@@ -18,6 +18,7 @@ if __name__ == '__main__':
new_case = {
'case': {
'case_id': case_id,
'task': 'fedot'
},
'history': history_json
}
......
test/fixtures/dict_fitted_operations.json
View file @ e6e15f67
This diff is collapsed.
test/fixtures/history.json
View file @ e6e15f67
......@@ -2,31 +2,7 @@
{
"history_id": "scoring",
"history_json": {
"_objective": {
"is_multi_objective": false,
"metrics": [
{
"_class_path": "fedot.core.composer.metrics/QualityMetric.get_value"
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.quality_metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.metrics_objective/MetricsObjective"
},
"archive_history": [
[
"b36b553a-9b23-4b6d-a730-a752f7c258d1"
],
[
"b36b553a-9b23-4b6d-a730-a752f7c258d1"
],
[
"b36b553a-9b23-4b6d-a730-a752f7c258d1"
]
],
"individuals": [
"_generations": [
{
"data": [
"b36b553a-9b23-4b6d-a730-a752f7c258d1"
......@@ -61,6 +37,30 @@
"_class_path": "golem.core.optimisers.opt_history_objects.generation/Generation"
}
],
"_objective": {
"is_multi_objective": false,
"metrics": [
{
"_class_path": "fedot.core.composer.metrics/QualityMetric.get_value"
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.metrics_objective/MetricsObjective"
},
"archive_history": [
[
"b36b553a-9b23-4b6d-a730-a752f7c258d1"
],
[
"b36b553a-9b23-4b6d-a730-a752f7c258d1"
],
[
"b36b553a-9b23-4b6d-a730-a752f7c258d1"
]
],
"individuals_pool": [
{
"fitness": {
......@@ -424,31 +424,7 @@
{
"history_id": "metocean",
"history_json": {
"_objective": {
"is_multi_objective": false,
"metrics": [
{
"_class_path": "fedot.core.composer.metrics/QualityMetric.get_value"
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.quality_metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.metrics_objective/MetricsObjective"
},
"archive_history": [
[
"79b9c2c7-22ce-432f-bd18-604fedd013be"
],
[
"79b9c2c7-22ce-432f-bd18-604fedd013be"
],
[
"79b9c2c7-22ce-432f-bd18-604fedd013be"
]
],
"individuals": [
"_generations": [
{
"data": [
"79b9c2c7-22ce-432f-bd18-604fedd013be"
......@@ -483,6 +459,30 @@
"_class_path": "golem.core.optimisers.opt_history_objects.generation/Generation"
}
],
"_objective": {
"is_multi_objective": false,
"metrics": [
{
"_class_path": "fedot.core.composer.metrics/QualityMetric.get_value"
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.metrics_objective/MetricsObjective"
},
"archive_history": [
[
"79b9c2c7-22ce-432f-bd18-604fedd013be"
],
[
"79b9c2c7-22ce-432f-bd18-604fedd013be"
],
[
"79b9c2c7-22ce-432f-bd18-604fedd013be"
]
],
"individuals_pool": [
{
"fitness": {
......@@ -871,31 +871,7 @@
{
"history_id": "oil",
"history_json": {
"_objective": {
"is_multi_objective": false,
"metrics": [
{
"_class_path": "fedot.core.composer.metrics/QualityMetric.get_value"
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.quality_metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.metrics_objective/MetricsObjective"
},
"archive_history": [
[
"a45d94bb-f32a-4a38-9937-9432b755c1d3"
],
[
"a45d94bb-f32a-4a38-9937-9432b755c1d3"
],
[
"a45d94bb-f32a-4a38-9937-9432b755c1d3"
]
],
"individuals": [
"_generations": [
{
"data": [
"a45d94bb-f32a-4a38-9937-9432b755c1d3"
......@@ -930,6 +906,30 @@
"_class_path": "golem.core.optimisers.opt_history_objects.generation/Generation"
}
],
"_objective": {
"is_multi_objective": false,
"metrics": [
{
"_class_path": "fedot.core.composer.metrics/QualityMetric.get_value"
},
{
"value": "node_number",
"_class_path": "fedot.core.repository.metrics_repository/ComplexityMetricsEnum"
}
],
"_class_path": "fedot.core.optimisers.objective.metrics_objective/MetricsObjective"
},
"archive_history": [
[
"a45d94bb-f32a-4a38-9937-9432b755c1d3"
],
[
"a45d94bb-f32a-4a38-9937-9432b755c1d3"
],
[
"a45d94bb-f32a-4a38-9937-9432b755c1d3"
]
],
"individuals_pool": [
{
"fitness": {
......
test/fixtures/pipelines.json
View file @ e6e15f67
......@@ -54,11 +54,15 @@
"operation_name": "XGBClassifier",
"custom_params": {
"eval_metric": "mlogloss",
"nthread": -1
"nthread": 1,
"n_jobs": 1,
"verbose": 0
},
"params": {
"eval_metric": "mlogloss",
"nthread": -1
"nthread": 1,
"n_jobs": 1,
"verbose": 0
},
"nodes_from": [
4,
......@@ -76,11 +80,15 @@
"operation_name": "XGBClassifier",
"custom_params": {
"eval_metric": "mlogloss",
"nthread": -1
"nthread": 1,
"n_jobs": 1,
"verbose": 0
},
"params": {
"eval_metric": "mlogloss",
"nthread": -1
"nthread": 1,
"n_jobs": 1,
"verbose": 0
},
"nodes_from": [
2,
......@@ -146,11 +154,15 @@
"operation_name": "XGBClassifier",
"custom_params": {
"eval_metric": "mlogloss",
"nthread": -1
"nthread": 1,
"n_jobs": 1,
"verbose": 0
},
"params": {
"eval_metric": "mlogloss",
"nthread": -1
"nthread": 1,
"n_jobs": 1,
"verbose": 0
},
"nodes_from": [
1,
......@@ -167,7 +179,7 @@
"preprocessing",
"data_preprocessor.pkl"
],
"descriptive_id": "(((/n_knn;;/n_lda;)/n_xgboost_{'eval_metric': 'mlogloss', 'nthread': -1};;/n_logit;)/n_xgboost_{'eval_metric': 'mlogloss', 'nthread': -1};;(/n_logit;;/n_pca_{'svd_solver': 'full', 'n_components': 0.7};)/n_knn;)/n_xgboost_{'eval_metric': 'mlogloss', 'nthread': -1}",
"descriptive_id": "(((/n_knn;;/n_lda;)/n_xgboost_{'eval_metric': 'mlogloss', 'nthread': 1, 'n_jobs': 1, 'verbose': 0};;/n_logit;)/n_xgboost_{'eval_metric': 'mlogloss', 'nthread': 1, 'n_jobs': 1, 'verbose': 0};;(/n_logit;;/n_pca_{'svd_solver': 'full', 'n_components': 0.7};)/n_knn;)/n_xgboost_{'eval_metric': 'mlogloss', 'nthread': 1, 'n_jobs': 1, 'verbose': 0}",
"fitted_operation_path": "operation_0",
"individual_id": "best_scoring_pipeline"
},
......@@ -211,10 +223,10 @@
"operation_type": "lagged",
"operation_name": "LaggedTransformationImplementation",
"custom_params": {
"window_size": 10
"window_size": 12
},
"params": {
"window_size": 10
"window_size": 12
},
"nodes_from": [],
"fitted_operation_path": [
......@@ -243,7 +255,7 @@
"preprocessing",
"data_preprocessor.pkl"
],
"descriptive_id": "(/n_lagged_{'window_size': 10};)/n_linear",
"descriptive_id": "(/n_lagged_{'window_size': 12};)/n_linear",
"fitted_operation_path": "operation_0",
"individual_id": "metocean_baseline"
},
......
test/unit/mocks/analytics_service_mocks.py
View file @ e6e15f67
......@@ -4,7 +4,7 @@ from uuid import uuid4
import numpy as np
from golem.core.optimisers.fitness import SingleObjFitness
from golem.core.utilities.data_structures import UniqueList
from golem.utilities.data_structures import UniqueList
from fedot.core.utils import DEFAULT_PARAMS_STUB
......
utils.py
View file @ e6e15f67
......@@ -27,3 +27,8 @@ def threading_lock(function):
if callable(getattr(function, method)) and not method.startswith("__")]
return wrapper
def clean_case_id(case_id: str):
case_id = case_id.replace('_full', '')
return case_id
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment