• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15
16"""Verification pipeline engine."""
17
18import logging
19import pytest
20
21from .components.icomponent import IDataComponent, IBuilderComponent, IExectorComponent, \
22    IVerifierComponent, IFIPolicyComponent, IERPolicyComponent, IComponent, \
23    IFacadeComponent
24from .utils import keyword
25
26
27def mindspore_test(verification_pipeline):
28    """
29    Run verification pipeline.
30
31    Args:
32        verification_pipeline (list): Pipeline designed to do verification.
33
34    Returns:
35    """
36
37    def decorate(get_verification_set):
38        verification_set = get_verification_set()
39
40        facade_components = []
41        data_components = []
42        builder_components = []
43        executor_components = []
44        verifier_components = []
45        fi_policy_components = []
46        er_policy_components = []
47        for component in verification_pipeline:
48            if issubclass(component, IFacadeComponent):
49                facade_components.append(component)
50            elif issubclass(component, IDataComponent):
51                data_components.append(component)
52            elif issubclass(component, IBuilderComponent):
53                builder_components.append(component)
54            elif issubclass(component, IExectorComponent):
55                executor_components.append(component)
56            elif issubclass(component, IVerifierComponent):
57                verifier_components.append(component)
58            elif issubclass(component, IFIPolicyComponent):
59                fi_policy_components.append(component)
60            elif issubclass(component, IERPolicyComponent):
61                er_policy_components.append(component)
62            else:
63                raise Exception(f'{component} is not a instance of {IComponent}')
64
65        for component in facade_components:
66            fc = component(verification_set)
67            verification_set = fc()
68
69        inputs = []
70        for component in data_components:
71            dc = component(verification_set)
72            item = dc()
73            inputs.extend(item)
74
75        if not inputs:
76            logging.warning("Inputs set is empty.")
77
78        functions = []
79        for component in builder_components:
80            bc = component(verification_set)
81            f = bc()
82            functions.extend(f)
83
84        if not functions:
85            logging.warning("Function set is empty.")
86
87        fis = []
88        for component in fi_policy_components:
89            fipc = component(verification_set, functions, inputs)
90            result = fipc()
91            fis.extend(result)
92
93        if not fis:
94            logging.warning("Function inputs pair set is empty.")
95
96        def test_case(args):
97            sut, inputs = args
98
99            results = []
100            for component in executor_components:
101                ec = component(verification_set, sut, inputs)
102                result = ec()
103                results.append(result)
104
105            if not results:
106                logging.warning("Result set is empty.")
107
108            expect_actuals = []
109            for component in er_policy_components:
110                erpc = component(verification_set, verification_set['expect'], results)
111                result = erpc()
112                expect_actuals.extend(result)
113
114            if not expect_actuals:
115                logging.warning("Expect Result pair set is empty.")
116
117            for ea in expect_actuals:
118                for component in verifier_components:
119                    vc = component(verification_set, *ea)
120                    vc()
121
122        def get_tc_name(f, inputs):
123            tc_id = f[keyword.id] + '-' + inputs[keyword.id]
124            group = f[keyword.group] + '-' + inputs[keyword.group]
125            return 'Group_' + group + '-' + 'Id_' + tc_id
126
127        if fis:
128            m = pytest.mark.parametrize('args', fis, ids=lambda fi: get_tc_name(*fi))(test_case)
129            m.__orig__ = get_verification_set
130            return m
131
132    return decorate
133