1# Copyright (c) 2021 Huawei Device Co., Ltd. 2# Licensed under the Apache License, Version 2.0 (the "License"); 3# you may not use this file except in compliance with the License. 4# You may obtain a copy of the License at 5# 6# http://www.apache.org/licenses/LICENSE-2.0 7# 8# Unless required by applicable law or agreed to in writing, software 9# distributed under the License is distributed on an "AS IS" BASIS, 10# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11# See the License for the specific language governing permissions and 12# limitations under the License. 13 14# frozen_string_literal: true 15 16require 'pathname' 17 18class Spec 19 attr_reader :spec, :orphaned 20 21 def initialize(spec, non_testable, testdir) 22 @spec = spec 23 @orphaned = [] 24 @non_testable = non_testable 25 @testdir = Pathname.new(testdir) 26 prepare_spec 27 prepare_non_testable 28 end 29 30 def add_testfile(testfile) 31 proc_testfile(testfile) 32 rescue StandardError => e 33 @orphaned << { 'file' => rel_path(testfile), 'error' => e, 'comment' => '' } 34 end 35 36 def compute_coverage 37 result = { 'tests' => compute_tests, 'assertions' => compute_assertions, 'coverage_by_groups' => [] } 38 result['coverage_metric'] = (result['assertions']['covered'].to_f / result['assertions']['testable']).round(2) 39 @spec['groups'].each do |g| 40 result['coverage_by_groups'] << { 'title' => g['title'], 'coverage_metric' => g['coverage_metric'] } 41 end 42 result 43 end 44 45 def uncovered 46 groups = [] 47 @spec['groups'].each do |g| 48 not_covered_i = g['instructions'].reject { |i| i['tests'].length.positive? || i['non_testable'] }.map { |i| except(i, %w[tests non_testable]) } 49 not_covered_d = g['description_tests'].reject { |d| d['tests'].length.positive? || d['non_testable'] } 50 not_covered_e = g['exceptions_tests'].reject { |d| d['tests'].length.positive? || d['non_testable'] } 51 not_covered_v = g['verification_tests'].reject { |d| d['tests'].length.positive? || d['non_testable'] } 52 next unless not_covered_i.any? || not_covered_d.any? || not_covered_e.any? || not_covered_v.any? 53 54 result = { 'title' => g['title'] } 55 if not_covered_d.any? 56 result['description'] = not_covered_d.map { |d| d['assertion'] }.join('. ').gsub(/\n/, ' ').rstrip 57 end 58 result['instructions'] = not_covered_i if not_covered_i.any? 59 result['exceptions'] = not_covered_e.map { |e| e['exception'] } if not_covered_e.any? 60 result['verification'] = not_covered_v.map { |v| v['verification'] } if not_covered_v.any? 61 groups << result 62 end 63 { 'groups' => groups } 64 end 65 66 private 67 68 def rel_path(testfile) 69 Pathname.new(testfile).relative_path_from(@testdir).to_s 70 end 71 72 def prepare_non_testable 73 @non_testable['groups']&.each do |ntg| 74 spec_group = @spec['groups'].find { |sg| sg['title'] == ntg['title'] } 75 next if spec_group.nil? 76 77 ntg['description'] && split(ntg['description']).map do |ntda| 78 spec_description = spec_group['description_tests'].find { |sd| same?(sd['assertion'], ntda) } 79 next if spec_description.nil? 80 81 spec_description['non_testable'] = true 82 end 83 84 ntg['instructions']&.each do |nti| 85 spec_instruction = spec_group['instructions'].find { |si| except(si, %w[tests non_testable]) == nti } 86 next if spec_instruction.nil? 87 88 spec_instruction['non_testable'] = true 89 end 90 91 ntg['exceptions']&.each do |nte| 92 spec_exception = spec_group['exceptions_tests'].find { |se| se['exception'] == nte } 93 next if spec_exception.nil? 94 95 spec_exception['non_testable'] = true 96 end 97 98 ntg['verification']&.each do |ntv| 99 spec_verification = spec_group['verification_tests'].find { |sv| sv['verification'] == ntv } 100 next if spec_verification.nil? 101 102 spec_verification['non_testable'] = true 103 end 104 end 105 end 106 107 def prepare_spec 108 @spec['groups'].each do |g| 109 g['instructions'].each do |i| 110 i['tests'] = [] 111 i['non_testable'] = false 112 end 113 g['description_tests'] = split(g['description']).map do |da| 114 { 'assertion' => da, 'tests' => [], 'non_testable' => false } 115 end 116 g['exceptions_tests'] = g['exceptions'].map { |e| { 'exception' => e, 'tests' => [], 'non_testable' => false } } 117 g['verification_tests'] = g['verification'].map { |v| { 'verification' => v, 'tests' => [], 'non_testable' => false } } 118 end 119 end 120 121 def split(description) 122 result = [] 123 small = false 124 description.split(/\./).each do |p| 125 if small 126 result[-1] += '.' + p 127 small = false if p.length > 5 128 elsif p.length > 5 129 result << p.lstrip 130 else 131 if result.length.zero? 132 result << p.lstrip 133 else 134 result[-1] += '.' + p 135 end 136 small = true 137 end 138 end 139 result 140 end 141 142 def same?(str1, str2) 143 str1.tr('^A-Za-z0-9', '').downcase == str2.tr('^A-Za-z0-9', '').downcase 144 end 145 146 def proc_testfile(testfile) 147 raw_data = read_test_data(testfile) 148 tdata = YAML.safe_load(raw_data) 149 if tdata.class != array 150 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Bad format, expected array of titles', 'comment' => raw_data } 151 return 152 end 153 154 tdata.each do |tg| 155 group = @spec['groups'].find { |x| x['title'] == tg['title'] } 156 if group.nil? 157 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Group with given title not found in the spec', 'comment' => tg } 158 next 159 end 160 161 proc_test_instructions(tg, group, testfile) 162 proc_test_description(tg, group, testfile) 163 proc_test_exceptions(tg, group, testfile) 164 proc_test_verification(tg, group, testfile) 165 end 166 end 167 168 def proc_test_instructions(testgroup, specgroup, testfile) 169 testgroup['instructions']&.each do |ti| 170 gi = specgroup['instructions'].find { |x| except(x, %w[tests non_testable]) == ti } 171 if gi.nil? 172 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given instruction not found in the spec', 'comment' => ti } 173 next 174 end 175 if gi['non_testable'] 176 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given instruction is non-testable', 'comment' => ti } 177 next 178 end 179 gi['tests'] << rel_path(testfile) 180 end 181 end 182 183 def proc_test_description(testgroup, specgroup, testfile) 184 testgroup['description'] && split(testgroup['description']).map do |tda| 185 sd = specgroup['description_tests']&.find { |sda| same?(sda['assertion'], tda) } 186 if sd.nil? 187 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given description assertion not found in the spec', 'comment' => tda } 188 next 189 end 190 if sd['non_testable'] 191 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given description is non-testable', 'comment' => tda } 192 next 193 end 194 sd['tests'] << rel_path(testfile) 195 end 196 end 197 198 def proc_test_exceptions(testgroup, specgroup, testfile) 199 testgroup['exceptions']&.each do |te| 200 se = specgroup['exceptions_tests'].find { |x| x['exception'] == te } 201 if se.nil? 202 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given exception assertion not found in the spec', 'comment' => te } 203 next 204 end 205 if se['non_testable'] 206 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given exception assertion is non-testable', 'comment' => te } 207 next 208 end 209 se['tests'] << rel_path(testfile) 210 end 211 end 212 213 def proc_test_verification(testgroup, specgroup, testfile) 214 testgroup['verification']&.each do |tv| 215 sv = specgroup['verification_tests'].find { |x| x['verification'] == tv } 216 if sv.nil? 217 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given verification assertion not found in the spec', 'comment' => tv } 218 next 219 end 220 if sv['non_testable'] 221 @orphaned << { 'file' => rel_path(testfile), 'error' => 'Given verification assertion is non-testable', 'comment' => tv } 222 next 223 end 224 sv['tests'] << rel_path(testfile) 225 end 226 end 227 228 def read_test_data(filename) 229 lines_array = [] 230 started = false 231 File.readlines(filename).each do |line| 232 if started 233 break if line[0] != '#' 234 235 lines_array << line[1..-1] 236 else 237 started = true if line[0..3] == '#---' 238 end 239 end 240 lines_array.join("\n") 241 end 242 243 def except(hash, keys) 244 hash.dup.tap do |x| 245 keys.each { |key| x.delete(key) } 246 end 247 end 248 249 def compute_tests 250 # Count mismatched tests 251 orphaned = @orphaned.map { |x| x['file'] }.uniq 252 253 # Count accepted tests 254 accepted = [] 255 @spec['groups'].each { |g| g['instructions']&.each { |i| accepted += i['tests'] } } 256 @spec['groups'].each { |g| g['description_tests']&.each { |i| accepted += i['tests'] } } 257 accepted = accepted.uniq 258 259 { 'counted_for_coverage' => accepted.length, 'orphaned' => orphaned.length, 'total' => (orphaned + accepted).uniq.length } 260 end 261 262 def compute_assertions 263 res = { 'testable' => 0, 'non_testable' => 0, 'covered' => 0, 'not_covered' => 0 } 264 265 @spec['groups'].each do |g| 266 g_testable = 0 267 g_non_testable = 0 268 g_covered = 0 269 g_not_covered = 0 270 271 %w[instructions description_tests exceptions_tests verification_tests].each do |k| 272 g[k].each do |i| 273 if i['non_testable'] 274 g_non_testable += 1 275 else 276 g_testable += 1 277 i['tests'].length.positive? ? g_covered += 1 : g_not_covered += 1 278 end 279 end 280 end 281 282 if g_testable > 0 283 g['coverage_metric'] = (g_covered.to_f / g_testable).round(2) 284 else 285 g['coverage_metric'] = 'Not testable' 286 end 287 288 res['testable'] += g_testable 289 res['non_testable'] += g_non_testable 290 res['covered'] += g_covered 291 res['not_covered'] += g_not_covered 292 end 293 294 res 295 end 296end 297