1// Copyright 2020 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4package gen_tasks_logic 5 6/* 7 This file contains logic related to task/job name schemas. 8*/ 9 10import ( 11 "encoding/json" 12 "fmt" 13 "log" 14 "os" 15 "regexp" 16 "strings" 17) 18 19// parts represents the key/value pairs which make up task and job names. 20type parts map[string]string 21 22// equal returns true if the given part of this job's name equals any of the 23// given values. Panics if no values are provided. 24func (p parts) equal(part string, eq ...string) bool { 25 if len(eq) == 0 { 26 log.Fatal("No values provided for equal!") 27 } 28 v := p[part] 29 for _, e := range eq { 30 if v == e { 31 return true 32 } 33 } 34 return false 35} 36 37// role returns true if the role for this job equals any of the given values. 38func (p parts) role(eq ...string) bool { 39 return p.equal("role", eq...) 40} 41 42// os returns true if the OS for this job equals any of the given values. 43func (p parts) os(eq ...string) bool { 44 return p.equal("os", eq...) 45} 46 47// compiler returns true if the compiler for this job equals any of the given 48// values. 49func (p parts) compiler(eq ...string) bool { 50 return p.equal("compiler", eq...) 51} 52 53// model returns true if the model for this job equals any of the given values. 54func (p parts) model(eq ...string) bool { 55 return p.equal("model", eq...) 56} 57 58// frequency returns true if the frequency for this job equals any of the given 59// values. 60func (p parts) frequency(eq ...string) bool { 61 return p.equal("frequency", eq...) 62} 63 64// cpu returns true if the task's cpu_or_gpu is "CPU" and the CPU for this 65// task equals any of the given values. If no values are provided, cpu returns 66// true if this task runs on CPU. 67func (p parts) cpu(eq ...string) bool { 68 if p["cpu_or_gpu"] == "CPU" { 69 if len(eq) == 0 { 70 return true 71 } 72 return p.equal("cpu_or_gpu_value", eq...) 73 } 74 return false 75} 76 77// gpu returns true if the task's cpu_or_gpu is "GPU" and the GPU for this task 78// equals any of the given values. If no values are provided, gpu returns true 79// if this task runs on GPU. 80func (p parts) gpu(eq ...string) bool { 81 if p["cpu_or_gpu"] == "GPU" { 82 if len(eq) == 0 { 83 return true 84 } 85 return p.equal("cpu_or_gpu_value", eq...) 86 } 87 return false 88} 89 90// arch returns true if the architecture for this job equals any of the 91// given values. 92func (p parts) arch(eq ...string) bool { 93 return p.equal("arch", eq...) || p.equal("target_arch", eq...) 94} 95 96// extraConfig returns true if any of the extra_configs for this job equals 97// any of the given values. If the extra_config starts with "SK_", 98// it is considered to be a single config. 99func (p parts) extraConfig(eq ...string) bool { 100 if len(eq) == 0 { 101 log.Fatal("No values provided for extraConfig()!") 102 } 103 ec := p["extra_config"] 104 if ec == "" { 105 return false 106 } 107 var cfgs []string 108 if strings.HasPrefix(ec, "SK_") { 109 cfgs = []string{ec} 110 } else { 111 cfgs = strings.Split(ec, "_") 112 } 113 for _, c := range cfgs { 114 for _, e := range eq { 115 if c == e { 116 return true 117 } 118 } 119 } 120 return false 121} 122 123// noExtraConfig returns true if there are no extra_configs for this job. 124func (p parts) noExtraConfig(eq ...string) bool { 125 ec := p["extra_config"] 126 return ec == "" 127} 128 129// matchPart returns true if the given part of this job's name matches any of 130// the given regular expressions. Note that a regular expression might match any 131// substring, so if you need an exact match on the entire string you'll need to 132// use `^` and `$`. Panics if no regular expressions are provided. 133func (p parts) matchPart(part string, re ...string) bool { 134 if len(re) == 0 { 135 log.Fatal("No regular expressions provided for matchPart()!") 136 } 137 v := p[part] 138 for _, r := range re { 139 if regexp.MustCompile(r).MatchString(v) { 140 return true 141 } 142 } 143 return false 144} 145 146// matchRole returns true if the role for this job matches any of the given 147// regular expressions. 148func (p parts) matchRole(re ...string) bool { 149 return p.matchPart("role", re...) 150} 151 152func (p parts) project(re ...string) bool { 153 return p.matchPart("project", re...) 154} 155 156// matchOs returns true if the OS for this job matches any of the given regular 157// expressions. 158func (p parts) matchOs(re ...string) bool { 159 return p.matchPart("os", re...) 160} 161 162// matchCompiler returns true if the compiler for this job matches any of the 163// given regular expressions. 164func (p parts) matchCompiler(re ...string) bool { 165 return p.matchPart("compiler", re...) 166} 167 168// matchModel returns true if the model for this job matches any of the given 169// regular expressions. 170func (p parts) matchModel(re ...string) bool { 171 return p.matchPart("model", re...) 172} 173 174// matchCpu returns true if the task's cpu_or_gpu is "CPU" and the CPU for this 175// task matches any of the given regular expressions. If no regular expressions 176// are provided, cpu returns true if this task runs on CPU. 177func (p parts) matchCpu(re ...string) bool { 178 if p["cpu_or_gpu"] == "CPU" { 179 if len(re) == 0 { 180 return true 181 } 182 return p.matchPart("cpu_or_gpu_value", re...) 183 } 184 return false 185} 186 187// matchGpu returns true if the task's cpu_or_gpu is "GPU" and the GPU for this task 188// matches any of the given regular expressions. If no regular expressions are 189// provided, gpu returns true if this task runs on GPU. 190func (p parts) matchGpu(re ...string) bool { 191 if p["cpu_or_gpu"] == "GPU" { 192 if len(re) == 0 { 193 return true 194 } 195 return p.matchPart("cpu_or_gpu_value", re...) 196 } 197 return false 198} 199 200// matchArch returns true if the architecture for this job matches any of the 201// given regular expressions. 202func (p parts) matchArch(re ...string) bool { 203 return p.matchPart("arch", re...) || p.matchPart("target_arch", re...) 204} 205 206// matchExtraConfig returns true if any of the extra_configs for this job matches 207// any of the given regular expressions. If the extra_config starts with "SK_", 208// it is considered to be a single config. 209func (p parts) matchExtraConfig(re ...string) bool { 210 if len(re) == 0 { 211 log.Fatal("No regular expressions provided for matchExtraConfig()!") 212 } 213 ec := p["extra_config"] 214 if ec == "" { 215 return false 216 } 217 var cfgs []string 218 if strings.HasPrefix(ec, "SK_") { 219 cfgs = []string{ec} 220 } else { 221 cfgs = strings.Split(ec, "_") 222 } 223 compiled := make([]*regexp.Regexp, 0, len(re)) 224 for _, r := range re { 225 compiled = append(compiled, regexp.MustCompile(r)) 226 } 227 for _, c := range cfgs { 228 for _, r := range compiled { 229 if r.MatchString(c) { 230 return true 231 } 232 } 233 } 234 return false 235} 236 237// debug returns true if this task runs in debug mode. 238func (p parts) debug() bool { 239 return p["configuration"] == "Debug" 240} 241 242// release returns true if this task runs in release mode. 243func (p parts) release() bool { 244 return p["configuration"] == "Release" 245} 246 247// isLinux returns true if the task runs on Linux. 248func (p parts) isLinux() bool { 249 return p.matchOs("Debian", "Ubuntu") 250} 251 252// TODO(borenet): The below really belongs in its own file, probably next to the 253// builder_name_schema.json file. 254 255// schema is a sub-struct of JobNameSchema. 256type schema struct { 257 Keys []string `json:"keys"` 258 OptionalKeys []string `json:"optional_keys"` 259 RecurseRoles []string `json:"recurse_roles"` 260} 261 262// JobNameSchema is a struct used for (de)constructing Job names in a 263// predictable format. 264type JobNameSchema struct { 265 Schema map[string]*schema `json:"builder_name_schema"` 266 Sep string `json:"builder_name_sep"` 267} 268 269// NewJobNameSchema returns a JobNameSchema instance based on the given JSON 270// file. 271func NewJobNameSchema(jsonFile string) (*JobNameSchema, error) { 272 var rv JobNameSchema 273 f, err := os.Open(jsonFile) 274 if err != nil { 275 return nil, err 276 } 277 defer func() { 278 if err := f.Close(); err != nil { 279 log.Println(fmt.Sprintf("Failed to close %s: %s", jsonFile, err)) 280 } 281 }() 282 if err := json.NewDecoder(f).Decode(&rv); err != nil { 283 return nil, err 284 } 285 return &rv, nil 286} 287 288// ParseJobName splits the given Job name into its component parts, according 289// to the schema. 290func (s *JobNameSchema) ParseJobName(n string) (map[string]string, error) { 291 popFront := func(items []string) (string, []string, error) { 292 if len(items) == 0 { 293 return "", nil, fmt.Errorf("Invalid job name: %s (not enough parts)", n) 294 } 295 return items[0], items[1:], nil 296 } 297 298 result := map[string]string{} 299 300 var parse func(int, string, []string) ([]string, error) 301 parse = func(depth int, role string, parts []string) ([]string, error) { 302 s, ok := s.Schema[role] 303 if !ok { 304 return nil, fmt.Errorf("Invalid job name; %q is not a valid role.", role) 305 } 306 if depth == 0 { 307 result["role"] = role 308 } else { 309 result[fmt.Sprintf("sub-role-%d", depth)] = role 310 } 311 var err error 312 for _, key := range s.Keys { 313 var value string 314 value, parts, err = popFront(parts) 315 if err != nil { 316 return nil, err 317 } 318 result[key] = value 319 } 320 for _, subRole := range s.RecurseRoles { 321 if len(parts) > 0 && parts[0] == subRole { 322 parts, err = parse(depth+1, parts[0], parts[1:]) 323 if err != nil { 324 return nil, err 325 } 326 } 327 } 328 for _, key := range s.OptionalKeys { 329 if len(parts) > 0 { 330 var value string 331 value, parts, err = popFront(parts) 332 if err != nil { 333 return nil, err 334 } 335 result[key] = value 336 } 337 } 338 if len(parts) > 0 { 339 return nil, fmt.Errorf("Invalid job name: %s (too many parts)", n) 340 } 341 return parts, nil 342 } 343 344 split := strings.Split(n, s.Sep) 345 if len(split) < 2 { 346 return nil, fmt.Errorf("Invalid job name: %s (not enough parts)", n) 347 } 348 role := split[0] 349 split = split[1:] 350 _, err := parse(0, role, split) 351 return result, err 352} 353 354// MakeJobName assembles the given parts of a Job name, according to the schema. 355func (s *JobNameSchema) MakeJobName(parts map[string]string) (string, error) { 356 rvParts := make([]string, 0, len(parts)) 357 358 var process func(int, map[string]string) (map[string]string, error) 359 process = func(depth int, parts map[string]string) (map[string]string, error) { 360 roleKey := "role" 361 if depth != 0 { 362 roleKey = fmt.Sprintf("sub-role-%d", depth) 363 } 364 role, ok := parts[roleKey] 365 if !ok { 366 return nil, fmt.Errorf("Invalid job parts; missing key %q", roleKey) 367 } 368 369 s, ok := s.Schema[role] 370 if !ok { 371 return nil, fmt.Errorf("Invalid job parts; unknown role %q", role) 372 } 373 rvParts = append(rvParts, role) 374 delete(parts, roleKey) 375 376 for _, key := range s.Keys { 377 value, ok := parts[key] 378 if !ok { 379 return nil, fmt.Errorf("Invalid job parts; missing %q", key) 380 } 381 rvParts = append(rvParts, value) 382 delete(parts, key) 383 } 384 385 if len(s.RecurseRoles) > 0 { 386 subRoleKey := fmt.Sprintf("sub-role-%d", depth+1) 387 subRole, ok := parts[subRoleKey] 388 if !ok { 389 return nil, fmt.Errorf("Invalid job parts; missing %q", subRoleKey) 390 } 391 rvParts = append(rvParts, subRole) 392 delete(parts, subRoleKey) 393 found := false 394 for _, recurseRole := range s.RecurseRoles { 395 if recurseRole == subRole { 396 found = true 397 var err error 398 parts, err = process(depth+1, parts) 399 if err != nil { 400 return nil, err 401 } 402 break 403 } 404 } 405 if !found { 406 return nil, fmt.Errorf("Invalid job parts; unknown sub-role %q", subRole) 407 } 408 } 409 for _, key := range s.OptionalKeys { 410 if value, ok := parts[key]; ok { 411 rvParts = append(rvParts, value) 412 delete(parts, key) 413 } 414 } 415 if len(parts) > 0 { 416 return nil, fmt.Errorf("Invalid job parts: too many parts: %v", parts) 417 } 418 return parts, nil 419 } 420 421 // Copy the parts map, so that we can modify at will. 422 partsCpy := make(map[string]string, len(parts)) 423 for k, v := range parts { 424 partsCpy[k] = v 425 } 426 if _, err := process(0, partsCpy); err != nil { 427 return "", err 428 } 429 return strings.Join(rvParts, s.Sep), nil 430} 431