1 #region Copyright notice and license 2 3 // Copyright 2018 gRPC authors. 4 // 5 // Licensed under the Apache License, Version 2.0 (the "License"); 6 // you may not use this file except in compliance with the License. 7 // You may obtain a copy of the License at 8 // 9 // http://www.apache.org/licenses/LICENSE-2.0 10 // 11 // Unless required by applicable law or agreed to in writing, software 12 // distributed under the License is distributed on an "AS IS" BASIS, 13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 // See the License for the specific language governing permissions and 15 // limitations under the License. 16 17 #endregion 18 19 using System; 20 using System.Collections.Generic; 21 using System.IO; 22 using System.Text; 23 using Microsoft.Build.Framework; 24 using Microsoft.Build.Utilities; 25 26 namespace Grpc.Tools 27 { 28 internal static class DepFileUtil 29 { 30 /* 31 Sample dependency files. Notable features we have to deal with: 32 * Slash doubling, must normalize them. 33 * Spaces in file names. Cannot just "unwrap" the line on backslash at eof; 34 rather, treat every line as containing one file name except for one with 35 the ':' separator, as containing exactly two. 36 * Deal with ':' also being drive letter separator (second example). 37 38 obj\Release\net45\/Foo.cs \ 39 obj\Release\net45\/FooGrpc.cs: C:/foo/include/google/protobuf/wrappers.proto\ 40 C:/projects/foo/src//foo.proto 41 42 C:\projects\foo\src\./foo.grpc.pb.cc \ 43 C:\projects\foo\src\./foo.grpc.pb.h \ 44 C:\projects\foo\src\./foo.pb.cc \ 45 C:\projects\foo\src\./foo.pb.h: C:/foo/include/google/protobuf/wrappers.proto\ 46 C:/foo/include/google/protobuf/any.proto\ 47 C:/foo/include/google/protobuf/source_context.proto\ 48 C:/foo/include/google/protobuf/type.proto\ 49 foo.proto 50 */ 51 52 /// <summary> 53 /// Read file names from the dependency file to the right of ':' 54 /// </summary> 55 /// <param name="protoDepDir">Relative path to the dependency cache, e. g. "out"</param> 56 /// <param name="proto">Relative path to the proto item, e. g. "foo/file.proto"</param> 57 /// <param name="log">A <see cref="TaskLoggingHelper"/> for logging</param> 58 /// <returns> 59 /// Array of the proto file <b>input</b> dependencies as written by protoc, or empty 60 /// array if the dependency file does not exist or cannot be parsed. 61 /// </returns> ReadDependencyInputs(string protoDepDir, string proto, TaskLoggingHelper log)62 public static string[] ReadDependencyInputs(string protoDepDir, string proto, 63 TaskLoggingHelper log) 64 { 65 string depFilename = GetDepFilenameForProto(protoDepDir, proto); 66 string[] lines = ReadDepFileLines(depFilename, false, log); 67 if (lines.Length == 0) 68 { 69 return lines; 70 } 71 72 var result = new List<string>(); 73 bool skip = true; 74 foreach (string line in lines) 75 { 76 // Start at the only line separating dependency outputs from inputs. 77 int ix = skip ? FindLineSeparator(line) : -1; 78 skip = skip && ix < 0; 79 if (skip) { continue; } 80 string file = ExtractFilenameFromLine(line, ix + 1, line.Length); 81 if (file == "") 82 { 83 log.LogMessage(MessageImportance.Low, 84 $"Skipping unparsable dependency file {depFilename}.\nLine with error: '{line}'"); 85 return new string[0]; 86 } 87 88 // Do not bend over backwards trying not to include a proto into its 89 // own list of dependencies. Since a file is not older than self, 90 // it is safe to add; this is purely a memory optimization. 91 if (file != proto) 92 { 93 result.Add(file); 94 } 95 } 96 return result.ToArray(); 97 } 98 99 /// <summary> 100 /// Read file names from the dependency file to the left of ':' 101 /// </summary> 102 /// <param name="depFilename">Path to dependency file written by protoc</param> 103 /// <param name="log">A <see cref="TaskLoggingHelper"/> for logging</param> 104 /// <returns> 105 /// Array of the protoc-generated outputs from the given dependency file 106 /// written by protoc, or empty array if the file does not exist or cannot 107 /// be parsed. 108 /// </returns> 109 /// <remarks> 110 /// Since this is called after a protoc invocation, an unparsable or missing 111 /// file causes an error-level message to be logged. 112 /// </remarks> ReadDependencyOutputs(string depFilename, TaskLoggingHelper log)113 public static string[] ReadDependencyOutputs(string depFilename, 114 TaskLoggingHelper log) 115 { 116 string[] lines = ReadDepFileLines(depFilename, true, log); 117 if (lines.Length == 0) 118 { 119 return lines; 120 } 121 122 var result = new List<string>(); 123 foreach (string line in lines) 124 { 125 int ix = FindLineSeparator(line); 126 string file = ExtractFilenameFromLine(line, 0, ix >= 0 ? ix : line.Length); 127 if (file == "") 128 { 129 log.LogError("Unable to parse generated dependency file {0}.\n" + 130 "Line with error: '{1}'", depFilename, line); 131 return new string[0]; 132 } 133 result.Add(file); 134 135 // If this is the line with the separator, do not read further. 136 if (ix >= 0) { break; } 137 } 138 return result.ToArray(); 139 } 140 141 /// <summary> 142 /// Construct relative dependency file name from directory hash and file name 143 /// </summary> 144 /// <param name="protoDepDir">Relative path to the dependency cache, e. g. "out"</param> 145 /// <param name="proto">Relative path to the proto item, e. g. "foo/file.proto"</param> 146 /// <returns> 147 /// Full relative path to the dependency file, e. g. 148 /// "out/deadbeef12345678_file.protodep" 149 /// </returns> 150 /// <remarks> 151 /// See <see cref="GetDirectoryHash"/> for notes on directory hash. 152 /// </remarks> GetDepFilenameForProto(string protoDepDir, string proto)153 public static string GetDepFilenameForProto(string protoDepDir, string proto) 154 { 155 string dirhash = GetDirectoryHash(proto); 156 string filename = Path.GetFileNameWithoutExtension(proto); 157 return Path.Combine(protoDepDir, $"{dirhash}_{filename}.protodep"); 158 } 159 160 /// <summary> 161 /// Construct relative output directory with directory hash 162 /// </summary> 163 /// <param name="outputDir">Relative path to the output directory, e. g. "out"</param> 164 /// <param name="proto">Relative path to the proto item, e. g. "foo/file.proto"</param> 165 /// <returns> 166 /// Full relative path to the directory, e. g. "out/deadbeef12345678" 167 /// </returns> 168 /// <remarks> 169 /// See <see cref="GetDirectoryHash"/> for notes on directory hash. 170 /// </remarks> GetOutputDirWithHash(string outputDir, string proto)171 public static string GetOutputDirWithHash(string outputDir, string proto) 172 { 173 string dirhash = GetDirectoryHash(proto); 174 return Path.Combine(outputDir, dirhash); 175 } 176 177 /// <summary> 178 /// Construct the directory hash from a relative file name 179 /// </summary> 180 /// <param name="proto">Relative path to the proto item, e. g. "foo/file.proto"</param> 181 /// <returns> 182 /// Directory hash based on the file name, e. g. "deadbeef12345678" 183 /// </returns> 184 /// <remarks> 185 /// Since a project may contain proto files with the same filename but in different 186 /// directories, a unique directory for the generated files is constructed based on the 187 /// proto file names directory. The directory path can be arbitrary, for example, 188 /// it can be outside of the project, or an absolute path including a drive letter, 189 /// or a UNC network path. A name constructed from such a path by, for example, 190 /// replacing disallowed name characters with an underscore, may well be over 191 /// filesystem's allowed path length, since it will be located under the project 192 /// and solution directories, which are also some level deep from the root. 193 /// Instead of creating long and unwieldy names for these proto sources, we cache 194 /// the full path of the name without the filename, as in e. g. "foo/file.proto" 195 /// will yield the name "deadbeef12345678", where that is a presumed hash value 196 /// of the string "foo". This allows the path to be short, unique (up to a hash 197 /// collision), and still allowing the user to guess their provenance. 198 /// </remarks> GetDirectoryHash(string proto)199 private static string GetDirectoryHash(string proto) 200 { 201 string dirname = Path.GetDirectoryName(proto); 202 if (Platform.IsFsCaseInsensitive) 203 { 204 dirname = dirname.ToLowerInvariant(); 205 } 206 207 return HashString64Hex(dirname); 208 } 209 210 // Get a 64-bit hash for a directory string. We treat it as if it were 211 // unique, since there are not so many distinct proto paths in a project. 212 // We take the first 64 bit of the string SHA1. 213 // Internal for tests access only. HashString64Hex(string str)214 internal static string HashString64Hex(string str) 215 { 216 using (var sha1 = System.Security.Cryptography.SHA1.Create()) 217 { 218 byte[] hash = sha1.ComputeHash(Encoding.UTF8.GetBytes(str)); 219 var hashstr = new StringBuilder(16); 220 for (int i = 0; i < 8; i++) 221 { 222 hashstr.Append(hash[i].ToString("x2")); 223 } 224 return hashstr.ToString(); 225 } 226 } 227 228 // Extract filename between 'beg' (inclusive) and 'end' (exclusive) from 229 // line 'line', skipping over trailing and leading whitespace, and, when 230 // 'end' is immediately past end of line 'line', also final '\' (used 231 // as a line continuation token in the dep file). 232 // Returns an empty string if the filename cannot be extracted. ExtractFilenameFromLine(string line, int beg, int end)233 static string ExtractFilenameFromLine(string line, int beg, int end) 234 { 235 while (beg < end && char.IsWhiteSpace(line[beg])) beg++; 236 if (beg < end && end == line.Length && line[end - 1] == '\\') end--; 237 while (beg < end && char.IsWhiteSpace(line[end - 1])) end--; 238 if (beg == end) return ""; 239 240 string filename = line.Substring(beg, end - beg); 241 try 242 { 243 // Normalize file name. 244 return Path.Combine(Path.GetDirectoryName(filename), Path.GetFileName(filename)); 245 } 246 catch (Exception ex) when (Exceptions.IsIoRelated(ex)) 247 { 248 return ""; 249 } 250 } 251 252 // Finds the index of the ':' separating dependency clauses in the line, 253 // not taking Windows drive spec into account. Returns the index of the 254 // separating ':', or -1 if no separator found. FindLineSeparator(string line)255 static int FindLineSeparator(string line) 256 { 257 // Mind this case where the first ':' is not separator: 258 // C:\foo\bar\.pb.h: C:/protobuf/wrappers.proto\ 259 int ix = line.IndexOf(':'); 260 if (ix <= 0 || ix == line.Length - 1 261 || (line[ix + 1] != '/' && line[ix + 1] != '\\') 262 || !char.IsLetter(line[ix - 1])) 263 { 264 return ix; // Not a windows drive: no letter before ':', or no '\' after. 265 } 266 for (int j = ix - 1; --j >= 0;) 267 { 268 if (!char.IsWhiteSpace(line[j])) 269 { 270 return ix; // Not space or BOL only before "X:/". 271 } 272 } 273 return line.IndexOf(':', ix + 1); 274 } 275 276 // Read entire dependency file. The 'required' parameter controls error 277 // logging behavior in case the file not found. We require this file when 278 // compiling, but reading it is optional when computing dependencies. ReadDepFileLines(string filename, bool required, TaskLoggingHelper log)279 static string[] ReadDepFileLines(string filename, bool required, 280 TaskLoggingHelper log) 281 { 282 try 283 { 284 var result = File.ReadAllLines(filename); 285 if (!required) 286 { 287 log.LogMessage(MessageImportance.Low, $"Using dependency file {filename}"); 288 } 289 return result; 290 } 291 catch (Exception ex) when (Exceptions.IsIoRelated(ex)) 292 { 293 if (required) 294 { 295 log.LogError($"Unable to load {filename}: {ex.GetType().Name}: {ex.Message}"); 296 } 297 else 298 { 299 log.LogMessage(MessageImportance.Low, $"Skipping {filename}: {ex.Message}"); 300 } 301 return new string[0]; 302 } 303 } 304 }; 305 } 306