1 /*
2 * This file has been donated to Jam.
3 */
4
5 /*
6 * Craig W. McPheeters, Alias|Wavefront.
7 *
8 * hcache.c hcache.h - handle caching of #includes in source files.
9 *
10 * Create a cache of files scanned for headers. When starting jam, look for the
11 * cache file and load it if present. When finished the binding phase, create a
12 * new header cache. The cache contains files, their timestamps and the header
13 * files found in their scan. During the binding phase of jam, look in the
14 * header cache first for the headers contained in a file. If the cache is
15 * present and valid, use its contents. This results in dramatic speedups with
16 * large projects (e.g. 3min -> 1min startup for one project.)
17 *
18 * External routines:
19 * hcache_init() - read and parse the local .jamdeps file.
20 * hcache_done() - write a new .jamdeps file.
21 * hcache() - return list of headers on target. Use cache or do a scan.
22 *
23 * The dependency file format is an ASCII file with 1 line per target. Each line
24 * has the following fields:
25 * @boundname@ timestamp_sec timestamp_nsec @file@ @file@ @file@ ...
26 */
27
28 #include "config.h"
29
30 #ifdef OPT_HEADER_CACHE_EXT
31
32 #include "jam.h"
33 #include "hcache.h"
34
35 #include "hash.h"
36 #include "headers.h"
37 #include "lists.h"
38 #include "modules.h"
39 #include "object.h"
40 #include "parse.h"
41 #include "regexp.h"
42 #include "rules.h"
43 #include "search.h"
44 #include "timestamp.h"
45 #include "variable.h"
46 #include "output.h"
47
48 #include <errno.h>
49 #include <string.h>
50
51 typedef struct hcachedata HCACHEDATA ;
52
53 struct hcachedata
54 {
55 OBJECT * boundname;
56 timestamp time;
57 LIST * includes;
58 LIST * hdrscan; /* the HDRSCAN value for this target */
59 int age; /* if too old, we will remove it from cache */
60 HCACHEDATA * next;
61 };
62
63
64 static struct hash * hcachehash = 0;
65 static HCACHEDATA * hcachelist = 0;
66
67 static int queries = 0;
68 static int hits = 0;
69
70 #define CACHE_FILE_VERSION "version 5"
71 #define CACHE_RECORD_HEADER "header"
72 #define CACHE_RECORD_END "end"
73
74
75 /*
76 * Return the name of the header cache file. May return NULL.
77 *
78 * The user sets this by setting the HCACHEFILE variable in a Jamfile. We cache
79 * the result so the user can not change the cache file during header scanning.
80 */
81
cache_name(void)82 static const char * cache_name( void )
83 {
84 static OBJECT * name = 0;
85 if ( !name )
86 {
87 LIST * const hcachevar = var_get( root_module(), constant_HCACHEFILE );
88
89 if ( !list_empty( hcachevar ) )
90 {
91 TARGET * const t = bindtarget( list_front( hcachevar ) );
92
93 pushsettings( root_module(), t->settings );
94 /* Do not expect the cache file to be generated, so pass 0 as the
95 * third argument to search. Expect the location to be specified via
96 * LOCATE, so pass 0 as the fourth argument.
97 */
98 object_free( t->boundname );
99 t->boundname = search( t->name, &t->time, 0, 0 );
100 popsettings( root_module(), t->settings );
101
102 name = object_copy( t->boundname );
103 }
104 }
105 return name ? object_str( name ) : 0;
106 }
107
108
109 /*
110 * Return the maximum age a cache entry can have before it is purged from the
111 * cache.
112 */
113
cache_maxage(void)114 static int cache_maxage( void )
115 {
116 int age = 100;
117 LIST * const var = var_get( root_module(), constant_HCACHEMAXAGE );
118 if ( !list_empty( var ) )
119 {
120 age = atoi( object_str( list_front( var ) ) );
121 if ( age < 0 )
122 age = 0;
123 }
124 return age;
125 }
126
127
128 /*
129 * Read a netstring. The caveat is that the string can not contain ASCII 0. The
130 * returned value is as returned by object_new().
131 */
132
read_netstring(FILE * f)133 OBJECT * read_netstring( FILE * f )
134 {
135 unsigned long len;
136 static char * buf = NULL;
137 static unsigned long buf_len = 0;
138
139 if ( fscanf( f, " %9lu", &len ) != 1 )
140 return NULL;
141 if ( fgetc( f ) != (int)'\t' )
142 return NULL;
143
144 if ( len > 1024 * 64 )
145 return NULL; /* sanity check */
146
147 if ( len > buf_len )
148 {
149 unsigned long new_len = buf_len * 2;
150 if ( new_len < len )
151 new_len = len;
152 buf = (char *)BJAM_REALLOC( buf, new_len + 1 );
153 if ( buf )
154 buf_len = new_len;
155 }
156
157 if ( !buf )
158 return NULL;
159
160 if ( fread( buf, 1, len, f ) != len )
161 return NULL;
162 if ( fgetc( f ) != (int)'\n' )
163 return NULL;
164
165 buf[ len ] = 0;
166 return object_new( buf );
167 }
168
169
170 /*
171 * Write a netstring.
172 */
173
write_netstring(FILE * f,char const * s)174 void write_netstring( FILE * f, char const * s )
175 {
176 if ( !s )
177 s = "";
178 fprintf( f, "%lu\t%s\n", (long unsigned)strlen( s ), s );
179 }
180
181
hcache_init()182 void hcache_init()
183 {
184 FILE * f;
185 OBJECT * version = 0;
186 int header_count = 0;
187 const char * hcachename;
188
189 if ( hcachehash )
190 return;
191
192 hcachehash = hashinit( sizeof( HCACHEDATA ), "hcache" );
193
194 if ( !( hcachename = cache_name() ) )
195 return;
196
197 if ( !( f = fopen( hcachename, "rb" ) ) )
198 {
199 if ( errno != ENOENT )
200 err_printf( "[errno %d] failed to read hcache file '%s': %s",
201 errno, hcachename, strerror(errno) );
202 return;
203 }
204
205 version = read_netstring( f );
206
207 if ( !version || strcmp( object_str( version ), CACHE_FILE_VERSION ) )
208 goto bail;
209
210 while ( 1 )
211 {
212 HCACHEDATA cachedata;
213 HCACHEDATA * c;
214 OBJECT * record_type = 0;
215 OBJECT * time_secs_str = 0;
216 OBJECT * time_nsecs_str = 0;
217 OBJECT * age_str = 0;
218 OBJECT * includes_count_str = 0;
219 OBJECT * hdrscan_count_str = 0;
220 int i;
221 int count;
222 LIST * l;
223 int found;
224
225 cachedata.boundname = 0;
226 cachedata.includes = 0;
227 cachedata.hdrscan = 0;
228
229 record_type = read_netstring( f );
230 if ( !record_type )
231 {
232 err_printf( "invalid %s\n", hcachename );
233 goto cleanup;
234 }
235 if ( !strcmp( object_str( record_type ), CACHE_RECORD_END ) )
236 {
237 object_free( record_type );
238 break;
239 }
240 if ( strcmp( object_str( record_type ), CACHE_RECORD_HEADER ) )
241 {
242 err_printf( "invalid %s with record separator <%s>\n",
243 hcachename, record_type ? object_str( record_type ) : "<null>" );
244 goto cleanup;
245 }
246
247 cachedata.boundname = read_netstring( f );
248 time_secs_str = read_netstring( f );
249 time_nsecs_str = read_netstring( f );
250 age_str = read_netstring( f );
251 includes_count_str = read_netstring( f );
252
253 if ( !cachedata.boundname || !time_secs_str || !time_nsecs_str ||
254 !age_str || !includes_count_str )
255 {
256 err_printf( "invalid %s\n", hcachename );
257 goto cleanup;
258 }
259
260 timestamp_init( &cachedata.time, atoi( object_str( time_secs_str ) ),
261 atoi( object_str( time_nsecs_str ) ) );
262 cachedata.age = atoi( object_str( age_str ) ) + 1;
263
264 count = atoi( object_str( includes_count_str ) );
265 for ( l = L0, i = 0; i < count; ++i )
266 {
267 OBJECT * const s = read_netstring( f );
268 if ( !s )
269 {
270 err_printf( "invalid %s\n", hcachename );
271 list_free( l );
272 goto cleanup;
273 }
274 l = list_push_back( l, s );
275 }
276 cachedata.includes = l;
277
278 hdrscan_count_str = read_netstring( f );
279 if ( !hdrscan_count_str )
280 {
281 err_printf( "invalid %s\n", hcachename );
282 goto cleanup;
283 }
284
285 count = atoi( object_str( hdrscan_count_str ) );
286 for ( l = L0, i = 0; i < count; ++i )
287 {
288 OBJECT * const s = read_netstring( f );
289 if ( !s )
290 {
291 err_printf( "invalid %s\n", hcachename );
292 list_free( l );
293 goto cleanup;
294 }
295 l = list_push_back( l, s );
296 }
297 cachedata.hdrscan = l;
298
299 c = (HCACHEDATA *)hash_insert( hcachehash, cachedata.boundname, &found )
300 ;
301 if ( !found )
302 {
303 c->boundname = cachedata.boundname;
304 c->includes = cachedata.includes;
305 c->hdrscan = cachedata.hdrscan;
306 c->age = cachedata.age;
307 timestamp_copy( &c->time, &cachedata.time );
308 }
309 else
310 {
311 err_printf( "can not insert header cache item, bailing on %s"
312 "\n", hcachename );
313 goto cleanup;
314 }
315
316 c->next = hcachelist;
317 hcachelist = c;
318
319 ++header_count;
320
321 object_free( record_type );
322 object_free( time_secs_str );
323 object_free( time_nsecs_str );
324 object_free( age_str );
325 object_free( includes_count_str );
326 object_free( hdrscan_count_str );
327 continue;
328
329 cleanup:
330
331 if ( record_type ) object_free( record_type );
332 if ( time_secs_str ) object_free( time_secs_str );
333 if ( time_nsecs_str ) object_free( time_nsecs_str );
334 if ( age_str ) object_free( age_str );
335 if ( includes_count_str ) object_free( includes_count_str );
336 if ( hdrscan_count_str ) object_free( hdrscan_count_str );
337
338 if ( cachedata.boundname ) object_free( cachedata.boundname );
339 if ( cachedata.includes ) list_free( cachedata.includes );
340 if ( cachedata.hdrscan ) list_free( cachedata.hdrscan );
341
342 goto bail;
343 }
344
345 if ( DEBUG_HEADER )
346 out_printf( "hcache read from file %s\n", hcachename );
347
348 bail:
349 if ( version )
350 object_free( version );
351 fclose( f );
352 }
353
354
hcache_done()355 void hcache_done()
356 {
357 FILE * f;
358 HCACHEDATA * c;
359 int header_count = 0;
360 const char * hcachename;
361 int maxage;
362
363 if ( !hcachehash )
364 return;
365
366 if ( !( hcachename = cache_name() ) )
367 goto cleanup;
368
369 if ( !( f = fopen( hcachename, "wb" ) ) )
370 {
371 err_printf( "[errno %d] failed to write hcache file '%s': %s",
372 errno, hcachename, strerror(errno) );
373 goto cleanup;
374 }
375
376 maxage = cache_maxage();
377
378 /* Print out the version. */
379 write_netstring( f, CACHE_FILE_VERSION );
380
381 c = hcachelist;
382 for ( c = hcachelist; c; c = c->next )
383 {
384 LISTITER iter;
385 LISTITER end;
386 char time_secs_str[ 30 ];
387 char time_nsecs_str[ 30 ];
388 char age_str[ 30 ];
389 char includes_count_str[ 30 ];
390 char hdrscan_count_str[ 30 ];
391
392 if ( maxage == 0 )
393 c->age = 0;
394 else if ( c->age > maxage )
395 continue;
396
397 sprintf( includes_count_str, "%lu", (long unsigned)list_length(
398 c->includes ) );
399 sprintf( hdrscan_count_str, "%lu", (long unsigned)list_length(
400 c->hdrscan ) );
401 sprintf( time_secs_str, "%lu", (long unsigned)c->time.secs );
402 sprintf( time_nsecs_str, "%lu", (long unsigned)c->time.nsecs );
403 sprintf( age_str, "%lu", (long unsigned)c->age );
404
405 write_netstring( f, CACHE_RECORD_HEADER );
406 write_netstring( f, object_str( c->boundname ) );
407 write_netstring( f, time_secs_str );
408 write_netstring( f, time_nsecs_str );
409 write_netstring( f, age_str );
410 write_netstring( f, includes_count_str );
411 for ( iter = list_begin( c->includes ), end = list_end( c->includes );
412 iter != end; iter = list_next( iter ) )
413 write_netstring( f, object_str( list_item( iter ) ) );
414 write_netstring( f, hdrscan_count_str );
415 for ( iter = list_begin( c->hdrscan ), end = list_end( c->hdrscan );
416 iter != end; iter = list_next( iter ) )
417 write_netstring( f, object_str( list_item( iter ) ) );
418 fputs( "\n", f );
419 ++header_count;
420 }
421 write_netstring( f, CACHE_RECORD_END );
422
423 if ( DEBUG_HEADER )
424 out_printf( "hcache written to %s. %d dependencies, %.0f%% hit rate\n",
425 hcachename, header_count, queries ? 100.0 * hits / queries : 0 );
426
427 fclose ( f );
428
429 cleanup:
430 for ( c = hcachelist; c; c = c->next )
431 {
432 list_free( c->includes );
433 list_free( c->hdrscan );
434 object_free( c->boundname );
435 }
436
437 hcachelist = 0;
438 if ( hcachehash )
439 hashdone( hcachehash );
440 hcachehash = 0;
441 }
442
443
hcache(TARGET * t,int rec,regexp * re[],LIST * hdrscan)444 LIST * hcache( TARGET * t, int rec, regexp * re[], LIST * hdrscan )
445 {
446 HCACHEDATA * c;
447
448 ++queries;
449
450 if ( ( c = (HCACHEDATA *)hash_find( hcachehash, t->boundname ) ) )
451 {
452 if ( !timestamp_cmp( &c->time, &t->time ) )
453 {
454 LIST * const l1 = hdrscan;
455 LIST * const l2 = c->hdrscan;
456 LISTITER iter1 = list_begin( l1 );
457 LISTITER const end1 = list_end( l1 );
458 LISTITER iter2 = list_begin( l2 );
459 LISTITER const end2 = list_end( l2 );
460 while ( iter1 != end1 && iter2 != end2 )
461 {
462 if ( !object_equal( list_item( iter1 ), list_item( iter2 ) ) )
463 iter1 = end1;
464 else
465 {
466 iter1 = list_next( iter1 );
467 iter2 = list_next( iter2 );
468 }
469 }
470 if ( iter1 != end1 || iter2 != end2 )
471 {
472 if ( DEBUG_HEADER )
473 {
474 out_printf( "HDRSCAN out of date in cache for %s\n",
475 object_str( t->boundname ) );
476 out_printf(" real : ");
477 list_print( hdrscan );
478 out_printf( "\n cached: " );
479 list_print( c->hdrscan );
480 out_printf( "\n" );
481 }
482
483 list_free( c->includes );
484 list_free( c->hdrscan );
485 c->includes = L0;
486 c->hdrscan = L0;
487 }
488 else
489 {
490 if ( DEBUG_HEADER )
491 out_printf( "using header cache for %s\n", object_str(
492 t->boundname ) );
493 c->age = 0;
494 ++hits;
495 return list_copy( c->includes );
496 }
497 }
498 else
499 {
500 if ( DEBUG_HEADER )
501 out_printf ("header cache out of date for %s\n", object_str(
502 t->boundname ) );
503 list_free( c->includes );
504 list_free( c->hdrscan );
505 c->includes = L0;
506 c->hdrscan = L0;
507 }
508 }
509 else
510 {
511 int found;
512 c = (HCACHEDATA *)hash_insert( hcachehash, t->boundname, &found );
513 if ( !found )
514 {
515 c->boundname = object_copy( t->boundname );
516 c->next = hcachelist;
517 hcachelist = c;
518 }
519 }
520
521 /* 'c' points at the cache entry. Its out of date. */
522 {
523 LIST * const l = headers1( L0, t->boundname, rec, re );
524
525 timestamp_copy( &c->time, &t->time );
526 c->age = 0;
527 c->includes = list_copy( l );
528 c->hdrscan = list_copy( hdrscan );
529
530 return l;
531 }
532 }
533
534 #endif /* OPT_HEADER_CACHE_EXT */
535