• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Entry point for "link" command."""
5
6import collections
7import functools
8import hashlib
9import itertools
10import multiprocessing
11import os
12import pathlib
13import pickle
14import posixpath
15import re
16import sys
17import zipfile
18
19from codegen import gen_jni_java
20from codegen import header_common
21from codegen import natives_header
22from codegen import register_natives
23import common
24import java_types
25import jni_generator
26import parse
27import proxy
28
29
30_THIS_DIR = os.path.dirname(__file__)
31
32
33def _ParseHelper(package_prefix, package_prefix_filter, path):
34  return parse.parse_java_file(path,
35                               package_prefix=package_prefix,
36                               package_prefix_filter=package_prefix_filter)
37
38
39def _LoadJniObjs(paths, namespace, package_prefix, package_prefix_filter):
40  ret = {}
41  if all(p.endswith('.jni.pickle') for p in paths):
42    for pickle_path in paths:
43      with open(pickle_path, 'rb') as f:
44        parsed_files = pickle.load(f)
45      ret[pickle_path] = [
46          jni_generator.JniObject(pf,
47                                  from_javap=False,
48                                  default_namespace=namespace)
49          for pf in parsed_files
50      ]
51  else:
52    func = functools.partial(_ParseHelper, package_prefix,
53                             package_prefix_filter)
54    with multiprocessing.Pool() as pool:
55      for pf in pool.imap_unordered(func, paths):
56        ret[pf.filename] = [
57            jni_generator.JniObject(pf,
58                                    from_javap=False,
59                                    default_namespace=namespace)
60        ]
61
62  return ret
63
64
65def _FilterJniObjs(jni_objs_by_path, include_test_only, module_name):
66  for jni_objs in jni_objs_by_path.values():
67    # Remove test-only methods.
68    if not include_test_only:
69      for jni_obj in jni_objs:
70        jni_obj.RemoveTestOnlyNatives()
71    # Ignoring non-active modules and empty natives lists.
72    jni_objs[:] = [
73        o for o in jni_objs if o.natives and o.module_name == module_name
74    ]
75
76
77def _Flatten(jni_objs_by_path, paths):
78  return itertools.chain(*(jni_objs_by_path[p] for p in paths))
79
80
81def _CollectNatives(jni_objs, jni_mode):
82  ret = []
83  for jni_obj in jni_objs:
84    ret += jni_obj.proxy_natives
85  if jni_mode.is_muxing:
86    # Order by simplest to most complex signatures.
87    ret.sort(key=lambda n:
88             (len(n.proxy_params), not n.return_type.is_void(), n.muxed_name))
89  else:
90    ret.sort(key=lambda n: (n.java_class, n.name))
91  return ret
92
93
94def _Generate(args,
95              jni_mode,
96              native_sources,
97              java_sources,
98              priority_java_sources,
99              never_omit_switch_num=False):
100  """Generates files required to perform JNI registration.
101
102  Generates a srcjar containing a single class, GEN_JNI, that contains all
103  native method declarations.
104
105  Optionally generates a header file that provides RegisterNatives to perform
106  JNI registration.
107
108  Args:
109    args: argparse.Namespace object.
110    jni_mode: JniMode object.
111    native_sources: A list of .jni.pickle or .java file paths taken from native
112        dependency tree. The source of truth.
113    java_sources: A list of .jni.pickle or .java file paths. Used to assert
114        against native_sources.
115    priority_java_sources: A list of .jni.pickle or .java file paths. Used to
116        put these listed java files first in multiplexing.
117    never_omit_switch_num: Relevent for multiplexing. Necessary when the
118        native library must be interchangeable with another that uses
119        priority_java_sources (aka, webview).
120  """
121  native_sources_set = set(native_sources)
122  java_sources_set = set(java_sources)
123
124  jni_objs_by_path = _LoadJniObjs(native_sources_set | java_sources_set,
125                                  args.namespace, args.package_prefix,
126                                  args.package_prefix_filter)
127  _FilterJniObjs(jni_objs_by_path, args.include_test_only, args.module_name)
128
129  present_jni_objs = list(
130      _Flatten(jni_objs_by_path, native_sources_set & java_sources_set))
131
132  # Can contain path not in present_jni_objs.
133  priority_set = set(priority_java_sources or [])
134  # Sort for determinism and to put priority_java_sources first.
135  present_jni_objs.sort(
136      key=lambda o: (o.filename not in priority_set, o.java_class))
137
138  whole_hash = None
139  priority_hash = None
140  muxed_aliases_by_sig = None
141  if jni_mode.is_muxing:
142    whole_hash, priority_hash = _GenerateHashes(
143        present_jni_objs,
144        priority_set,
145        never_omit_switch_num=never_omit_switch_num)
146    muxed_aliases_by_sig = proxy.populate_muxed_switch_num(
147        present_jni_objs, never_omit_switch_num=never_omit_switch_num)
148
149  java_only_jni_objs = _CheckForJavaNativeMismatch(args, jni_objs_by_path,
150                                                   native_sources_set,
151                                                   java_sources_set)
152
153  present_proxy_natives = _CollectNatives(present_jni_objs, jni_mode)
154  absent_proxy_natives = _CollectNatives(java_only_jni_objs, jni_mode)
155  boundary_proxy_natives = present_proxy_natives
156  if jni_mode.is_muxing:
157    boundary_proxy_natives = [
158        n for n in present_proxy_natives if n.muxed_switch_num <= 0
159    ]
160
161  short_gen_jni_class = proxy.get_gen_jni_class(
162      short=True,
163      name_prefix=args.module_name,
164      package_prefix=args.package_prefix,
165      package_prefix_filter=args.package_prefix_filter)
166  full_gen_jni_class = proxy.get_gen_jni_class(
167      short=False,
168      name_prefix=args.module_name,
169      package_prefix=args.package_prefix,
170      package_prefix_filter=args.package_prefix_filter)
171
172  if args.header_path:
173    if jni_mode.is_hashing or jni_mode.is_muxing:
174      gen_jni_class = short_gen_jni_class
175    else:
176      gen_jni_class = full_gen_jni_class
177    header_content = _CreateHeader(jni_mode, present_jni_objs,
178                                   boundary_proxy_natives, gen_jni_class, args,
179                                   muxed_aliases_by_sig, whole_hash,
180                                   priority_hash)
181    with common.atomic_output(args.header_path, mode='w') as f:
182      f.write(header_content)
183
184  with common.atomic_output(args.srcjar_path) as f:
185    with zipfile.ZipFile(f, 'w') as srcjar:
186      script_name = jni_generator.GetScriptName()
187      if jni_mode.is_hashing or jni_mode.is_muxing:
188        # org/jni_zero/GEN_JNI.java
189        common.add_to_zip_hermetic(
190            srcjar,
191            f'{full_gen_jni_class.full_name_with_slashes}.java',
192            data=gen_jni_java.generate_forwarding(jni_mode, script_name,
193                                                  full_gen_jni_class,
194                                                  short_gen_jni_class,
195                                                  present_proxy_natives,
196                                                  absent_proxy_natives))
197        # J/N.java
198        common.add_to_zip_hermetic(
199            srcjar,
200            f'{short_gen_jni_class.full_name_with_slashes}.java',
201            data=gen_jni_java.generate_impl(jni_mode,
202                                            script_name,
203                                            short_gen_jni_class,
204                                            boundary_proxy_natives,
205                                            absent_proxy_natives,
206                                            whole_hash=whole_hash,
207                                            priority_hash=priority_hash))
208      else:
209        # org/jni_zero/GEN_JNI.java
210        common.add_to_zip_hermetic(
211            srcjar,
212            f'{full_gen_jni_class.full_name_with_slashes}.java',
213            data=gen_jni_java.generate_impl(jni_mode, script_name,
214                                            full_gen_jni_class,
215                                            boundary_proxy_natives,
216                                            absent_proxy_natives))
217
218
219def _CheckForJavaNativeMismatch(args, jni_objs_by_path, native_sources_set,
220                                java_sources_set):
221  native_only = native_sources_set - java_sources_set
222  java_only = java_sources_set - native_sources_set
223
224  java_only_jni_objs = sorted(_Flatten(jni_objs_by_path, java_only),
225                              key=lambda jni_obj: jni_obj.filename)
226  native_only_jni_objs = sorted(_Flatten(jni_objs_by_path, native_only),
227                                key=lambda jni_obj: jni_obj.filename)
228  failed = False
229  if not args.add_stubs_for_missing_native and java_only_jni_objs:
230    failed = True
231    warning_message = '''Failed JNI assertion!
232We reference Java files which use JNI, but our native library does not depend on
233the corresponding generate_jni().
234To bypass this check, add stubs to Java with --add-stubs-for-missing-jni.
235Excess Java files:
236'''
237    sys.stderr.write(warning_message)
238    sys.stderr.write('\n'.join(o.filename for o in java_only_jni_objs))
239    sys.stderr.write('\n')
240  if not args.remove_uncalled_methods and native_only_jni_objs:
241    failed = True
242    warning_message = '''Failed JNI assertion!
243Our native library depends on generate_jnis which reference Java files that we
244do not include in our final dex.
245To bypass this check, delete these extra methods with --remove-uncalled-jni.
246Unneeded Java files:
247'''
248    sys.stderr.write(warning_message)
249    sys.stderr.write('\n'.join(o.filename for o in native_only_jni_objs))
250    sys.stderr.write('\n')
251  if failed:
252    sys.exit(1)
253
254  return java_only_jni_objs
255
256
257def _GenerateHashes(jni_objs, priority_set, *, never_omit_switch_num):
258  # We assume that if we have identical files and they are in the same order, we
259  # will have switch number alignment. We do this, instead of directly
260  # inspecting the switch numbers, because differentiating the priority sources
261  # is a big pain.
262  whole_hash = hashlib.md5()
263  priority_hash = hashlib.md5()
264  had_priority = False
265
266  for i, jni_obj in enumerate(jni_objs):
267    path = os.path.relpath(jni_obj.filename, start=_THIS_DIR)
268    encoded = path.encode('utf-8')
269    whole_hash.update(encoded)
270    if jni_obj.filename in priority_set:
271      had_priority = True
272      priority_hash.update(encoded)
273
274  uint64_t_max = (1 << 64) - 1
275  int64_t_min = -(1 << 63)
276  to_int64 = lambda h: (int(h.hexdigest(), 16) % uint64_t_max) + int64_t_min
277  whole_ret = to_int64(whole_hash)
278
279  # Make it clear when there are no priority items.
280  priority_ret = to_int64(priority_hash) if had_priority else 0
281  if never_omit_switch_num:
282    priority_ret ^= 1
283    whole_ret ^= 1
284  return whole_ret, priority_ret
285
286
287def _CreateHeader(jni_mode, jni_objs, boundary_proxy_natives, gen_jni_class,
288                  args, muxed_aliases_by_sig, whole_hash, priority_hash):
289  """Returns the content of the header file."""
290  header_guard = os.path.splitext(args.header_path)[0].upper() + '_'
291  header_guard = re.sub(r'[/.-]', '_', header_guard)
292
293  preamble, epilogue = header_common.header_preamble(
294      jni_generator.GetScriptName(),
295      gen_jni_class,
296      system_includes=['iterator'],  # For std::size().
297      user_includes=['third_party/jni_zero/jni_zero_internal.h'],
298      header_guard=header_guard)
299
300  module_name = args.module_name or ''
301
302  sb = common.StringBuilder()
303  sb.line(preamble)
304  if jni_mode.is_muxing:
305    sb(f"""\
306extern const int64_t kJniZeroHash{module_name}Whole = {whole_hash}LL;
307extern const int64_t kJniZeroHash{module_name}Priority = {priority_hash}LL;
308""")
309
310  non_proxy_natives_java_classes = [
311      o.java_class for o in jni_objs if o.non_proxy_natives
312  ]
313  non_proxy_natives_java_classes.sort()
314
315  if non_proxy_natives_java_classes:
316    with sb.section('Class Accessors.'):
317      header_common.class_accessors(sb, non_proxy_natives_java_classes,
318                                    module_name)
319
320  with sb.section('Forward Declarations.'):
321    for jni_obj in jni_objs:
322      for native in jni_obj.natives:
323        with sb.statement():
324          natives_header.entry_point_declaration(sb, jni_mode, jni_obj, native,
325                                                 gen_jni_class)
326
327  if jni_mode.is_muxing and boundary_proxy_natives:
328    with sb.section('Multiplexing Methods.'):
329      for native in boundary_proxy_natives:
330        muxed_aliases = muxed_aliases_by_sig[native.muxed_signature]
331        natives_header.multiplexing_boundary_method(sb, muxed_aliases,
332                                                    gen_jni_class)
333
334  if args.manual_jni_registration:
335    # Helper methods use presence of gen_jni_class to denote presense of proxy
336    # methods.
337    if not boundary_proxy_natives:
338      gen_jni_class = None
339
340    with sb.section('Helper Methods.'):
341      with sb.namespace(''):
342        if boundary_proxy_natives:
343          register_natives.gen_jni_register_function(sb, jni_mode,
344                                                     gen_jni_class,
345                                                     boundary_proxy_natives)
346
347        for jni_obj in jni_objs:
348          if jni_obj.non_proxy_natives:
349            register_natives.non_proxy_register_function(sb, jni_obj)
350
351    with sb.section('Main Register Function.'):
352      register_natives.main_register_function(sb, jni_objs, args.namespace,
353                                              gen_jni_class)
354  sb(epilogue)
355  return sb.to_string()
356
357
358def _ParseSourceList(path):
359  # Path can have duplicates.
360  with open(path) as f:
361    return sorted(set(f.read().splitlines()))
362
363
364def _write_depfile(depfile_path, first_gn_output, inputs):
365  def _process_path(path):
366    assert not os.path.isabs(path), f'Found abs path in depfile: {path}'
367    if os.path.sep != posixpath.sep:
368      path = str(pathlib.Path(path).as_posix())
369    assert '\\' not in path, f'Found \\ in depfile: {path}'
370    return path.replace(' ', '\\ ')
371
372  sb = []
373  sb.append(_process_path(first_gn_output))
374  if inputs:
375    # Sort and uniquify to ensure file is hermetic.
376    # One path per line to keep it human readable.
377    sb.append(': \\\n ')
378    sb.append(' \\\n '.join(sorted(_process_path(p) for p in set(inputs))))
379  else:
380    sb.append(': ')
381  sb.append('\n')
382
383  pathlib.Path(depfile_path).write_text(''.join(sb))
384
385
386def main(parser, args, jni_mode):
387  if not args.header_path and args.manual_jni_registration:
388    parser.error('--manual-jni-registration requires --header-path.')
389  if not args.header_path and jni_mode.is_muxing:
390    parser.error('--enable-jni-multiplexing requires --header-path.')
391  if args.remove_uncalled_methods and not args.native_sources_file:
392    parser.error('--remove-uncalled-methods requires --native-sources-file.')
393  if args.priority_java_sources_file:
394    if not jni_mode.is_muxing:
395      parser.error('--priority-java-sources is only for multiplexing.')
396    if not args.never_omit_switch_num:
397      # We could arguably just set this rather than error out, but it's
398      # important that the flag also be set on the library that does not set
399      # --priority-java-sources.
400      parser.error('--priority-java-sources requires --never-omit-switch-num.')
401
402  java_sources = _ParseSourceList(args.java_sources_file)
403  if args.native_sources_file:
404    native_sources = _ParseSourceList(args.native_sources_file)
405  else:
406    if args.add_stubs_for_missing_native:
407      # This will create a fully stubbed out GEN_JNI.
408      native_sources = []
409    else:
410      # Just treating it like we have perfect alignment between native and java
411      # when only looking at java.
412      native_sources = java_sources
413  if args.priority_java_sources_file:
414    priority_java_sources = _ParseSourceList(args.priority_java_sources_file)
415  else:
416    priority_java_sources = None
417
418  _Generate(args,
419            jni_mode,
420            native_sources,
421            java_sources,
422            priority_java_sources,
423            never_omit_switch_num=args.never_omit_switch_num)
424
425  if args.depfile:
426    # GN does not declare a dep on the sources files to avoid circular
427    # dependencies, so they need to be listed here.
428    all_inputs = native_sources + java_sources + [args.java_sources_file]
429    if args.native_sources_file:
430      all_inputs.append(args.native_sources_file)
431    _write_depfile(args.depfile, args.srcjar_path, all_inputs)
432