mirror of
https://github.com/llvm/llvm-project.git
synced 2025-04-29 02:16:06 +00:00
3884f07a55
As described in a comment on D140637, PluginInlineOrderTest.NoInlineFoo will fail with: InlineOrderPlugin.so: undefined symbol: _ZN4llvm13AllAnalysesOnINS_6ModuleEE6SetKeyE The symbol is unique in libLLVMCore and ends up local in AnalysisTests. Fix this by exporting all unique symbols found in libraries. AnalysisTests.symbols change in line count: 12464 -> 12499 Differential Revision: https://reviews.llvm.org/D146731
535 lines
21 KiB
Python
Executable File
535 lines
21 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
"""A tool for extracting a list of symbols to export
|
|
|
|
When exporting symbols from a dll or exe we either need to mark the symbols in
|
|
the source code as __declspec(dllexport) or supply a list of symbols to the
|
|
linker. This program automates the latter by inspecting the symbol tables of a
|
|
list of link inputs and deciding which of those symbols need to be exported.
|
|
|
|
We can't just export all the defined symbols, as there's a limit of 65535
|
|
exported symbols and in clang we go way over that, particularly in a debug
|
|
build. Therefore a large part of the work is pruning symbols either which can't
|
|
be imported, or which we think are things that have definitions in public header
|
|
files (i.e. template instantiations) and we would get defined in the thing
|
|
importing these symbols anyway.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
import sys
|
|
import re
|
|
import os
|
|
import subprocess
|
|
import multiprocessing
|
|
import argparse
|
|
|
|
# Define functions which extract a list of pairs of (symbols, is_def) from a
|
|
# library using several different tools. We use subprocess.Popen and yield a
|
|
# symbol at a time instead of using subprocess.check_output and returning a list
|
|
# as, especially on Windows, waiting for the entire output to be ready can take
|
|
# a significant amount of time.
|
|
|
|
def dumpbin_get_symbols(lib):
|
|
process = subprocess.Popen(['dumpbin','/symbols',lib], bufsize=1,
|
|
stdout=subprocess.PIPE, stdin=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
process.stdin.close()
|
|
for line in process.stdout:
|
|
# Look for external symbols
|
|
match = re.match("^.+(SECT|UNDEF).+External\s+\|\s+(\S+).*$", line)
|
|
if match:
|
|
yield (match.group(2), match.group(1) != "UNDEF")
|
|
process.wait()
|
|
|
|
def nm_get_symbols(lib):
|
|
# -P means the output is in portable format, and -g means we only get global
|
|
# symbols.
|
|
cmd = ['nm','-P','-g']
|
|
if sys.platform.startswith('aix'):
|
|
cmd += ['-Xany','-C','-p']
|
|
process = subprocess.Popen(cmd+[lib], bufsize=1,
|
|
stdout=subprocess.PIPE, stdin=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
process.stdin.close()
|
|
for line in process.stdout:
|
|
# Look for external symbols that are defined in some section
|
|
# The POSIX format is:
|
|
# name type value size
|
|
# The -P flag displays the size field for symbols only when applicable,
|
|
# so the last field is optional. There's no space after the value field,
|
|
# but \s+ match newline also, so \s+\S* will match the optional size field.
|
|
match = re.match("^(\S+)\s+[BDGRSTuVW]\s+\S+\s+\S*$", line)
|
|
if match:
|
|
yield (match.group(1), True)
|
|
# Look for undefined symbols, which have type U and may or may not
|
|
# (depending on which nm is being used) have value and size.
|
|
match = re.match("^(\S+)\s+U\s+(\S+\s+\S*)?$", line)
|
|
if match:
|
|
yield (match.group(1), False)
|
|
process.wait()
|
|
|
|
def readobj_get_symbols(lib):
|
|
process = subprocess.Popen(['llvm-readobj','--symbols',lib], bufsize=1,
|
|
stdout=subprocess.PIPE, stdin=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
process.stdin.close()
|
|
for line in process.stdout:
|
|
# When looking through the output of llvm-readobj we expect to see Name,
|
|
# Section, then StorageClass, so record Name and Section when we see
|
|
# them and decide if this is an external symbol when we see
|
|
# StorageClass.
|
|
match = re.search('Name: (\S+)', line)
|
|
if match:
|
|
name = match.group(1)
|
|
match = re.search('Section: (\S+)', line)
|
|
if match:
|
|
section = match.group(1)
|
|
match = re.search('StorageClass: (\S+)', line)
|
|
if match:
|
|
storageclass = match.group(1)
|
|
if section != 'IMAGE_SYM_ABSOLUTE' and \
|
|
storageclass == 'External':
|
|
yield (name, section != 'IMAGE_SYM_UNDEFINED')
|
|
process.wait()
|
|
|
|
# Define functions which determine if the target is 32-bit Windows (as that's
|
|
# where calling convention name decoration happens).
|
|
|
|
def dumpbin_is_32bit_windows(lib):
|
|
# dumpbin /headers can output a huge amount of data (>100MB in a debug
|
|
# build) so we read only up to the 'machine' line then close the output.
|
|
process = subprocess.Popen(['dumpbin','/headers',lib], bufsize=1,
|
|
stdout=subprocess.PIPE, stdin=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
process.stdin.close()
|
|
retval = False
|
|
for line in process.stdout:
|
|
match = re.match('.+machine \((\S+)\)', line)
|
|
if match:
|
|
retval = (match.group(1) == 'x86')
|
|
break
|
|
process.stdout.close()
|
|
process.wait()
|
|
return retval
|
|
|
|
def objdump_is_32bit_windows(lib):
|
|
output = subprocess.check_output(['objdump','-f',lib],
|
|
universal_newlines=True)
|
|
for line in output.splitlines():
|
|
match = re.match('.+file format (\S+)', line)
|
|
if match:
|
|
return (match.group(1) == 'pe-i386')
|
|
return False
|
|
|
|
def readobj_is_32bit_windows(lib):
|
|
output = subprocess.check_output(['llvm-readobj','--file-header',lib],
|
|
universal_newlines=True)
|
|
for line in output.splitlines():
|
|
match = re.match('Format: (\S+)', line)
|
|
if match:
|
|
return (match.group(1) == 'COFF-i386')
|
|
return False
|
|
|
|
# On AIX, there isn't an easy way to detect 32-bit windows objects with the system toolchain,
|
|
# so just assume false.
|
|
def aix_is_32bit_windows(lib):
|
|
return False
|
|
|
|
# MSVC mangles names to ?<identifier_mangling>@<type_mangling>. By examining the
|
|
# identifier/type mangling we can decide which symbols could possibly be
|
|
# required and which we can discard.
|
|
def should_keep_microsoft_symbol(symbol, calling_convention_decoration):
|
|
# Keep unmangled (i.e. extern "C") names
|
|
if not '?' in symbol:
|
|
if calling_convention_decoration:
|
|
# Remove calling convention decoration from names
|
|
match = re.match('[_@]([^@]+)', symbol)
|
|
if match:
|
|
symbol = match.group(1)
|
|
# Discard floating point/SIMD constants.
|
|
if symbol.startswith(("__xmm@", "__ymm@", "__real@")):
|
|
return None
|
|
return symbol
|
|
# Deleting destructors start with ?_G or ?_E and can be discarded because
|
|
# link.exe gives you a warning telling you they can't be exported if you
|
|
# don't
|
|
elif symbol.startswith('??_G') or symbol.startswith('??_E'):
|
|
return None
|
|
# An anonymous namespace is mangled as ?A(maybe hex number)@. Any symbol
|
|
# that mentions an anonymous namespace can be discarded, as the anonymous
|
|
# namespace doesn't exist outside of that translation unit.
|
|
elif re.search('\?A(0x\w+)?@', symbol):
|
|
return None
|
|
# Skip X86GenMnemonicTables functions, they are not exposed from llvm/include/.
|
|
elif re.match('\?is[A-Z0-9]*@X86@llvm', symbol):
|
|
return None
|
|
# Keep mangled llvm:: and clang:: function symbols. How we detect these is a
|
|
# bit of a mess and imprecise, but that avoids having to completely demangle
|
|
# the symbol name. The outermost namespace is at the end of the identifier
|
|
# mangling, and the identifier mangling is followed by the type mangling, so
|
|
# we look for (llvm|clang)@@ followed by something that looks like a
|
|
# function type mangling. To spot a function type we use (this is derived
|
|
# from clang/lib/AST/MicrosoftMangle.cpp):
|
|
# <function-type> ::= <function-class> <this-cvr-qualifiers>
|
|
# <calling-convention> <return-type>
|
|
# <argument-list> <throw-spec>
|
|
# <function-class> ::= [A-Z]
|
|
# <this-cvr-qualifiers> ::= [A-Z0-9_]*
|
|
# <calling-convention> ::= [A-JQ]
|
|
# <return-type> ::= .+
|
|
# <argument-list> ::= X (void)
|
|
# ::= .+@ (list of types)
|
|
# ::= .*Z (list of types, varargs)
|
|
# <throw-spec> ::= exceptions are not allowed
|
|
elif re.search('(llvm|clang)@@[A-Z][A-Z0-9_]*[A-JQ].+(X|.+@|.*Z)$', symbol):
|
|
return symbol
|
|
return None
|
|
|
|
# Itanium manglings are of the form _Z<identifier_mangling><type_mangling>. We
|
|
# demangle the identifier mangling to identify symbols that can be safely
|
|
# discarded.
|
|
def should_keep_itanium_symbol(symbol, calling_convention_decoration):
|
|
# Start by removing any calling convention decoration (which we expect to
|
|
# see on all symbols, even mangled C++ symbols)
|
|
if calling_convention_decoration and symbol.startswith('_'):
|
|
symbol = symbol[1:]
|
|
# Keep unmangled names
|
|
if not symbol.startswith('_') and not symbol.startswith('.'):
|
|
return symbol
|
|
# Discard manglings that aren't nested names
|
|
match = re.match('_Z(T[VTIS])?(N.+)', symbol)
|
|
if not match:
|
|
return None
|
|
# Demangle the name. If the name is too complex then we don't need to keep
|
|
# it, but it the demangling fails then keep the symbol just in case.
|
|
try:
|
|
names, _ = parse_itanium_nested_name(match.group(2))
|
|
except TooComplexName:
|
|
return None
|
|
if not names:
|
|
return symbol
|
|
# Keep llvm:: and clang:: names
|
|
elif names[0][0] == '4llvm' or names[0][0] == '5clang':
|
|
return symbol
|
|
# Discard everything else
|
|
else:
|
|
return None
|
|
|
|
# Certain kinds of complex manglings we assume cannot be part of a public
|
|
# interface, and we handle them by raising an exception.
|
|
class TooComplexName(Exception):
|
|
pass
|
|
|
|
# Parse an itanium mangled name from the start of a string and return a
|
|
# (name, rest of string) pair.
|
|
def parse_itanium_name(arg):
|
|
# Check for a normal name
|
|
match = re.match('(\d+)(.+)', arg)
|
|
if match:
|
|
n = int(match.group(1))
|
|
name = match.group(1)+match.group(2)[:n]
|
|
rest = match.group(2)[n:]
|
|
return name, rest
|
|
# Check for constructor/destructor names
|
|
match = re.match('([CD][123])(.+)', arg)
|
|
if match:
|
|
return match.group(1), match.group(2)
|
|
# Assume that a sequence of characters that doesn't end a nesting is an
|
|
# operator (this is very imprecise, but appears to be good enough)
|
|
match = re.match('([^E]+)(.+)', arg)
|
|
if match:
|
|
return match.group(1), match.group(2)
|
|
# Anything else: we can't handle it
|
|
return None, arg
|
|
|
|
# Parse an itanium mangled template argument list from the start of a string
|
|
# and throw it away, returning the rest of the string.
|
|
def skip_itanium_template(arg):
|
|
# A template argument list starts with I
|
|
assert arg.startswith('I'), arg
|
|
tmp = arg[1:]
|
|
while tmp:
|
|
# Check for names
|
|
match = re.match('(\d+)(.+)', tmp)
|
|
if match:
|
|
n = int(match.group(1))
|
|
tmp = match.group(2)[n:]
|
|
continue
|
|
# Check for substitutions
|
|
match = re.match('S[A-Z0-9]*_(.+)', tmp)
|
|
if match:
|
|
tmp = match.group(1)
|
|
# Start of a template
|
|
elif tmp.startswith('I'):
|
|
tmp = skip_itanium_template(tmp)
|
|
# Start of a nested name
|
|
elif tmp.startswith('N'):
|
|
_, tmp = parse_itanium_nested_name(tmp)
|
|
# Start of an expression: assume that it's too complicated
|
|
elif tmp.startswith('L') or tmp.startswith('X'):
|
|
raise TooComplexName
|
|
# End of the template
|
|
elif tmp.startswith('E'):
|
|
return tmp[1:]
|
|
# Something else: probably a type, skip it
|
|
else:
|
|
tmp = tmp[1:]
|
|
return None
|
|
|
|
# Parse an itanium mangled nested name and transform it into a list of pairs of
|
|
# (name, is_template), returning (list, rest of string).
|
|
def parse_itanium_nested_name(arg):
|
|
# A nested name starts with N
|
|
assert arg.startswith('N'), arg
|
|
ret = []
|
|
|
|
# Skip past the N, and possibly a substitution
|
|
match = re.match('NS[A-Z0-9]*_(.+)', arg)
|
|
if match:
|
|
tmp = match.group(1)
|
|
else:
|
|
tmp = arg[1:]
|
|
|
|
# Skip past CV-qualifiers and ref qualifiers
|
|
match = re.match('[rVKRO]*(.+)', tmp);
|
|
if match:
|
|
tmp = match.group(1)
|
|
|
|
# Repeatedly parse names from the string until we reach the end of the
|
|
# nested name
|
|
while tmp:
|
|
# An E ends the nested name
|
|
if tmp.startswith('E'):
|
|
return ret, tmp[1:]
|
|
# Parse a name
|
|
name_part, tmp = parse_itanium_name(tmp)
|
|
if not name_part:
|
|
# If we failed then we don't know how to demangle this
|
|
return None, None
|
|
is_template = False
|
|
# If this name is a template record that, then skip the template
|
|
# arguments
|
|
if tmp.startswith('I'):
|
|
tmp = skip_itanium_template(tmp)
|
|
is_template = True
|
|
# Add the name to the list
|
|
ret.append((name_part, is_template))
|
|
|
|
# If we get here then something went wrong
|
|
return None, None
|
|
|
|
# Parse a microsoft mangled symbol and return a list of pairs of
|
|
# (name, is_template). This is very rudimentary and does just enough
|
|
# in order to determine if the first or second component is a template.
|
|
def parse_microsoft_mangling(arg):
|
|
# If the name doesn't start with ? this isn't a mangled name
|
|
if not arg.startswith('?'):
|
|
return [(arg, False)]
|
|
arg = arg[1:]
|
|
components = []
|
|
while len(arg) > 0:
|
|
# If we see an empty component we've reached the end
|
|
if arg.startswith('@'):
|
|
return components
|
|
# Check for a simple name
|
|
match = re.match('(\w+)@(.+)', arg)
|
|
if match:
|
|
components.append((match.group(1), False))
|
|
arg = match.group(2)
|
|
continue
|
|
# Check for a special function name
|
|
match = re.match('(\?_?\w)(.+)', arg)
|
|
if match:
|
|
components.append((match.group(1), False))
|
|
arg = match.group(2)
|
|
continue
|
|
# Check for a template name
|
|
match = re.match('\?\$(\w+)@[^@]+@(.+)', arg)
|
|
if match:
|
|
components.append((match.group(1), True))
|
|
arg = match.group(2)
|
|
continue
|
|
# Some other kind of name that we can't handle
|
|
components.append((arg, False))
|
|
return components
|
|
return components
|
|
|
|
def extract_symbols(arg):
|
|
get_symbols, should_keep_symbol, calling_convention_decoration, lib = arg
|
|
symbol_defs = dict()
|
|
symbol_refs = set()
|
|
for (symbol, is_def) in get_symbols(lib):
|
|
symbol = should_keep_symbol(symbol, calling_convention_decoration)
|
|
if symbol:
|
|
if is_def:
|
|
symbol_defs[symbol] = 1 + symbol_defs.setdefault(symbol,0)
|
|
else:
|
|
symbol_refs.add(symbol)
|
|
return (symbol_defs, symbol_refs)
|
|
|
|
def get_template_name(sym, mangling):
|
|
# Parse the mangling into a list of (name, is_template)
|
|
try:
|
|
if mangling == 'microsoft':
|
|
names = parse_microsoft_mangling(sym)
|
|
else:
|
|
match = re.match('_Z(T[VTIS])?(N.+)', sym)
|
|
if match:
|
|
names, _ = parse_itanium_nested_name(match.group(2))
|
|
else:
|
|
names = None
|
|
except TooComplexName:
|
|
return None
|
|
|
|
if not names:
|
|
return None
|
|
|
|
# If any component is a template then return it
|
|
for name, is_template in names:
|
|
if is_template:
|
|
return name
|
|
|
|
# Not a template
|
|
return None
|
|
|
|
if __name__ == '__main__':
|
|
tool_exes = ['dumpbin','nm','objdump','llvm-readobj']
|
|
parser = argparse.ArgumentParser(
|
|
description='Extract symbols to export from libraries')
|
|
parser.add_argument('--mangling', choices=['itanium','microsoft'],
|
|
required=True, help='expected symbol mangling scheme')
|
|
parser.add_argument('--tools', choices=tool_exes, nargs='*',
|
|
help='tools to use to extract symbols and determine the'
|
|
' target')
|
|
parser.add_argument('libs', metavar='lib', type=str, nargs='+',
|
|
help='libraries to extract symbols from')
|
|
parser.add_argument('-o', metavar='file', type=str, help='output to file')
|
|
args = parser.parse_args()
|
|
|
|
# Determine the function to use to get the list of symbols from the inputs,
|
|
# and the function to use to determine if the target is 32-bit windows.
|
|
tools = { 'dumpbin' : (dumpbin_get_symbols, dumpbin_is_32bit_windows),
|
|
'nm' : (nm_get_symbols, None),
|
|
'objdump' : (None, objdump_is_32bit_windows),
|
|
'llvm-readobj' : (readobj_get_symbols, readobj_is_32bit_windows) }
|
|
get_symbols = None
|
|
is_32bit_windows = aix_is_32bit_windows if sys.platform.startswith('aix') else None
|
|
# If we have a tools argument then use that for the list of tools to check
|
|
if args.tools:
|
|
tool_exes = args.tools
|
|
# Find a tool to use by trying each in turn until we find one that exists
|
|
# (subprocess.call will throw OSError when the program does not exist)
|
|
get_symbols = None
|
|
for exe in tool_exes:
|
|
try:
|
|
# Close std streams as we don't want any output and we don't
|
|
# want the process to wait for something on stdin.
|
|
p = subprocess.Popen([exe], stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
stdin=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
p.stdout.close()
|
|
p.stderr.close()
|
|
p.stdin.close()
|
|
p.wait()
|
|
# Keep going until we have a tool to use for both get_symbols and
|
|
# is_32bit_windows
|
|
if not get_symbols:
|
|
get_symbols = tools[exe][0]
|
|
if not is_32bit_windows:
|
|
is_32bit_windows = tools[exe][1]
|
|
if get_symbols and is_32bit_windows:
|
|
break
|
|
except OSError:
|
|
continue
|
|
if not get_symbols:
|
|
print("Couldn't find a program to read symbols with", file=sys.stderr)
|
|
exit(1)
|
|
if not is_32bit_windows:
|
|
print("Couldn't find a program to determining the target", file=sys.stderr)
|
|
exit(1)
|
|
|
|
# How we determine which symbols to keep and which to discard depends on
|
|
# the mangling scheme
|
|
if args.mangling == 'microsoft':
|
|
should_keep_symbol = should_keep_microsoft_symbol
|
|
else:
|
|
should_keep_symbol = should_keep_itanium_symbol
|
|
|
|
# Get the list of libraries to extract symbols from
|
|
libs = list()
|
|
for lib in args.libs:
|
|
# When invoked by cmake the arguments are the cmake target names of the
|
|
# libraries, so we need to add .lib/.a to the end and maybe lib to the
|
|
# start to get the filename. Also allow objects.
|
|
suffixes = ['.lib','.a','.obj','.o']
|
|
if not any([lib.endswith(s) for s in suffixes]):
|
|
for s in suffixes:
|
|
if os.path.exists(lib+s):
|
|
lib = lib+s
|
|
break
|
|
if os.path.exists('lib'+lib+s):
|
|
lib = 'lib'+lib+s
|
|
break
|
|
if not any([lib.endswith(s) for s in suffixes]):
|
|
print("Don't know what to do with argument "+lib, file=sys.stderr)
|
|
exit(1)
|
|
libs.append(lib)
|
|
|
|
# Check if calling convention decoration is used by inspecting the first
|
|
# library in the list
|
|
calling_convention_decoration = is_32bit_windows(libs[0])
|
|
|
|
# Extract symbols from libraries in parallel. This is a huge time saver when
|
|
# doing a debug build, as there are hundreds of thousands of symbols in each
|
|
# library.
|
|
pool = multiprocessing.Pool()
|
|
try:
|
|
# Only one argument can be passed to the mapping function, and we can't
|
|
# use a lambda or local function definition as that doesn't work on
|
|
# windows, so create a list of tuples which duplicates the arguments
|
|
# that are the same in all calls.
|
|
vals = [(get_symbols, should_keep_symbol, calling_convention_decoration, x) for x in libs]
|
|
# Do an async map then wait for the result to make sure that
|
|
# KeyboardInterrupt gets caught correctly (see
|
|
# http://bugs.python.org/issue8296)
|
|
result = pool.map_async(extract_symbols, vals)
|
|
pool.close()
|
|
libs_symbols = result.get(3600)
|
|
except KeyboardInterrupt:
|
|
# On Ctrl-C terminate everything and exit
|
|
pool.terminate()
|
|
pool.join()
|
|
exit(1)
|
|
|
|
# Merge everything into a single dict
|
|
symbol_defs = dict()
|
|
symbol_refs = set()
|
|
for (this_lib_defs, this_lib_refs) in libs_symbols:
|
|
for k,v in list(this_lib_defs.items()):
|
|
symbol_defs[k] = v + symbol_defs.setdefault(k,0)
|
|
for sym in list(this_lib_refs):
|
|
symbol_refs.add(sym)
|
|
|
|
# Find which template instantiations are referenced at least once.
|
|
template_instantiation_refs = set()
|
|
for sym in list(symbol_refs):
|
|
template = get_template_name(sym, args.mangling)
|
|
if template:
|
|
template_instantiation_refs.add(template)
|
|
|
|
# Print symbols which both:
|
|
# * Appear in exactly one input, as symbols defined in multiple
|
|
# objects/libraries are assumed to have public definitions.
|
|
# * Are not a template instantiation that isn't referenced anywhere. This
|
|
# is because we need to export any explicitly instantiated templates,
|
|
# and we expect those to be referenced in some object.
|
|
if args.o:
|
|
outfile = open(args.o,'w')
|
|
else:
|
|
outfile = sys.stdout
|
|
for k,v in list(symbol_defs.items()):
|
|
template = get_template_name(k, args.mangling)
|
|
if v == 1 and (not template or template in template_instantiation_refs):
|
|
print(k, file=outfile)
|