← Back
Editing: cparser.py
from . import model from .commontypes import COMMON_TYPES, resolve_common_type from .error import FFIError, CDefError try: from . import _pycparser as pycparser except ImportError: import pycparser import weakref, re, sys try: if sys.version_info < (3,): import thread as _thread else: import _thread lock = _thread.allocate_lock() except ImportError: lock = None def _workaround_for_static_import_finders(): # Issue #392: packaging tools like cx_Freeze can not find these # because pycparser uses exec dynamic import. This is an obscure # workaround. This function is never called. import pycparser.yacctab import pycparser.lextab CDEF_SOURCE_STRING = "<cdef source string>" _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", re.DOTALL | re.MULTILINE) _r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)" r"\b((?:[^\n\\]|\\.)*?)$", re.DOTALL | re.MULTILINE) _r_line_directive = re.compile(r"^[ \t]*#[ \t]*(?:line|\d+)\b.*$", re.MULTILINE) _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}") _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$") _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") _r_words = re.compile(r"\w+|\S") _parser_cache = None _r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE) _r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") _r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") _r_cdecl = re.compile(r"\b__cdecl\b") _r_extern_python = re.compile(r'\bextern\s*"' r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') _r_star_const_space = re.compile( # matches "* const " r"[*]\s*((const|volatile|restrict)\b\s*)+") _r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" r"\.\.\.") _r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") def _get_parser(): global _parser_cache if _parser_cache is None: _parser_cache = pycparser.CParser() return _parser_cache def _workaround_for_old_pycparser(csource): # Workaround for a pycparser issue (fixed between pycparser 2.10 and # 2.14): "char*const***" gives us a wrong syntax tree, the same as # for "char***(*const)". This means we can't tell the difference # afterwards. But "char(*const(***))" gives us the right syntax # tree. The issue only occurs if there are several stars in # sequence with no parenthesis inbetween, just possibly qualifiers. # Attempt to fix it by adding some parentheses in the source: each # time we see "* const" or "* const *", we add an opening # parenthesis before each star---the hard part is figuring out where # to close them. parts = [] while True: match = _r_star_const_space.search(csource) if not match: break #print repr(''.join(parts)+csource), '=>', parts.append(csource[:match.start()]) parts.append('('); closing = ')' parts.append(match.group()) # e.g. "* const " endpos = match.end() if csource.startswith('*', endpos): parts.append('('); closing += ')' level = 0 i = endpos while i < len(csource): c = csource[i] if c == '(': level += 1 elif c == ')': if level == 0: break level -= 1 elif c in ',;=': if level == 0: break i += 1 csource = csource[endpos:i] + closing + csource[i:] #print repr(''.join(parts)+csource) parts.append(csource) return ''.join(parts) def _preprocess_extern_python(csource): # input: `extern "Python" int foo(int);` or # `extern "Python" { int foo(int); }` # output: # void __cffi_extern_python_start; # int foo(int); # void __cffi_extern_python_stop; # # input: `extern "Python+C" int foo(int);` # output: # void __cffi_extern_python_plus_c_start; # int foo(int); # void __cffi_extern_python_stop; parts = [] while True: match = _r_extern_python.search(csource) if not match: break endpos = match.end() - 1 #print #print ''.join(parts)+csource #print '=>' parts.append(csource[:match.start()]) if 'C' in match.group(1): parts.append('void __cffi_extern_python_plus_c_start; ') else: parts.append('void __cffi_extern_python_start; ') if csource[endpos] == '{': # grouping variant closing = csource.find('}', endpos) if closing < 0: raise CDefError("'extern \"Python\" {': no '}' found") if csource.find('{', endpos + 1, closing) >= 0: raise NotImplementedError("cannot use { } inside a block " "'extern \"Python\" { ... }'") parts.append(csource[endpos+1:closing]) csource = csource[closing+1:] else: # non-grouping variant semicolon = csource.find(';', endpos) if semicolon < 0: raise CDefError("'extern \"Python\": no ';' found") parts.append(csource[endpos:semicolon+1]) csource = csource[semicolon+1:] parts.append(' void __cffi_extern_python_stop;') #print ''.join(parts)+csource #print parts.append(csource) return ''.join(parts) def _warn_for_string_literal(csource): if '"' not in csource: return for line in csource.splitlines(): if '"' in line and not line.lstrip().startswith('#'): import warnings warnings.warn("String literal found in cdef() or type source. " "String literals are ignored here, but you should " "remove them anyway because some character sequences " "confuse pre-parsing.") break def _warn_for_non_extern_non_static_global_variable(decl): if not decl.storage: import warnings warnings.warn("Global variable '%s' in cdef(): for consistency " "with C it should have a storage class specifier " "(usually 'extern')" % (decl.name,)) def _remove_line_directives(csource): # _r_line_directive matches whole lines, without the final \n, if they # start with '#line' with some spacing allowed, or '#NUMBER'. This # function stores them away and replaces them with exactly the string # '#line@N', where N is the index in the list 'line_directives'. line_directives = [] def replace(m): i = len(line_directives) line_directives.append(m.group()) return '#line@%d' % i csource = _r_line_directive.sub(replace, csource) return csource, line_directives def _put_back_line_directives(csource, line_directives): def replace(m): s = m.group() if not s.startswith('#line@'): raise AssertionError("unexpected #line directive " "(should have been processed and removed") return line_directives[int(s[6:])] return _r_line_directive.sub(replace, csource) def _preprocess(csource): # First, remove the lines of the form '#line N "filename"' because # the "filename" part could confuse the rest csource, line_directives = _remove_line_directives(csource) # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literals (except in line directives)! def replace_keeping_newlines(m): return ' ' + m.group().count('\n') * '\n' csource = _r_comment.sub(replace_keeping_newlines, csource) # Remove the "#define FOO x" lines macros = {} for match in _r_define.finditer(csource): macroname, macrovalue = match.groups() macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) # # BIG HACK: replace WINAPI or __stdcall with "volatile const". # It doesn't make sense for the return type of a function to be # "volatile volatile const", so we abuse it to detect __stdcall... # Hack number 2 is that "int(volatile *fptr)();" is not valid C # syntax, so we place the "volatile" before the opening parenthesis. csource = _r_stdcall2.sub(' volatile volatile const(', csource) csource = _r_stdcall1.sub(' volatile volatile const ', csource) csource = _r_cdecl.sub(' ', csource) # # Replace `extern "Python"` with start/end markers csource = _preprocess_extern_python(csource) # # Now there should not be any string literal left; warn if we get one _warn_for_string_literal(csource) # # Replace "[...]" with "[__dotdotdotarray__]" csource = _r_partial_array.sub('[__dotdotdotarray__]', csource) # # Replace "...}" with "__dotdotdotNUM__}". This construction should # occur only at the end of enums; at the end of structs we have "...;}" # and at the end of vararg functions "...);". Also replace "=...[,}]" # with ",__dotdotdotNUM__[,}]": this occurs in the enums too, when # giving an unknown value. matches = list(_r_partial_enum.finditer(csource)) for number, match in enumerate(reversed(matches)): p = match.start() if csource[p] == '=': p2 = csource.find('...', p, match.end()) assert p2 > p csource = '%s,__dotdotdot%d__ %s' % (csource[:p], number, csource[p2+3:]) else: assert csource[p:p+3] == '...' csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, csource[p+3:]) # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) # Replace "float ..." or "double..." with "__dotdotdotfloat__" csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) # Replace all remaining "..." with the same name, "__dotdotdot__", # which is declared with a typedef for the purpose of C parsing. csource = csource.replace('...', ' __dotdotdot__ ') # Finally, put back the line directives csource = _put_back_line_directives(csource, line_directives) return csource, macros def _common_type_names(csource): # Look in the source for what looks like usages of types from the # list of common types. A "usage" is approximated here as the # appearance of the word, minus a "definition" of the type, which # is the last word in a "typedef" statement. Approximative only # but should be fine for all the common types. look_for_words = set(COMMON_TYPES) look_for_words.add(';') look_for_words.add(',') look_for_words.add('(') look_for_words.add(')') look_for_words.add('typedef') words_used = set() is_typedef = False paren = 0 previous_word = '' for word in _r_words.findall(csource): if word in look_for_words: if word == ';': if is_typedef: words_used.discard(previous_word) look_for_words.discard(previous_word) is_typedef = False elif word == 'typedef': is_typedef = True paren = 0 elif word == '(': paren += 1 elif word == ')': paren -= 1 elif word == ',': if is_typedef and paren == 0: words_used.discard(previous_word) look_for_words.discard(previous_word) else: # word in COMMON_TYPES words_used.add(word) previous_word = word return words_used class Parser(object): def __init__(self): self._declarations = {} self._included_declarations = set() self._anonymous_counter = 0 self._structnode2type = weakref.WeakKeyDictionary() self._options = {} self._int_constants = {} self._recomplete = [] self._uses_new_feature = None def _parse(self, csource): csource, macros = _preprocess(csource) # XXX: for more efficiency we would need to poke into the # internals of CParser... the following registers the # typedefs, because their presence or absence influences the # parsing itself (but what they are typedef'ed to plays no role) ctn = _common_type_names(csource) typenames = [] for name in sorted(self._declarations): if name.startswith('typedef '): name = name[8:] typenames.append(name) ctn.discard(name) typenames += sorted(ctn) # csourcelines = [] csourcelines.append('# 1 "<cdef automatic initialization code>"') for typename in typenames: csourcelines.append('typedef int %s;' % typename) csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' ' __dotdotdot__;') # this forces pycparser to consider the following in the file # called <cdef source string> from line 1 csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,)) csourcelines.append(csource) fullcsource = '\n'.join(csourcelines) if lock is not None: lock.acquire() # pycparser is not thread-safe... try: ast = _get_parser().parse(fullcsource) except pycparser.c_parser.ParseError as e: self.convert_pycparser_error(e, csource) finally: if lock is not None: lock.release() # csource will be used to find buggy source text return ast, macros, csource def _convert_pycparser_error(self, e, csource): # xxx look for "<cdef source string>:NUM:" at the start of str(e) # and interpret that as a line number. This will not work if # the user gives explicit ``# NUM "FILE"`` directives. line = None msg = str(e) match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg) if match: linenum = int(match.group(1), 10) csourcelines = csource.splitlines() if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] return line def convert_pycparser_error(self, e, csource): line = self._convert_pycparser_error(e, csource) msg = str(e) if line: msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) raise CDefError(msg) def parse(self, csource, override=False, packed=False, pack=None, dllexport=False): if packed: if packed != True: raise ValueError("'packed' should be False or True; use " "'pack' to give another value") if pack: raise ValueError("cannot give both 'pack' and 'packed'") pack = 1 elif pack: if pack & (pack - 1): raise ValueError("'pack' must be a power of two, not %r" % (pack,)) else: pack = 0 prev_options = self._options try: self._options = {'override': override, 'packed': pack, 'dllexport': dllexport} self._internal_parse(csource) finally: self._options = prev_options def _internal_parse(self, csource): ast, macros, csource = self._parse(csource) # add the macros self._process_macros(macros) # find the first "__dotdotdot__" and use that as a separator # between the repeated typedefs and the real csource iterator = iter(ast.ext) for decl in iterator: if decl.name == '__dotdotdot__': break else: assert 0 current_decl = None # try: self._inside_extern_python = '__cffi_extern_python_stop' for decl in iterator: current_decl = decl if isinstance(decl, pycparser.c_ast.Decl): self._parse_decl(decl) elif isinstance(decl, pycparser.c_ast.Typedef): if not decl.name: raise CDefError("typedef does not declare any name", decl) quals = 0 if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and decl.type.type.names[-1].startswith('__dotdotdot')): realtype = self._get_unknown_type(decl) elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and isinstance(decl.type.type.type, pycparser.c_ast.IdentifierType) and decl.type.type.type.names[-1].startswith('__dotdotdot')): realtype = self._get_unknown_ptr_type(decl) else: realtype, quals = self._get_type_and_quals( decl.type, name=decl.name, partial_length_ok=True, typedef_example="*(%s *)0" % (decl.name,)) self._declare('typedef ' + decl.name, realtype, quals=quals) elif decl.__class__.__name__ == 'Pragma': pass # skip pragma, only in pycparser 2.15 else: raise CDefError("unexpected <%s>: this construct is valid " "C but not valid in cdef()" % decl.__class__.__name__, decl) except CDefError as e: if len(e.args) == 1: e.args = e.args + (current_decl,) raise except FFIError as e: msg = self._convert_pycparser_error(e, csource) if msg: e.args = (e.args[0] + "\n *** Err: %s" % msg,) raise def _add_constants(self, key, val): if key in self._int_constants: if self._int_constants[key] == val: return # ignore identical double declarations raise FFIError( "multiple declarations of constant: %s" % (key,)) self._int_constants[key] = val def _add_integer_constant(self, name, int_str): int_str = int_str.lower().rstrip("ul") neg = int_str.startswith('-') if neg: int_str = int_str[1:] # "010" is not valid oct in py3 if (int_str.startswith("0") and int_str != '0' and not int_str.startswith("0x")): int_str = "0o" + int_str[1:] pyvalue = int(int_str, 0) if neg: pyvalue = -pyvalue self._add_constants(name, pyvalue) self._declare('macro ' + name, pyvalue) def _process_macros(self, macros): for key, value in macros.items(): value = value.strip() if _r_int_literal.match(value): self._add_integer_constant(key, value) elif value == '...': self._declare('macro ' + key, value) else: raise CDefError( 'only supports one of the following syntax:\n' ' #define %s ... (literally dot-dot-dot)\n' ' #define %s NUMBER (with NUMBER an integer' ' constant, decimal/hex/octal)\n' 'got:\n' ' #define %s %s' % (key, key, key, value)) def _declare_function(self, tp, quals, decl): tp = self._get_type_pointer(tp, quals) if self._options.get('dllexport'): tag = 'dllexport_python ' elif self._inside_extern_python == '__cffi_extern_python_start': tag = 'extern_python ' elif self._inside_extern_python == '__cffi_extern_python_plus_c_start': tag = 'extern_python_plus_c ' else: tag = 'function ' self._declare(tag + decl.name, tp) def _parse_decl(self, decl): node = decl.type if isinstance(node, pycparser.c_ast.FuncDecl): tp, quals = self._get_type_and_quals(node, name=decl.name) assert isinstance(tp, model.RawFunctionType) self._declare_function(tp, quals, decl) else: if isinstance(node, pycparser.c_ast.Struct): self._get_struct_union_enum_type('struct', node) elif isinstance(node, pycparser.c_ast.Union): self._get_struct_union_enum_type('union', node) elif isinstance(node, pycparser.c_ast.Enum): self._get_struct_union_enum_type('enum', node) elif not decl.name: raise CDefError("construct does not declare any variable", decl) # if decl.name: tp, quals = self._get_type_and_quals(node, partial_length_ok=True) if tp.is_raw_function: self._declare_function(tp, quals, decl) elif (tp.is_integer_type() and hasattr(decl, 'init') and hasattr(decl.init, 'value') and _r_int_literal.match(decl.init.value)): self._add_integer_constant(decl.name, decl.init.value) elif (tp.is_integer_type() and isinstance(decl.init, pycparser.c_ast.UnaryOp) and decl.init.op == '-' and hasattr(decl.init.expr, 'value') and _r_int_literal.match(decl.init.expr.value)): self._add_integer_constant(decl.name, '-' + decl.init.expr.value) elif (tp is model.void_type and decl.name.startswith('__cffi_extern_python_')): # hack: `extern "Python"` in the C source is replaced # with "void __cffi_extern_python_start;" and # "void __cffi_extern_python_stop;" self._inside_extern_python = decl.name else: if self._inside_extern_python !='__cffi_extern_python_stop': raise CDefError( "cannot declare constants or " "variables with 'extern \"Python\"'") if (quals & model.Q_CONST) and not tp.is_array_type: self._declare('constant ' + decl.name, tp, quals=quals) else: _warn_for_non_extern_non_static_global_variable(decl) self._declare('variable ' + decl.name, tp, quals=quals) def parse_type(self, cdecl): return self.parse_type_and_quals(cdecl)[0] def parse_type_and_quals(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2] assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise CDefError("unknown identifier '%s'" % (exprnode.name,)) return self._get_type_and_quals(exprnode.type) def _declare(self, name, obj, included=False, quals=0): if name in self._declarations: prevobj, prevquals = self._declarations[name] if prevobj is obj and prevquals == quals: return if not self._options.get('override'): raise FFIError( "multiple declarations of %s (for interactive usage, " "try cdef(xx, override=True))" % (name,)) assert '__dotdotdot__' not in name.split() self._declarations[name] = (obj, quals) if included: self._included_declarations.add(obj) def _extract_quals(self, type): quals = 0 if isinstance(type, (pycparser.c_ast.TypeDecl, pycparser.c_ast.PtrDecl)): if 'const' in type.quals: quals |= model.Q_CONST if 'volatile' in type.quals: quals |= model.Q_VOLATILE if 'restrict' in type.quals: quals |= model.Q_RESTRICT return quals def _get_type_pointer(self, type, quals, declname=None): if isinstance(type, model.RawFunctionType): return type.as_function_pointer() if (isinstance(type, model.StructOrUnionOrEnum) and type.name.startswith('$') and type.name[1:].isdigit() and type.forcename is None and declname is not None): return model.NamedPointerType(type, declname, quals) return model.PointerType(type, quals) def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False, typedef_example=None): # first, dereference typedefs, if we have it already parsed, we're good if (isinstance(typenode, pycparser.c_ast.TypeDecl) and isinstance(typenode.type, pycparser.c_ast.IdentifierType) and len(typenode.type.names) == 1 and ('typedef ' + typenode.type.names[0]) in self._declarations): tp, quals = self._declarations['typedef ' + typenode.type.names[0]] quals |= self._extract_quals(typenode) return tp, quals # if isinstance(typenode, pycparser.c_ast.ArrayDecl): # array type if typenode.dim is None: length = None else: length = self._parse_constant( typenode.dim, partial_length_ok=partial_length_ok) # a hack: in 'typedef int foo_t[...][...];', don't use '...' as # the length but use directly the C expression that would be # generated by recompiler.py. This lets the typedef be used in # many more places within recompiler.py if typedef_example is not None: if length == '...': length = '_cffi_array_len(%s)' % (typedef_example,) typedef_example = "*" + typedef_example # tp, quals = self._get_type_and_quals(typenode.type, partial_length_ok=partial_length_ok, typedef_example=typedef_example) return model.ArrayType(tp, length), quals # if isinstance(typenode, pycparser.c_ast.PtrDecl): # pointer type itemtype, itemquals = self._get_type_and_quals(typenode.type) tp = self._get_type_pointer(itemtype, itemquals, declname=name) quals = self._extract_quals(typenode) return tp, quals # if isinstance(typenode, pycparser.c_ast.TypeDecl): quals = self._extract_quals(typenode) type = typenode.type if isinstance(type, pycparser.c_ast.IdentifierType): # assume a primitive type. get it from .names, but reduce # synonyms to a single chosen combination names = list(type.names) if names != ['signed', 'char']: # keep this unmodified prefixes = {} while names: name = names[0] if name in ('short', 'long', 'signed', 'unsigned'): prefixes[name] = prefixes.get(name, 0) + 1 del names[0] else: break # ignore the 'signed' prefix below, and reorder the others newnames = [] for prefix in ('unsigned', 'short', 'long'): for i in range(prefixes.get(prefix, 0)): newnames.append(prefix) if not names: names = ['int'] # implicitly if names == ['int']: # but kill it if 'short' or 'long' if 'short' in prefixes or 'long' in prefixes: names = [] names = newnames + names ident = ' '.join(names) if ident == 'void': return model.void_type, quals if ident == '__dotdotdot__': raise FFIError(':%d: bad usage of "..."' % typenode.coord.line) tp0, quals0 = resolve_common_type(self, ident) return tp0, (quals | quals0) # if isinstance(type, pycparser.c_ast.Struct): # 'struct foobar' tp = self._get_struct_union_enum_type('struct', type, name) return tp, quals # if isinstance(type, pycparser.c_ast.Union): # 'union foobar' tp = self._get_struct_union_enum_type('union', type, name) return tp, quals # if isinstance(type, pycparser.c_ast.Enum): # 'enum foobar' tp = self._get_struct_union_enum_type('enum', type, name) return tp, quals # if isinstance(typenode, pycparser.c_ast.FuncDecl): # a function type return self._parse_function_type(typenode, name), 0 # # nested anonymous structs or unions end up here if isinstance(typenode, pycparser.c_ast.Struct): return self._get_struct_union_enum_type('struct', typenode, name, nested=True), 0 if isinstance(typenode, pycparser.c_ast.Union): return self._get_struct_union_enum_type('union', typenode, name, nested=True), 0 # raise FFIError(":%d: bad or unsupported type declaration" % typenode.coord.line) def _parse_function_type(self, typenode, funcname=None): params = list(getattr(typenode.args, 'params', [])) for i, arg in enumerate(params): if not hasattr(arg, 'type'): raise CDefError("%s arg %d: unknown type '%s'" " (if you meant to use the old C syntax of giving" " untyped arguments, it is not supported)" % (funcname or 'in expression', i + 1, getattr(arg, 'name', '?'))) ellipsis = ( len(params) > 0 and isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and isinstance(params[-1].type.type, pycparser.c_ast.IdentifierType) and params[-1].type.type.names == ['__dotdotdot__']) if ellipsis: params.pop() if not params: raise CDefError( "%s: a function with only '(...)' as argument" " is not correct C" % (funcname or 'in expression')) args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type)) for argdeclnode in params] if not ellipsis and args == [model.void_type]: args = [] result, quals = self._get_type_and_quals(typenode.type) # the 'quals' on the result type are ignored. HACK: we absure them # to detect __stdcall functions: we textually replace "__stdcall" # with "volatile volatile const" above. abi = None if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']: abi = '__stdcall' return model.RawFunctionType(tuple(args), result, ellipsis, abi) def _as_func_arg(self, type, quals): if isinstance(type, model.ArrayType): return model.PointerType(type.item, quals) elif isinstance(type, model.RawFunctionType): return type.as_function_pointer() else: return type def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): # First, a level of caching on the exact 'type' node of the AST. # This is obscure, but needed because pycparser "unrolls" declarations # such as "typedef struct { } foo_t, *foo_p" and we end up with # an AST that is not a tree, but a DAG, with the "type" node of the # two branches foo_t and foo_p of the trees being the same node. # It's a bit silly but detecting "DAG-ness" in the AST tree seems # to be the only way to distinguish this case from two independent # structs. See test_struct_with_two_usages. try: return self._structnode2type[type] except KeyError: pass # # Note that this must handle parsing "struct foo" any number of # times and always return the same StructType object. Additionally, # one of these times (not necessarily the first), the fields of # the struct can be specified with "struct foo { ...fields... }". # If no name is given, then we have to create a new anonymous struct # with no caching; in this case, the fields are either specified # right now or never. # force_name = name name = type.name # # get the type or create it if needed if name is None: # 'force_name' is used to guess a more readable name for # anonymous structs, for the common case "typedef struct { } foo". if force_name is not None: explicit_name = '$%s' % force_name else: self._anonymous_counter += 1 explicit_name = '$%d' % self._anonymous_counter tp = None else: explicit_name = name key = '%s %s' % (kind, name) tp, _ = self._declarations.get(key, (None, None)) # if tp is None: if kind == 'struct': tp = model.StructType(explicit_name, None, None, None) elif kind == 'union': tp = model.UnionType(explicit_name, None, None, None) elif kind == 'enum': if explicit_name == '__dotdotdot__': raise CDefError("Enums cannot be declared with ...") tp = self._build_enum_type(explicit_name, type.values) else: raise AssertionError("kind = %r" % (kind,)) if name is not None: self._declare(key, tp) else: if kind == 'enum' and type.values is not None: raise NotImplementedError( "enum %s: the '{}' declaration should appear on the first " "time the enum is mentioned, not later" % explicit_name) if not tp.forcename: tp.force_the_name(force_name) if tp.forcename and '$' in tp.name: self._declare('anonymous %s' % tp.forcename, tp) # self._structnode2type[type] = tp # # enums: done here if kind == 'enum': return tp # # is there a 'type.decls'? If yes, then this is the place in the # C sources that declare the fields. If no, then just return the # existing type, possibly still incomplete. if type.decls is None: return tp # if tp.fldnames is not None: raise CDefError("duplicate declaration of struct %s" % name) fldnames = [] fldtypes = [] fldbitsize = [] fldquals = [] for decl in type.decls: if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and ''.join(decl.type.names) == '__dotdotdot__'): # XXX pycparser is inconsistent: 'names' should be a list # of strings, but is sometimes just one string. Use # str.join() as a way to cope with both. self._make_partial(tp, nested) continue if decl.bitsize is None: bitsize = -1 else: bitsize = self._parse_constant(decl.bitsize) self._partial_length = False type, fqual = self._get_type_and_quals(decl.type, partial_length_ok=True) if self._partial_length: self._make_partial(tp, nested) if isinstance(type, model.StructType) and type.partial: self._make_partial(tp, nested) fldnames.append(decl.name or '') fldtypes.append(type) fldbitsize.append(bitsize) fldquals.append(fqual) tp.fldnames = tuple(fldnames) tp.fldtypes = tuple(fldtypes) tp.fldbitsize = tuple(fldbitsize) tp.fldquals = tuple(fldquals) if fldbitsize != [-1] * len(fldbitsize): if isinstance(tp, model.StructType) and tp.partial: raise NotImplementedError("%s: using both bitfields and '...;'" % (tp,)) tp.packed = self._options.get('packed') if tp.completed: # must be re-completed: it is not opaque any more tp.completed = 0 self._recomplete.append(tp) return tp def _make_partial(self, tp, nested): if not isinstance(tp, model.StructOrUnion): raise CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) tp.partial = True def _parse_constant(self, exprnode, partial_length_ok=False): # for now, limited to expressions that are an immediate number # or positive/negative number if isinstance(exprnode, pycparser.c_ast.Constant): s = exprnode.value if '0' <= s[0] <= '9': s = s.rstrip('uUlL') try: if s.startswith('0'): return int(s, 8) else: return int(s, 10) except ValueError: if len(s) > 1: if s.lower()[0:2] == '0x': return int(s, 16) elif s.lower()[0:2] == '0b': return int(s, 2) raise CDefError("invalid constant %r" % (s,)) elif s[0] == "'" and s[-1] == "'" and ( len(s) == 3 or (len(s) == 4 and s[1] == "\\")): return ord(s[-2]) else: raise CDefError("invalid constant %r" % (s,)) # if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and exprnode.op == '+'): return self._parse_constant(exprnode.expr) # if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and exprnode.op == '-'): return -self._parse_constant(exprnode.expr) # load previously defined int constant if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name in self._int_constants): return self._int_constants[exprnode.name] # if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): if partial_length_ok: self._partial_length = True return '...' raise FFIError(":%d: unsupported '[...]' here, cannot derive " "the actual array length in this context" % exprnode.coord.line) # if isinstance(exprnode, pycparser.c_ast.BinaryOp): left = self._parse_constant(exprnode.left) right = self._parse_constant(exprnode.right) if exprnode.op == '+': return left + right elif exprnode.op == '-': return left - right elif exprnode.op == '*': return left * right elif exprnode.op == '/': return self._c_div(left, right) elif exprnode.op == '%': return left - self._c_div(left, right) * right elif exprnode.op == '<<': return left << right elif exprnode.op == '>>': return left >> right elif exprnode.op == '&': return left & right elif exprnode.op == '|': return left | right elif exprnode.op == '^': return left ^ right # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) def _c_div(self, a, b): result = a // b if ((a < 0) ^ (b < 0)) and (a % b) != 0: result += 1 return result def _build_enum_type(self, explicit_name, decls): if decls is not None: partial = False enumerators = [] enumvalues = [] nextenumvalue = 0 for enum in decls.enumerators: if _r_enum_dotdotdot.match(enum.name): partial = True continue if enum.value is not None: nextenumvalue = self._parse_constant(enum.value) enumerators.append(enum.name) enumvalues.append(nextenumvalue) self._add_constants(enum.name, nextenumvalue) nextenumvalue += 1 enumerators = tuple(enumerators) enumvalues = tuple(enumvalues) tp = model.EnumType(explicit_name, enumerators, enumvalues) tp.partial = partial else: # opaque enum tp = model.EnumType(explicit_name, (), ()) return tp def include(self, other): for name, (tp, quals) in other._declarations.items(): if name.startswith('anonymous $enum_$'): continue # fix for test_anonymous_enum_include kind = name.split(' ', 1)[0] if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef'): self._declare(name, tp, included=True, quals=quals) for k, v in other._int_constants.items(): self._add_constants(k, v) def _get_unknown_type(self, decl): typenames = decl.type.type.names if typenames == ['__dotdotdot__']: return model.unknown_type(decl.name) if typenames == ['__dotdotdotint__']: if self._uses_new_feature is None: self._uses_new_feature = "'typedef int... %s'" % decl.name return model.UnknownIntegerType(decl.name) if typenames == ['__dotdotdotfloat__']: # note: not for 'long double' so far if self._uses_new_feature is None: self._uses_new_feature = "'typedef float... %s'" % decl.name return model.UnknownFloatType(decl.name) raise FFIError(':%d: unsupported usage of "..." in typedef' % decl.coord.line) def _get_unknown_ptr_type(self, decl): if decl.type.type.type.names == ['__dotdotdot__']: return model.unknown_ptr_type(decl.name) raise FFIError(':%d: unsupported usage of "..." in typedef' % decl.coord.line)
Save File
Cancel