Lichen

Annotated modules.py

583:aed28d04304d
2017-02-13 Paul Boddie Re-added size information to string instances as the __size__ attribute. This fixes problems introduced when using strlen on data likely to contain embedded nulls, which was the reason for having size information explicitly stored in the first place. attr-strvalue-without-size
paul@0 1
#!/usr/bin/env python
paul@0 2
paul@0 3
"""
paul@0 4
Module abstractions.
paul@0 5
paul@0 6
Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013,
paul@479 7
              2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk>
paul@0 8
paul@0 9
This program is free software; you can redistribute it and/or modify it under
paul@0 10
the terms of the GNU General Public License as published by the Free Software
paul@0 11
Foundation; either version 3 of the License, or (at your option) any later
paul@0 12
version.
paul@0 13
paul@0 14
This program is distributed in the hope that it will be useful, but WITHOUT
paul@0 15
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
paul@0 16
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
paul@0 17
details.
paul@0 18
paul@0 19
You should have received a copy of the GNU General Public License along with
paul@0 20
this program.  If not, see <http://www.gnu.org/licenses/>.
paul@0 21
"""
paul@0 22
paul@538 23
from common import get_builtin_class, get_builtin_module, init_item, \
paul@538 24
                   remove_items, CommonModule
paul@553 25
from encoders import decode_modifiers, decode_usage, encode_modifiers, encode_usage
paul@0 26
from referencing import decode_reference, Reference
paul@12 27
from results import ResolvedNameRef
paul@0 28
import sys
paul@0 29
paul@0 30
class BasicModule(CommonModule):
paul@0 31
paul@0 32
    "The basic module information."
paul@0 33
paul@0 34
    def __init__(self, name, importer):
paul@0 35
        CommonModule.__init__(self, name, importer)
paul@0 36
paul@18 37
        # Import details, primarily for cache output.
paul@18 38
paul@18 39
        self.imports = set()
paul@18 40
        self.required = set()
paul@35 41
        self.deferred = []
paul@18 42
paul@0 43
        # Global name information.
paul@0 44
paul@0 45
        self.objects = {}
paul@0 46
        self.special = {}
paul@0 47
paul@0 48
        # Class relationships.
paul@0 49
paul@0 50
        self.classes = {}
paul@0 51
paul@0 52
        # Attributes.
paul@0 53
paul@0 54
        self.class_attrs = {}
paul@0 55
        self.instance_attrs = {}
paul@0 56
        self.instance_attr_constants = {}
paul@0 57
        self.module_attrs = set()
paul@0 58
paul@40 59
        # Names used in each namespace.
paul@0 60
paul@0 61
        self.names_used = {}
paul@0 62
paul@0 63
        # Function details.
paul@0 64
paul@0 65
        self.function_parameters = {}
paul@0 66
        self.function_defaults = {}
paul@0 67
        self.function_locals = {}
paul@0 68
        self.scope_globals = {}
paul@0 69
paul@0 70
        # Invocation details.
paul@0 71
paul@0 72
        self.function_targets = {}
paul@0 73
        self.function_arguments = {}
paul@0 74
paul@479 75
        # Exception handler details.
paul@479 76
paul@479 77
        self.exception_namespaces = set()
paul@479 78
paul@0 79
        # Attribute usage at module and function levels.
paul@0 80
paul@0 81
        self.attr_usage = {}
paul@0 82
        self.name_initialisers = {}
paul@0 83
paul@0 84
        # General attribute access expressions.
paul@0 85
paul@0 86
        self.attr_accesses = {}
paul@0 87
        self.const_accesses = {}
paul@0 88
paul@0 89
        # Attribute accessor definition details.
paul@0 90
paul@0 91
        self.attr_accessors = {}
paul@0 92
paul@0 93
        # Assignment details for accesses.
paul@0 94
paul@0 95
        self.attr_access_modifiers = {}
paul@0 96
paul@13 97
        # Name resolution details.
paul@13 98
paul@13 99
        self.name_references = {} # references to globals
paul@13 100
paul@13 101
        # Initialisation-related details.
paul@13 102
paul@13 103
        self.initialised_names = {}
paul@13 104
        self.aliased_names = {}
paul@13 105
paul@0 106
    def __repr__(self):
paul@0 107
        return "BasicModule(%r, %r)" % (self.name, self.importer)
paul@0 108
paul@0 109
    # Derived information methods.
paul@0 110
paul@0 111
    def propagate(self):
paul@0 112
paul@0 113
        "Finalise and propagate module information."
paul@0 114
paul@0 115
        self.propagate_attrs()
paul@0 116
        self.propagate_name_references()
paul@0 117
        self.propagate_attr_accesses()
paul@0 118
        self.propagate_constants()
paul@0 119
paul@0 120
    def unpropagate(self):
paul@0 121
paul@0 122
        """
paul@0 123
        Retract information from the importer including information about this
paul@0 124
        module derived by the importer.
paul@0 125
        """
paul@0 126
paul@0 127
        del self.importer.all_module_attrs[self.name]
paul@0 128
paul@0 129
        for name in self.classes.keys():
paul@68 130
            del self.importer.classes[name]
paul@0 131
            del self.importer.all_class_attrs[name]
paul@0 132
            del self.importer.all_instance_attrs[name]
paul@0 133
            del self.importer.all_instance_attr_constants[name]
paul@0 134
paul@0 135
            for name, bases in self.classes.items():
paul@0 136
                for base in bases:
paul@0 137
paul@0 138
                    # Get the identity of the class from the reference.
paul@0 139
paul@0 140
                    base = base.get_origin()
paul@0 141
paul@0 142
                    try:
paul@0 143
                        self.importer.subclasses[base].remove(name)
paul@0 144
                    except (KeyError, ValueError):
paul@0 145
                        pass
paul@0 146
paul@0 147
        remove_items(self.importer.all_name_references, self.name_references)
paul@0 148
        remove_items(self.importer.all_initialised_names, self.initialised_names)
paul@0 149
        remove_items(self.importer.all_aliased_names, self.aliased_names)
paul@0 150
        remove_items(self.importer.all_attr_accesses, self.attr_accesses)
paul@0 151
        remove_items(self.importer.all_const_accesses, self.const_accesses)
paul@0 152
        remove_items(self.importer.all_attr_access_modifiers, self.attr_access_modifiers)
paul@0 153
        remove_items(self.importer.all_constants, self.constants)
paul@0 154
        remove_items(self.importer.all_constant_values, self.constant_values)
paul@0 155
paul@0 156
        # Remove this module's objects from the importer. Objects are
paul@0 157
        # automatically propagated when defined.
paul@0 158
paul@68 159
        ref = self.importer.objects.get(self.name)
paul@68 160
        if ref and ref.has_kind("<module>"):
paul@68 161
            del self.importer.objects[self.name]
paul@68 162
paul@0 163
        for name, ref in self.objects.items():
paul@20 164
            if not ref.has_kind("<module>"):
paul@20 165
                del self.importer.objects[name]
paul@0 166
paul@68 167
    def collect(self):
paul@68 168
paul@68 169
        "Collect removed objects."
paul@68 170
paul@68 171
        for name, ref in self.objects.items():
paul@68 172
            if not self.importer.objects.has_key(ref.get_origin()) and self.importer.objects.has_key(name):
paul@68 173
                del self.importer.objects[name]
paul@68 174
paul@0 175
    def propagate_attrs(self):
paul@0 176
paul@0 177
        "Derive attributes from the class and module member details."
paul@0 178
paul@0 179
        # Initialise class attribute records for all classes.
paul@0 180
paul@0 181
        for name in self.classes.keys():
paul@0 182
            self.importer.all_class_attrs[name] = self.class_attrs[name] = {}
paul@0 183
paul@0 184
        # Separate the objects into module and class attributes.
paul@0 185
paul@0 186
        for name in self.objects.keys():
paul@0 187
            if "." in name:
paul@0 188
                parent, attrname = name.rsplit(".", 1)
paul@0 189
                if self.classes.has_key(parent):
paul@0 190
                    self.class_attrs[parent][attrname] = name
paul@0 191
                elif parent == self.name:
paul@0 192
                    self.module_attrs.add(attrname)
paul@0 193
paul@0 194
        # Propagate the module attributes.
paul@0 195
paul@0 196
        self.importer.all_module_attrs[self.name] = self.module_attrs
paul@0 197
paul@0 198
    def propagate_name_references(self):
paul@0 199
paul@0 200
        "Propagate name references for the module."
paul@0 201
paul@0 202
        self.importer.all_initialised_names.update(self.initialised_names)
paul@0 203
        self.importer.all_aliased_names.update(self.aliased_names)
paul@0 204
paul@0 205
    def propagate_attr_accesses(self):
paul@0 206
paul@0 207
        "Propagate attribute accesses for the module."
paul@0 208
paul@0 209
        self.importer.all_attr_accesses.update(self.attr_accesses)
paul@0 210
        self.importer.all_const_accesses.update(self.const_accesses)
paul@0 211
        self.importer.all_attr_access_modifiers.update(self.attr_access_modifiers)
paul@0 212
paul@0 213
    def propagate_constants(self):
paul@0 214
paul@0 215
        "Propagate constant values and aliases for the module."
paul@0 216
paul@0 217
        self.importer.all_constants.update(self.constants)
paul@0 218
        self.importer.all_constant_values.update(self.constant_values)
paul@0 219
paul@0 220
        for name in self.classes.keys():
paul@0 221
            self.importer.all_instance_attrs[name] = self.instance_attrs.get(name) or {}
paul@0 222
            self.importer.all_instance_attr_constants[name] = self.instance_attr_constants.get(name) or {}
paul@0 223
paul@26 224
    def set_object(self, name, value=None):
paul@26 225
paul@26 226
        "Set an object with the given 'name' and the given 'value'."
paul@26 227
paul@35 228
        # Decode any string value, with a new reference being returned even
paul@35 229
        # given a provided reference.
paul@35 230
paul@26 231
        ref = decode_reference(value, name)
paul@35 232
        self.add_deferred(ref)
paul@35 233
        self._set_object(name, ref)
paul@35 234
paul@35 235
    def _set_object(self, name, ref):
paul@35 236
paul@35 237
        # Determine how the object properties will be defined.
paul@35 238
paul@319 239
        multiple = self.objects.has_key(name) and self.objects[name] != ref
paul@26 240
        self.importer.objects[name] = self.objects[name] = multiple and ref.as_var() or ref
paul@26 241
paul@26 242
    def queue_module(self, name, required=False):
paul@26 243
paul@26 244
        """
paul@26 245
        Queue the module with the given 'name'. If 'required' is true (it is
paul@26 246
        false by default), indicate that the module is required in the final
paul@26 247
        program.
paul@26 248
        """
paul@26 249
paul@26 250
        self.importer.queue_module(name, self, required)
paul@26 251
        if required:
paul@26 252
            self.required.add(name)
paul@26 253
        self.imports.add(name)
paul@26 254
paul@26 255
class InspectionNaming:
paul@26 256
paul@26 257
    "Name operations related to inspection."
paul@26 258
paul@0 259
    # Module-relative naming.
paul@0 260
paul@0 261
    def is_global(self, name):
paul@0 262
paul@0 263
        """
paul@0 264
        Return whether 'name' is registered as a global in the current
paul@0 265
        namespace.
paul@0 266
        """
paul@0 267
paul@0 268
        path = self.get_namespace_path()
paul@0 269
        return name in self.scope_globals.get(path, [])
paul@0 270
paul@0 271
    def get_global(self, name):
paul@0 272
paul@0 273
        """
paul@0 274
        Get the global of the given 'name' from this module, returning a
paul@0 275
        reference incorporating the original definition details.
paul@0 276
        """
paul@0 277
paul@0 278
        path = self.get_global_path(name)
paul@0 279
        return self.objects.get(path)
paul@0 280
paul@0 281
    # Name definition discovery.
paul@0 282
paul@0 283
    def get_global_or_builtin(self, name):
paul@0 284
paul@0 285
        """
paul@12 286
        Return a reference for the given 'name' found in this module or in the
paul@12 287
        __builtins__.
paul@0 288
        """
paul@0 289
paul@0 290
        return self.get_global(name) or self.get_builtin(name)
paul@0 291
paul@0 292
    def get_builtin(self, name):
paul@0 293
paul@12 294
        "Return a reference to the built-in with the given 'name'."
paul@0 295
paul@18 296
        self.queue_module("__builtins__")
paul@35 297
        ref = Reference("<depends>", "__builtins__.%s" % name)
paul@35 298
        self.deferred.append(ref)
paul@35 299
        return ref
paul@0 300
paul@12 301
    def get_builtin_class(self, name):
paul@0 302
paul@12 303
        "Return a reference to the actual object providing 'name'."
paul@0 304
paul@538 305
        objpath = get_builtin_class(name)
paul@538 306
        module_name = get_builtin_module(name)
paul@188 307
paul@30 308
        if self.name != module_name:
paul@30 309
            self.queue_module(module_name, True)
paul@366 310
paul@538 311
        return Reference("<class>", objpath)
paul@0 312
paul@152 313
    def get_object(self, path, defer=True):
paul@0 314
paul@0 315
        """
paul@12 316
        Get the details of an object with the given 'path'. Where the object
paul@152 317
        cannot be resolved, an unresolved reference is returned if 'defer' is
paul@152 318
        set to a true value (the default). Otherwise, None is returned.
paul@0 319
        """
paul@0 320
paul@0 321
        if self.objects.has_key(path):
paul@0 322
            return self.objects[path]
paul@152 323
        elif defer:
paul@35 324
            ref = Reference("<depends>", path)
paul@35 325
            self.deferred.append(ref)
paul@35 326
            return ref
paul@152 327
        else:
paul@152 328
            return None
paul@0 329
paul@12 330
    def import_name_from_module(self, name, module_name):
paul@12 331
paul@12 332
        "Import 'name' from the module having the given 'module_name'."
paul@12 333
paul@12 334
        if module_name != self.name:
paul@18 335
            self.queue_module(module_name)
paul@166 336
        ref = Reference("<depends>", "%s.%s" % (module_name, name))
paul@166 337
        self.deferred.append(ref)
paul@166 338
        return ref
paul@12 339
paul@35 340
    def add_deferred(self, ref):
paul@35 341
paul@35 342
        "Record 'ref' as a deferred reference."
paul@35 343
paul@35 344
        if ref.has_kind("<depends>"):
paul@35 345
            self.deferred.append(ref)
paul@35 346
paul@0 347
class CachedModule(BasicModule):
paul@0 348
paul@0 349
    "A cached module."
paul@0 350
paul@0 351
    def __repr__(self):
paul@0 352
        return "CachedModule(%r, %r)" % (self.name, self.importer)
paul@0 353
paul@35 354
    def set_object(self, name, value=None):
paul@35 355
paul@35 356
        "Set an object with the given 'name' and the given 'value'."
paul@35 357
paul@35 358
        # Decode any string value, with a new reference being returned even
paul@35 359
        # given a provided reference.
paul@35 360
paul@35 361
        ref = decode_reference(value, name)
paul@35 362
        self._set_object(name, ref)
paul@35 363
paul@0 364
    def to_cache(self, filename):
paul@0 365
paul@0 366
        "Not actually writing the module back to 'filename'."
paul@0 367
paul@0 368
        pass
paul@0 369
paul@0 370
    def from_cache(self, filename):
paul@0 371
paul@0 372
        """
paul@0 373
        Read a module's details from the file with the given 'filename' as
paul@0 374
        described in the to_cache method of InspectedModule.
paul@0 375
        """
paul@0 376
paul@0 377
        f = open(filename)
paul@0 378
        try:
paul@0 379
            self.filename = f.readline().rstrip()
paul@0 380
paul@0 381
            f.readline() # (empty line)
paul@0 382
paul@18 383
            self._get_imports(f)
paul@391 384
            self._get_deferred(f)
paul@134 385
            self._get_special(f)
paul@0 386
            self._get_members(f)
paul@0 387
            self._get_class_relationships(f)
paul@0 388
            self._get_instance_attrs(f)
paul@0 389
            self._get_instance_attr_constants(f)
paul@0 390
            self.from_lines(f, self.names_used)     # "names used:"
paul@0 391
            self._get_name_references(f)
paul@0 392
            self._get_initialised_names(f)
paul@0 393
            self._get_aliased_names(f)
paul@0 394
            self._get_function_parameters(f)
paul@0 395
            self._get_function_defaults(f)
paul@0 396
            self._get_function_locals(f)
paul@0 397
            self.from_lines(f, self.scope_globals)  # "scope globals:"
paul@0 398
            self._get_function_targets(f)
paul@0 399
            self._get_function_arguments(f)
paul@0 400
            self._get_attribute_usage(f)
paul@0 401
            self._get_attr_accesses(f)
paul@0 402
            self._get_const_accesses(f)
paul@0 403
            self._get_attr_accessors(f)
paul@0 404
            self._get_attr_access_modifiers(f)
paul@0 405
            self._get_constant_literals(f)
paul@0 406
            self._get_constant_values(f)
paul@479 407
            self._get_exception_namespaces(f)
paul@0 408
paul@0 409
        finally:
paul@0 410
            f.close()
paul@0 411
paul@12 412
    def complete(self):
paul@0 413
        self.propagate()
paul@0 414
paul@18 415
    def _get_imports(self, f):
paul@18 416
        f.readline() # "imports:"
paul@18 417
        line = f.readline().strip()
paul@18 418
        self.required = line != "{}" and set(line.split(", ")) or set()
paul@18 419
        line = f.readline().strip()
paul@18 420
        self.imports = line != "{}" and set(line.split(", ")) or set()
paul@18 421
        f.readline()
paul@18 422
paul@18 423
        for name in self.required:
paul@18 424
            self.queue_module(name, True)
paul@18 425
        for name in self.imports:
paul@18 426
            self.queue_module(name)
paul@18 427
paul@391 428
    def _get_deferred(self, f):
paul@391 429
        f.readline() # "deferred:"
paul@391 430
        line = f.readline().rstrip()
paul@524 431
        self.deferred = map(decode_reference, line.split(", "))
paul@391 432
        f.readline()
paul@391 433
paul@134 434
    def _get_special(self, f):
paul@134 435
        f.readline() # "special:"
paul@134 436
        line = f.readline().rstrip()
paul@134 437
        while line:
paul@423 438
            name, ref, paths = self._get_fields(line, 3)
paul@423 439
            self.special[name] = decode_reference(ref), paths.split(", ")
paul@134 440
            line = f.readline().rstrip()
paul@134 441
paul@0 442
    def _get_members(self, f):
paul@0 443
        f.readline() # "members:"
paul@0 444
        line = f.readline().rstrip()
paul@0 445
        while line:
paul@0 446
            name, ref = line.split(" ", 1)
paul@0 447
            self.set_object(name, ref)
paul@0 448
            line = f.readline().rstrip()
paul@0 449
paul@0 450
    def _get_class_relationships(self, f):
paul@0 451
        f.readline() # "class relationships:"
paul@0 452
        line = f.readline().rstrip()
paul@0 453
        while line:
paul@0 454
            name, value = self._get_fields(line)
paul@0 455
            values = value and value.split(", ") or []
paul@0 456
            self.importer.classes[name] = self.classes[name] = map(decode_reference, values)
paul@0 457
            self.importer.subclasses[name] = set()
paul@0 458
            line = f.readline().rstrip()
paul@0 459
paul@0 460
    def _get_instance_attrs(self, f):
paul@0 461
        f.readline() # "instance attributes:"
paul@0 462
        line = f.readline().rstrip()
paul@0 463
        while line:
paul@0 464
            name, value = self._get_fields(line)
paul@0 465
            self.importer.all_instance_attrs[name] = self.instance_attrs[name] = set(value and value.split(", ") or [])
paul@0 466
            line = f.readline().rstrip()
paul@0 467
paul@0 468
    def _get_instance_attr_constants(self, f):
paul@0 469
        f.readline() # "instance attribute constants:"
paul@0 470
        line = f.readline().rstrip()
paul@0 471
        while line:
paul@0 472
            name, attrname, ref = self._get_fields(line, 3)
paul@0 473
            init_item(self.instance_attr_constants, name, dict)
paul@0 474
            self.instance_attr_constants[name][attrname] = decode_reference(ref)
paul@0 475
            line = f.readline().rstrip()
paul@0 476
paul@0 477
    def _get_name_references(self, f):
paul@0 478
        f.readline() # "name references:"
paul@0 479
        line = f.readline().rstrip()
paul@0 480
        while line:
paul@37 481
            name, ref = self._get_fields(line)
paul@37 482
            self.importer.all_name_references[name] = self.name_references[name] = decode_reference(ref)
paul@0 483
            line = f.readline().rstrip()
paul@0 484
paul@0 485
    def _get_initialised_names(self, f):
paul@0 486
        f.readline() # "initialised names:"
paul@0 487
        line = f.readline().rstrip()
paul@0 488
        while line:
paul@0 489
            name, version, value = self._get_fields(line, 3)
paul@0 490
            init_item(self.initialised_names, name, dict)
paul@0 491
            self.initialised_names[name][int(version)] = decode_reference(value)
paul@0 492
            line = f.readline().rstrip()
paul@0 493
paul@0 494
    def _get_aliased_names(self, f):
paul@0 495
        f.readline() # "aliased names:"
paul@0 496
        line = f.readline().rstrip()
paul@0 497
        while line:
paul@0 498
            name, version, original_name, attrnames, number = self._get_fields(line, 5)
paul@0 499
            init_item(self.aliased_names, name, dict)
paul@0 500
            if number == "{}": number = None
paul@0 501
            else: number = int(number)
paul@0 502
            self.aliased_names[name][int(version)] = (original_name, attrnames != "{}" and attrnames or None, number)
paul@0 503
            line = f.readline().rstrip()
paul@0 504
paul@0 505
    def _get_function_parameters(self, f):
paul@0 506
        f.readline() # "function parameters:"
paul@0 507
        line = f.readline().rstrip()
paul@0 508
        while line:
paul@0 509
            function, names = self._get_fields(line)
paul@0 510
            self.importer.function_parameters[function] = \
paul@49 511
                self.function_parameters[function] = names != "{}" and names.split(", ") or []
paul@0 512
            line = f.readline().rstrip()
paul@0 513
paul@0 514
    def _get_function_defaults(self, f):
paul@0 515
        f.readline() # "function default parameters:"
paul@0 516
        line = f.readline().rstrip()
paul@0 517
        while line:
paul@0 518
            function, defaults = self._get_fields(line)
paul@0 519
            self.importer.function_defaults[function] = \
paul@0 520
                self.function_defaults[function] = l = []
paul@0 521
            if defaults != "{}":
paul@0 522
                for value in defaults.split(", "):
paul@0 523
                    name, default = value.split("=")
paul@0 524
                    default = decode_reference(default)
paul@0 525
                    l.append((name, default))
paul@0 526
            line = f.readline().rstrip()
paul@0 527
paul@0 528
    def _get_function_locals(self, f):
paul@0 529
        f.readline() # "function locals:"
paul@0 530
        line = f.readline().rstrip()
paul@0 531
        while line:
paul@0 532
            function, name, value = self._get_fields(line, 3)
paul@0 533
            init_item(self.function_locals, function, dict)
paul@115 534
            init_item(self.importer.function_locals, function, dict)
paul@0 535
            if name != "{}":
paul@115 536
                self.importer.function_locals[function][name] = \
paul@109 537
                    self.function_locals[function][name] = decode_reference(value)
paul@0 538
            line = f.readline().rstrip()
paul@0 539
paul@0 540
    def _get_function_targets(self, f):
paul@0 541
        f.readline() # "function targets:"
paul@0 542
        line = f.readline().rstrip()
paul@0 543
        while line:
paul@0 544
            function, n = self._get_fields(line)
paul@0 545
            self.importer.function_targets[function] = \
paul@0 546
                self.function_targets[function] = int(n)
paul@0 547
            line = f.readline().rstrip()
paul@0 548
paul@0 549
    def _get_function_arguments(self, f):
paul@0 550
        f.readline() # "function arguments:"
paul@0 551
        line = f.readline().rstrip()
paul@0 552
        while line:
paul@0 553
            function, n = self._get_fields(line)
paul@0 554
            self.importer.function_arguments[function] = \
paul@0 555
                self.function_arguments[function] = int(n)
paul@0 556
            line = f.readline().rstrip()
paul@0 557
paul@0 558
    def _get_attribute_usage(self, f):
paul@0 559
        f.readline() # "attribute usage:"
paul@0 560
        line = f.readline().rstrip()
paul@0 561
        while line:
paul@0 562
            unit, value = self._get_fields(line)
paul@0 563
            init_item(self.attr_usage, unit, dict)
paul@0 564
            self.usage_from_cache(value, self.attr_usage[unit])
paul@0 565
            line = f.readline().rstrip()
paul@0 566
paul@0 567
    def _get_attr_accesses(self, f):
paul@0 568
        f.readline() # "attribute accesses:"
paul@0 569
        line = f.readline().rstrip()
paul@0 570
        while line:
paul@0 571
            name, value = self._get_fields(line)
paul@0 572
            self.attr_accesses[name] = set(value.split(", "))
paul@0 573
            line = f.readline().rstrip()
paul@0 574
paul@0 575
    def _get_const_accesses(self, f):
paul@0 576
        f.readline() # "constant accesses:"
paul@0 577
        line = f.readline().rstrip()
paul@0 578
        while line:
paul@0 579
            name, original_name, attrnames, objpath, ref, remaining = self._get_fields(line, 6)
paul@0 580
            if attrnames == "{}": attrnames = None
paul@0 581
            init_item(self.const_accesses, name, dict)
paul@0 582
            self.const_accesses[name][(original_name, attrnames)] = (objpath, decode_reference(ref), remaining != "{}" and remaining or "")
paul@0 583
            line = f.readline().rstrip()
paul@0 584
paul@0 585
    def _get_attr_accessors(self, f):
paul@0 586
        f.readline() # "attribute access usage:"
paul@0 587
        line = f.readline().rstrip()
paul@0 588
        while line:
paul@0 589
            objpath, name, attrname, value = self._get_fields(line, 4)
paul@0 590
            if attrname == "{}": attrname = None
paul@0 591
            access = name, attrname
paul@0 592
            init_item(self.attr_accessors, objpath, dict)
paul@0 593
            init_item(self.attr_accessors[objpath], access, list)
paul@0 594
            positions = map(int, value.split(", "))
paul@0 595
            self.attr_accessors[objpath][access].append(positions)
paul@0 596
            line = f.readline().rstrip()
paul@0 597
paul@0 598
    def _get_attr_access_modifiers(self, f):
paul@0 599
        f.readline() # "attribute access modifiers:"
paul@0 600
        line = f.readline().rstrip()
paul@0 601
        while line:
paul@0 602
            objpath, name, attrnames, value = self._get_fields(line, 4)
paul@0 603
            if name == "{}": name = None
paul@0 604
            if attrnames == "{}": attrnames = None
paul@0 605
            access = name, attrnames
paul@0 606
            init_item(self.attr_access_modifiers, objpath, dict)
paul@0 607
            init_item(self.attr_access_modifiers[objpath], access, list)
paul@553 608
            modifiers = decode_modifiers(value)
paul@0 609
            self.attr_access_modifiers[objpath][access] = modifiers
paul@0 610
            line = f.readline().rstrip()
paul@0 611
paul@0 612
    def _get_constant_literals(self, f):
paul@0 613
        f.readline() # "constant literals:"
paul@0 614
        line = f.readline().rstrip()
paul@0 615
        last_path = None
paul@0 616
        n = None
paul@0 617
        while line:
paul@406 618
            path, value_type, encoding, value = self._get_fields(line, 4)
paul@0 619
            if path != last_path:
paul@0 620
                n = 0
paul@0 621
                last_path = path
paul@0 622
            else:
paul@0 623
                n += 1
paul@0 624
            init_item(self.constants, path, dict)
paul@397 625
            value = eval(value)
paul@406 626
            encoding = encoding != "{}" and encoding or None
paul@406 627
            self.constants[path][(value, value_type, encoding)] = n
paul@0 628
            line = f.readline().rstrip()
paul@0 629
paul@0 630
    def _get_constant_values(self, f):
paul@0 631
        f.readline() # "constant values:"
paul@0 632
        line = f.readline().rstrip()
paul@0 633
        while line:
paul@406 634
            name, value_type, encoding, value = self._get_fields(line, 4)
paul@406 635
            value = eval(value)
paul@406 636
            encoding = encoding != "{}" and encoding or None
paul@406 637
            self.constant_values[name] = value, value_type, encoding
paul@0 638
            line = f.readline().rstrip()
paul@0 639
paul@479 640
    def _get_exception_namespaces(self, f):
paul@479 641
        f.readline() # "exception namespaces:"
paul@481 642
        value = f.readline().rstrip()
paul@481 643
        self.exception_namespaces = value and set(value.split(", ")) or set()
paul@479 644
        f.readline()
paul@479 645
paul@0 646
    # Generic parsing methods.
paul@0 647
paul@0 648
    def from_lines(self, f, d):
paul@0 649
paul@0 650
        "Read lines from 'f', populating 'd'."
paul@0 651
paul@0 652
        f.readline() # section heading
paul@0 653
        line = f.readline().rstrip()
paul@0 654
        while line:
paul@0 655
            name, value = self._get_fields(line)
paul@0 656
            d[name] = set(value and value.split(", ") or [])
paul@0 657
            line = f.readline().rstrip()
paul@0 658
paul@0 659
    def usage_from_cache(self, value, mapping):
paul@0 660
paul@0 661
        """
paul@0 662
        Interpret the given 'value' containing name and usage information,
paul@0 663
        storing the information in the given 'mapping'.
paul@0 664
        """
paul@0 665
paul@0 666
        local, usage = self._get_fields(value)
paul@0 667
        init_item(mapping, local, list)
paul@0 668
        self._usage_from_cache(mapping[local], usage)
paul@0 669
paul@0 670
    def _usage_from_cache(self, d, usage):
paul@0 671
paul@0 672
        # Interpret descriptions of each version of the name.
paul@0 673
paul@0 674
        all_usages = set()
paul@0 675
        for attrnames in usage.split("; "):
paul@0 676
            if attrnames == "{}":
paul@0 677
                all_attrnames = ()
paul@0 678
            else:
paul@88 679
                all_attrnames = decode_usage(attrnames)
paul@88 680
            all_usages.add(all_attrnames)
paul@0 681
paul@0 682
        d.append(all_usages)
paul@0 683
paul@0 684
    def _get_fields(self, s, n=2):
paul@0 685
        result = s.split(" ", n-1)
paul@0 686
        if len(result) == n:
paul@0 687
            return result
paul@0 688
        else:
paul@0 689
            return tuple(result) + tuple([""] * (n - len(result)))
paul@0 690
paul@0 691
class CacheWritingModule:
paul@0 692
paul@0 693
    """
paul@0 694
    A mix-in providing cache-writing support, to be combined with BasicModule.
paul@0 695
    """
paul@0 696
paul@0 697
    def to_cache(self, filename):
paul@0 698
paul@0 699
        """
paul@524 700
        Write a cached representation of the inspected module to the file having
paul@524 701
        the given 'filename'.
paul@0 702
        """
paul@0 703
paul@0 704
        f = open(filename, "w")
paul@0 705
        try:
paul@0 706
            print >>f, self.filename
paul@0 707
paul@0 708
            print >>f
paul@18 709
            print >>f, "imports:"
paul@18 710
            required = list(self.required)
paul@18 711
            required.sort()
paul@18 712
            print >>f, required and ", ".join(required) or "{}"
paul@18 713
            imports = list(self.imports)
paul@18 714
            imports.sort()
paul@18 715
            print >>f, imports and ", ".join(imports) or "{}"
paul@18 716
paul@18 717
            print >>f
paul@391 718
            print >>f, "deferred:"
paul@471 719
            deferred = map(str, set(self.deferred))
paul@471 720
            deferred.sort()
paul@524 721
            print >>f, ", ".join(deferred)
paul@391 722
paul@391 723
            print >>f
paul@134 724
            print >>f, "special:"
paul@134 725
            names = self.special.keys()
paul@134 726
            names.sort()
paul@134 727
            for name in names:
paul@423 728
                ref, paths = self.special[name]
paul@423 729
                print >>f, name, ref, ", ".join(paths)
paul@134 730
paul@134 731
            print >>f
paul@0 732
            print >>f, "members:"
paul@0 733
            objects = self.objects.keys()
paul@0 734
            objects.sort()
paul@0 735
            for name in objects:
paul@0 736
                print >>f, name, self.objects[name]
paul@0 737
paul@0 738
            print >>f
paul@0 739
            print >>f, "class relationships:"
paul@0 740
            classes = self.classes.keys()
paul@0 741
            classes.sort()
paul@0 742
            for class_ in classes:
paul@0 743
                bases = self.classes[class_]
paul@0 744
                if bases:
paul@0 745
                    print >>f, class_, ", ".join(map(str, bases))
paul@0 746
                else:
paul@0 747
                    print >>f, class_
paul@0 748
paul@0 749
            self.to_lines(f, "instance attributes:", self.instance_attrs)
paul@0 750
paul@0 751
            print >>f
paul@0 752
            print >>f, "instance attribute constants:"
paul@0 753
            classes = self.instance_attr_constants.items()
paul@0 754
            classes.sort()
paul@0 755
            for name, attrs in classes:
paul@0 756
                attrs = attrs.items()
paul@0 757
                attrs.sort()
paul@0 758
                for attrname, ref in attrs:
paul@0 759
                    print >>f, name, attrname, ref
paul@0 760
paul@0 761
            self.to_lines(f, "names used:", self.names_used)
paul@0 762
paul@0 763
            print >>f
paul@0 764
            print >>f, "name references:"
paul@0 765
            refs = self.name_references.items()
paul@0 766
            refs.sort()
paul@0 767
            for name, ref in refs:
paul@0 768
                print >>f, name, ref
paul@0 769
paul@0 770
            print >>f
paul@0 771
            print >>f, "initialised names:"
paul@0 772
            assignments = self.initialised_names.items()
paul@0 773
            assignments.sort()
paul@0 774
            for name, refs in assignments:
paul@0 775
                versions = refs.items()
paul@0 776
                versions.sort()
paul@0 777
                for version, ref in versions:
paul@0 778
                    print >>f, name, version, ref
paul@0 779
paul@0 780
            print >>f
paul@0 781
            print >>f, "aliased names:"
paul@0 782
            assignments = self.aliased_names.items()
paul@0 783
            assignments.sort()
paul@0 784
            for name, aliases in assignments:
paul@0 785
                versions = aliases.items()
paul@0 786
                versions.sort()
paul@0 787
                for version, alias in versions:
paul@0 788
                    original_name, attrnames, number = alias
paul@0 789
                    print >>f, name, version, original_name, attrnames or "{}", number is None and "{}" or number
paul@0 790
paul@0 791
            print >>f
paul@0 792
            print >>f, "function parameters:"
paul@0 793
            functions = self.function_parameters.keys()
paul@0 794
            functions.sort()
paul@0 795
            for function in functions:
paul@49 796
                parameters = self.function_parameters[function]
paul@49 797
                if parameters:
paul@49 798
                    print >>f, function, ", ".join(parameters)
paul@49 799
                else:
paul@49 800
                    print >>f, function, "{}"
paul@0 801
paul@0 802
            print >>f
paul@0 803
            print >>f, "function default parameters:"
paul@0 804
            functions = self.function_defaults.keys()
paul@0 805
            functions.sort()
paul@0 806
            for function in functions:
paul@0 807
                parameters = self.function_defaults[function]
paul@0 808
                if parameters:
paul@0 809
                    print >>f, function, ", ".join([("%s=%s" % (name, default)) for (name, default) in parameters])
paul@0 810
                else:
paul@0 811
                    print >>f, function, "{}"
paul@0 812
paul@0 813
            print >>f
paul@0 814
            print >>f, "function locals:"
paul@0 815
            functions = self.function_locals.keys()
paul@0 816
            functions.sort()
paul@0 817
            for function in functions:
paul@0 818
                names = self.function_locals[function].items()
paul@0 819
                if names:
paul@0 820
                    names.sort()
paul@0 821
                    for name, value in names:
paul@0 822
                        print >>f, function, name, value
paul@0 823
                else:
paul@0 824
                    print >>f, function, "{}"
paul@0 825
paul@0 826
            self.to_lines(f, "scope globals:", self.scope_globals)
paul@0 827
paul@0 828
            print >>f
paul@0 829
            print >>f, "function targets:"
paul@0 830
            functions = self.function_targets.keys()
paul@0 831
            functions.sort()
paul@0 832
            for function in functions:
paul@0 833
                print >>f, function, self.function_targets[function]
paul@0 834
paul@0 835
            print >>f
paul@0 836
            print >>f, "function arguments:"
paul@0 837
            functions = self.function_arguments.keys()
paul@0 838
            functions.sort()
paul@0 839
            for function in functions:
paul@0 840
                print >>f, function, self.function_arguments[function]
paul@0 841
paul@0 842
            print >>f
paul@0 843
            print >>f, "attribute usage:"
paul@0 844
            units = self.attr_usage.keys()
paul@0 845
            units.sort()
paul@0 846
            for unit in units:
paul@0 847
                d = self.attr_usage[unit]
paul@0 848
                self.usage_to_cache(d, f, unit)
paul@0 849
paul@0 850
            print >>f
paul@0 851
            print >>f, "attribute accesses:"
paul@0 852
            paths = self.attr_accesses.keys()
paul@0 853
            paths.sort()
paul@0 854
            for path in paths:
paul@0 855
                accesses = list(self.attr_accesses[path])
paul@0 856
                accesses.sort()
paul@0 857
                print >>f, path, ", ".join(accesses)
paul@0 858
paul@0 859
            print >>f
paul@0 860
            print >>f, "constant accesses:"
paul@0 861
            paths = self.const_accesses.keys()
paul@0 862
            paths.sort()
paul@0 863
            for path in paths:
paul@0 864
                accesses = self.const_accesses[path].items()
paul@0 865
                accesses.sort()
paul@0 866
                for (original_name, attrnames), (objpath, ref, remaining_attrnames) in accesses:
paul@0 867
                    print >>f, path, original_name, attrnames, objpath, ref, remaining_attrnames or "{}"
paul@0 868
paul@0 869
            print >>f
paul@0 870
            print >>f, "attribute access usage:"
paul@0 871
            paths = self.attr_accessors.keys()
paul@0 872
            paths.sort()
paul@0 873
            for path in paths:
paul@0 874
                all_accesses = self.attr_accessors[path].items()
paul@0 875
                all_accesses.sort()
paul@0 876
                for (name, attrname), accesses in all_accesses:
paul@0 877
                    for positions in accesses:
paul@0 878
                        positions = map(str, positions)
paul@0 879
                        print >>f, path, name, attrname or "{}", ", ".join(positions)
paul@0 880
paul@0 881
            print >>f
paul@0 882
            print >>f, "attribute access modifiers:"
paul@0 883
            paths = self.attr_access_modifiers.keys()
paul@0 884
            paths.sort()
paul@0 885
            for path in paths:
paul@0 886
                all_accesses = self.attr_access_modifiers[path].items()
paul@0 887
                all_accesses.sort()
paul@0 888
                for (name, attrnames), modifiers in all_accesses:
paul@0 889
                    print >>f, path, name or "{}", attrnames or "{}", encode_modifiers(modifiers)
paul@0 890
paul@0 891
            print >>f
paul@0 892
            print >>f, "constant literals:"
paul@0 893
            paths = self.constants.keys()
paul@0 894
            paths.sort()
paul@0 895
            for path in paths:
paul@397 896
                constants = []
paul@406 897
                for (value, value_type, encoding), n in self.constants[path].items():
paul@406 898
                    constants.append((n, value_type, encoding, value))
paul@0 899
                constants.sort()
paul@406 900
                for n, value_type, encoding, value in constants:
paul@406 901
                    print >>f, path, value_type, encoding or "{}", repr(value)
paul@0 902
paul@0 903
            print >>f
paul@0 904
            print >>f, "constant values:"
paul@0 905
            names = self.constant_values.keys()
paul@0 906
            names.sort()
paul@0 907
            for name in names:
paul@406 908
                value, value_type, encoding = self.constant_values[name]
paul@406 909
                print >>f, name, value_type, encoding or "{}", repr(value)
paul@0 910
paul@479 911
            print >>f
paul@479 912
            print >>f, "exception namespaces:"
paul@479 913
            paths = list(self.exception_namespaces)
paul@479 914
            paths.sort()
paul@479 915
            print >>f, ", ".join(paths)
paul@479 916
paul@0 917
        finally:
paul@0 918
            f.close()
paul@0 919
paul@0 920
    def to_lines(self, f, heading, d):
paul@0 921
paul@0 922
        "Write lines to 'f' with the given 'heading', using 'd'."
paul@0 923
paul@0 924
        print >>f
paul@0 925
        print >>f, heading
paul@0 926
        keys = d.keys()
paul@0 927
        keys.sort()
paul@0 928
        for key in keys:
paul@0 929
            attrs = list(d[key])
paul@0 930
            if attrs:
paul@0 931
                attrs.sort()
paul@0 932
                print >>f, key, ", ".join(attrs)
paul@0 933
paul@0 934
    def usage_to_cache(self, details, f, prefix):
paul@0 935
paul@0 936
        "Write the given namespace usage details to the cache."
paul@0 937
paul@0 938
        names = list(details.keys())
paul@0 939
        if names:
paul@0 940
            names.sort()
paul@0 941
            for name in names:
paul@0 942
                if details[name]:
paul@0 943
paul@0 944
                    # Produce descriptions for each version of the name.
paul@0 945
paul@0 946
                    for version in details[name]:
paul@0 947
                        all_usages = []
paul@0 948
                        for usage in version:
paul@0 949
                            all_usages.append(encode_usage(usage))
paul@0 950
paul@0 951
                        print >>f, "%s %s %s" % (prefix, name, "; ".join(all_usages))
paul@0 952
paul@0 953
# vim: tabstop=4 expandtab shiftwidth=4