Lichen

Annotated modules.py

125:16b19e447ead
2016-10-24 Paul Boddie Account for the context position in parameter lists and table entries.
paul@0 1
#!/usr/bin/env python
paul@0 2
paul@0 3
"""
paul@0 4
Module abstractions.
paul@0 5
paul@0 6
Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013,
paul@0 7
              2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk>
paul@0 8
paul@0 9
This program is free software; you can redistribute it and/or modify it under
paul@0 10
the terms of the GNU General Public License as published by the Free Software
paul@0 11
Foundation; either version 3 of the License, or (at your option) any later
paul@0 12
version.
paul@0 13
paul@0 14
This program is distributed in the hope that it will be useful, but WITHOUT
paul@0 15
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
paul@0 16
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
paul@0 17
details.
paul@0 18
paul@0 19
You should have received a copy of the GNU General Public License along with
paul@0 20
this program.  If not, see <http://www.gnu.org/licenses/>.
paul@0 21
"""
paul@0 22
paul@16 23
from common import init_item, remove_items, CommonModule
paul@88 24
from encoders import decode_modifier_term, decode_usage, encode_modifiers, encode_usage
paul@0 25
from referencing import decode_reference, Reference
paul@12 26
from results import ResolvedNameRef
paul@0 27
import sys
paul@0 28
paul@0 29
class BasicModule(CommonModule):
paul@0 30
paul@0 31
    "The basic module information."
paul@0 32
paul@0 33
    def __init__(self, name, importer):
paul@0 34
        CommonModule.__init__(self, name, importer)
paul@0 35
paul@18 36
        # Import details, primarily for cache output.
paul@18 37
paul@18 38
        self.imports = set()
paul@18 39
        self.required = set()
paul@35 40
        self.deferred = []
paul@18 41
paul@0 42
        # Global name information.
paul@0 43
paul@0 44
        self.objects = {}
paul@0 45
        self.special = {}
paul@0 46
paul@0 47
        # Class relationships.
paul@0 48
paul@0 49
        self.classes = {}
paul@0 50
paul@0 51
        # Attributes.
paul@0 52
paul@0 53
        self.class_attrs = {}
paul@0 54
        self.instance_attrs = {}
paul@0 55
        self.instance_attr_constants = {}
paul@0 56
        self.module_attrs = set()
paul@0 57
paul@40 58
        # Names used in each namespace.
paul@0 59
paul@0 60
        self.names_used = {}
paul@0 61
paul@0 62
        # Function details.
paul@0 63
paul@0 64
        self.function_parameters = {}
paul@0 65
        self.function_defaults = {}
paul@0 66
        self.function_locals = {}
paul@0 67
        self.scope_globals = {}
paul@0 68
paul@0 69
        # Invocation details.
paul@0 70
paul@0 71
        self.function_targets = {}
paul@0 72
        self.function_arguments = {}
paul@0 73
paul@0 74
        # Attribute usage at module and function levels.
paul@0 75
paul@0 76
        self.attr_usage = {}
paul@0 77
        self.name_initialisers = {}
paul@0 78
paul@0 79
        # General attribute access expressions.
paul@0 80
paul@0 81
        self.attr_accesses = {}
paul@0 82
        self.const_accesses = {}
paul@0 83
paul@0 84
        # Attribute accessor definition details.
paul@0 85
paul@0 86
        self.attr_accessors = {}
paul@0 87
paul@0 88
        # Assignment details for accesses.
paul@0 89
paul@0 90
        self.attr_access_modifiers = {}
paul@0 91
paul@13 92
        # Name resolution details.
paul@13 93
paul@13 94
        self.name_references = {} # references to globals
paul@13 95
paul@13 96
        # Initialisation-related details.
paul@13 97
paul@13 98
        self.initialised_names = {}
paul@13 99
        self.aliased_names = {}
paul@13 100
paul@0 101
    def __repr__(self):
paul@0 102
        return "BasicModule(%r, %r)" % (self.name, self.importer)
paul@0 103
paul@0 104
    # Derived information methods.
paul@0 105
paul@0 106
    def propagate(self):
paul@0 107
paul@0 108
        "Finalise and propagate module information."
paul@0 109
paul@0 110
        self.propagate_attrs()
paul@0 111
        self.propagate_name_references()
paul@0 112
        self.propagate_attr_accesses()
paul@0 113
        self.propagate_constants()
paul@0 114
paul@0 115
    def unpropagate(self):
paul@0 116
paul@0 117
        """
paul@0 118
        Retract information from the importer including information about this
paul@0 119
        module derived by the importer.
paul@0 120
        """
paul@0 121
paul@0 122
        del self.importer.all_module_attrs[self.name]
paul@0 123
paul@0 124
        for name in self.classes.keys():
paul@68 125
            del self.importer.classes[name]
paul@0 126
            del self.importer.all_class_attrs[name]
paul@0 127
            del self.importer.all_instance_attrs[name]
paul@0 128
            del self.importer.all_instance_attr_constants[name]
paul@0 129
paul@0 130
            for name, bases in self.classes.items():
paul@0 131
                for base in bases:
paul@0 132
paul@0 133
                    # Get the identity of the class from the reference.
paul@0 134
paul@0 135
                    base = base.get_origin()
paul@0 136
paul@0 137
                    try:
paul@0 138
                        self.importer.subclasses[base].remove(name)
paul@0 139
                    except (KeyError, ValueError):
paul@0 140
                        pass
paul@0 141
paul@0 142
        remove_items(self.importer.all_name_references, self.name_references)
paul@0 143
        remove_items(self.importer.all_initialised_names, self.initialised_names)
paul@0 144
        remove_items(self.importer.all_aliased_names, self.aliased_names)
paul@0 145
        remove_items(self.importer.all_attr_accesses, self.attr_accesses)
paul@0 146
        remove_items(self.importer.all_const_accesses, self.const_accesses)
paul@0 147
        remove_items(self.importer.all_attr_access_modifiers, self.attr_access_modifiers)
paul@0 148
        remove_items(self.importer.all_constants, self.constants)
paul@0 149
        remove_items(self.importer.all_constant_values, self.constant_values)
paul@0 150
paul@0 151
        # Remove this module's objects from the importer. Objects are
paul@0 152
        # automatically propagated when defined.
paul@0 153
paul@68 154
        ref = self.importer.objects.get(self.name)
paul@68 155
        if ref and ref.has_kind("<module>"):
paul@68 156
            del self.importer.objects[self.name]
paul@68 157
paul@0 158
        for name, ref in self.objects.items():
paul@20 159
            if not ref.has_kind("<module>"):
paul@20 160
                del self.importer.objects[name]
paul@0 161
paul@68 162
    def collect(self):
paul@68 163
paul@68 164
        "Collect removed objects."
paul@68 165
paul@68 166
        for name, ref in self.objects.items():
paul@68 167
            if not self.importer.objects.has_key(ref.get_origin()) and self.importer.objects.has_key(name):
paul@68 168
                del self.importer.objects[name]
paul@68 169
paul@0 170
    def propagate_attrs(self):
paul@0 171
paul@0 172
        "Derive attributes from the class and module member details."
paul@0 173
paul@0 174
        # Initialise class attribute records for all classes.
paul@0 175
paul@0 176
        for name in self.classes.keys():
paul@0 177
            self.importer.all_class_attrs[name] = self.class_attrs[name] = {}
paul@0 178
paul@0 179
        # Separate the objects into module and class attributes.
paul@0 180
paul@0 181
        for name in self.objects.keys():
paul@0 182
            if "." in name:
paul@0 183
                parent, attrname = name.rsplit(".", 1)
paul@0 184
                if self.classes.has_key(parent):
paul@0 185
                    self.class_attrs[parent][attrname] = name
paul@0 186
                elif parent == self.name:
paul@0 187
                    self.module_attrs.add(attrname)
paul@0 188
paul@0 189
        # Propagate the module attributes.
paul@0 190
paul@0 191
        self.importer.all_module_attrs[self.name] = self.module_attrs
paul@0 192
paul@0 193
    def propagate_name_references(self):
paul@0 194
paul@0 195
        "Propagate name references for the module."
paul@0 196
paul@0 197
        self.importer.all_initialised_names.update(self.initialised_names)
paul@0 198
        self.importer.all_aliased_names.update(self.aliased_names)
paul@0 199
paul@0 200
    def propagate_attr_accesses(self):
paul@0 201
paul@0 202
        "Propagate attribute accesses for the module."
paul@0 203
paul@0 204
        self.importer.all_attr_accesses.update(self.attr_accesses)
paul@0 205
        self.importer.all_const_accesses.update(self.const_accesses)
paul@0 206
        self.importer.all_attr_access_modifiers.update(self.attr_access_modifiers)
paul@0 207
paul@0 208
    def propagate_constants(self):
paul@0 209
paul@0 210
        "Propagate constant values and aliases for the module."
paul@0 211
paul@0 212
        self.importer.all_constants.update(self.constants)
paul@0 213
        self.importer.all_constant_values.update(self.constant_values)
paul@0 214
paul@0 215
        for name in self.classes.keys():
paul@0 216
            self.importer.all_instance_attrs[name] = self.instance_attrs.get(name) or {}
paul@0 217
            self.importer.all_instance_attr_constants[name] = self.instance_attr_constants.get(name) or {}
paul@0 218
paul@26 219
    def set_object(self, name, value=None):
paul@26 220
paul@26 221
        "Set an object with the given 'name' and the given 'value'."
paul@26 222
paul@35 223
        # Decode any string value, with a new reference being returned even
paul@35 224
        # given a provided reference.
paul@35 225
paul@26 226
        ref = decode_reference(value, name)
paul@35 227
        self.add_deferred(ref)
paul@35 228
        self._set_object(name, ref)
paul@35 229
paul@35 230
    def _set_object(self, name, ref):
paul@35 231
paul@35 232
        # Determine how the object properties will be defined.
paul@35 233
paul@26 234
        multiple = self.objects.has_key(name) and self.objects[name].get_kind() != ref.get_kind()
paul@26 235
        self.importer.objects[name] = self.objects[name] = multiple and ref.as_var() or ref
paul@26 236
paul@26 237
    def queue_module(self, name, required=False):
paul@26 238
paul@26 239
        """
paul@26 240
        Queue the module with the given 'name'. If 'required' is true (it is
paul@26 241
        false by default), indicate that the module is required in the final
paul@26 242
        program.
paul@26 243
        """
paul@26 244
paul@26 245
        self.importer.queue_module(name, self, required)
paul@26 246
        if required:
paul@26 247
            self.required.add(name)
paul@26 248
        self.imports.add(name)
paul@26 249
paul@26 250
class InspectionNaming:
paul@26 251
paul@26 252
    "Name operations related to inspection."
paul@26 253
paul@0 254
    # Module-relative naming.
paul@0 255
paul@0 256
    def is_global(self, name):
paul@0 257
paul@0 258
        """
paul@0 259
        Return whether 'name' is registered as a global in the current
paul@0 260
        namespace.
paul@0 261
        """
paul@0 262
paul@0 263
        path = self.get_namespace_path()
paul@0 264
        return name in self.scope_globals.get(path, [])
paul@0 265
paul@0 266
    def get_global(self, name):
paul@0 267
paul@0 268
        """
paul@0 269
        Get the global of the given 'name' from this module, returning a
paul@0 270
        reference incorporating the original definition details.
paul@0 271
        """
paul@0 272
paul@0 273
        path = self.get_global_path(name)
paul@0 274
        return self.objects.get(path)
paul@0 275
paul@0 276
    # Name definition discovery.
paul@0 277
paul@0 278
    def get_global_or_builtin(self, name):
paul@0 279
paul@0 280
        """
paul@12 281
        Return a reference for the given 'name' found in this module or in the
paul@12 282
        __builtins__.
paul@0 283
        """
paul@0 284
paul@0 285
        return self.get_global(name) or self.get_builtin(name)
paul@0 286
paul@0 287
    def get_builtin(self, name):
paul@0 288
paul@12 289
        "Return a reference to the built-in with the given 'name'."
paul@0 290
paul@18 291
        self.queue_module("__builtins__")
paul@35 292
        ref = Reference("<depends>", "__builtins__.%s" % name)
paul@35 293
        self.deferred.append(ref)
paul@35 294
        return ref
paul@0 295
paul@12 296
    def get_builtin_class(self, name):
paul@0 297
paul@12 298
        "Return a reference to the actual object providing 'name'."
paul@0 299
paul@12 300
        # NOTE: This makes assumptions about the __builtins__ structure.
paul@0 301
paul@30 302
        module_name = "__builtins__.%s" % name
paul@30 303
        if self.name != module_name:
paul@30 304
            self.queue_module(module_name, True)
paul@12 305
        return Reference("<class>", "__builtins__.%s.%s" % (name, name))
paul@0 306
paul@0 307
    def get_object(self, path):
paul@0 308
paul@0 309
        """
paul@12 310
        Get the details of an object with the given 'path'. Where the object
paul@12 311
        cannot be resolved, an unresolved reference is returned.
paul@0 312
        """
paul@0 313
paul@0 314
        if self.objects.has_key(path):
paul@0 315
            return self.objects[path]
paul@0 316
        else:
paul@35 317
            ref = Reference("<depends>", path)
paul@35 318
            self.deferred.append(ref)
paul@35 319
            return ref
paul@0 320
paul@12 321
    def import_name_from_module(self, name, module_name):
paul@12 322
paul@12 323
        "Import 'name' from the module having the given 'module_name'."
paul@12 324
paul@12 325
        if module_name != self.name:
paul@18 326
            self.queue_module(module_name)
paul@12 327
        return Reference("<depends>", "%s.%s" % (module_name, name))
paul@12 328
paul@35 329
    def add_deferred(self, ref):
paul@35 330
paul@35 331
        "Record 'ref' as a deferred reference."
paul@35 332
paul@35 333
        if ref.has_kind("<depends>"):
paul@35 334
            self.deferred.append(ref)
paul@35 335
paul@0 336
class CachedModule(BasicModule):
paul@0 337
paul@0 338
    "A cached module."
paul@0 339
paul@0 340
    def __repr__(self):
paul@0 341
        return "CachedModule(%r, %r)" % (self.name, self.importer)
paul@0 342
paul@35 343
    def set_object(self, name, value=None):
paul@35 344
paul@35 345
        "Set an object with the given 'name' and the given 'value'."
paul@35 346
paul@35 347
        # Decode any string value, with a new reference being returned even
paul@35 348
        # given a provided reference.
paul@35 349
paul@35 350
        ref = decode_reference(value, name)
paul@35 351
        self._set_object(name, ref)
paul@35 352
paul@0 353
    def to_cache(self, filename):
paul@0 354
paul@0 355
        "Not actually writing the module back to 'filename'."
paul@0 356
paul@0 357
        pass
paul@0 358
paul@0 359
    def from_cache(self, filename):
paul@0 360
paul@0 361
        """
paul@0 362
        Read a module's details from the file with the given 'filename' as
paul@0 363
        described in the to_cache method of InspectedModule.
paul@0 364
        """
paul@0 365
paul@0 366
        f = open(filename)
paul@0 367
        try:
paul@0 368
            self.filename = f.readline().rstrip()
paul@0 369
paul@0 370
            f.readline() # (empty line)
paul@0 371
paul@18 372
            self._get_imports(f)
paul@0 373
            self._get_members(f)
paul@0 374
            self._get_class_relationships(f)
paul@0 375
            self._get_instance_attrs(f)
paul@0 376
            self._get_instance_attr_constants(f)
paul@0 377
            self.from_lines(f, self.names_used)     # "names used:"
paul@0 378
            self._get_name_references(f)
paul@0 379
            self._get_initialised_names(f)
paul@0 380
            self._get_aliased_names(f)
paul@0 381
            self._get_function_parameters(f)
paul@0 382
            self._get_function_defaults(f)
paul@0 383
            self._get_function_locals(f)
paul@0 384
            self.from_lines(f, self.scope_globals)  # "scope globals:"
paul@0 385
            self._get_function_targets(f)
paul@0 386
            self._get_function_arguments(f)
paul@0 387
            self._get_attribute_usage(f)
paul@0 388
            self._get_attr_accesses(f)
paul@0 389
            self._get_const_accesses(f)
paul@0 390
            self._get_attr_accessors(f)
paul@0 391
            self._get_attr_access_modifiers(f)
paul@0 392
            self._get_constant_literals(f)
paul@0 393
            self._get_constant_values(f)
paul@0 394
paul@0 395
        finally:
paul@0 396
            f.close()
paul@0 397
paul@12 398
    def complete(self):
paul@0 399
        self.propagate()
paul@0 400
paul@18 401
    def _get_imports(self, f):
paul@18 402
        f.readline() # "imports:"
paul@18 403
        line = f.readline().strip()
paul@18 404
        self.required = line != "{}" and set(line.split(", ")) or set()
paul@18 405
        line = f.readline().strip()
paul@18 406
        self.imports = line != "{}" and set(line.split(", ")) or set()
paul@18 407
        f.readline()
paul@18 408
paul@18 409
        for name in self.required:
paul@18 410
            self.queue_module(name, True)
paul@18 411
        for name in self.imports:
paul@18 412
            self.queue_module(name)
paul@18 413
paul@0 414
    def _get_members(self, f):
paul@0 415
        f.readline() # "members:"
paul@0 416
        line = f.readline().rstrip()
paul@0 417
        while line:
paul@0 418
            name, ref = line.split(" ", 1)
paul@0 419
            self.set_object(name, ref)
paul@0 420
            line = f.readline().rstrip()
paul@0 421
paul@0 422
    def _get_class_relationships(self, f):
paul@0 423
        f.readline() # "class relationships:"
paul@0 424
        line = f.readline().rstrip()
paul@0 425
        while line:
paul@0 426
            name, value = self._get_fields(line)
paul@0 427
            values = value and value.split(", ") or []
paul@0 428
            self.importer.classes[name] = self.classes[name] = map(decode_reference, values)
paul@0 429
            self.importer.subclasses[name] = set()
paul@0 430
            line = f.readline().rstrip()
paul@0 431
paul@0 432
    def _get_instance_attrs(self, f):
paul@0 433
        f.readline() # "instance attributes:"
paul@0 434
        line = f.readline().rstrip()
paul@0 435
        while line:
paul@0 436
            name, value = self._get_fields(line)
paul@0 437
            self.importer.all_instance_attrs[name] = self.instance_attrs[name] = set(value and value.split(", ") or [])
paul@0 438
            line = f.readline().rstrip()
paul@0 439
paul@0 440
    def _get_instance_attr_constants(self, f):
paul@0 441
        f.readline() # "instance attribute constants:"
paul@0 442
        line = f.readline().rstrip()
paul@0 443
        while line:
paul@0 444
            name, attrname, ref = self._get_fields(line, 3)
paul@0 445
            init_item(self.instance_attr_constants, name, dict)
paul@0 446
            self.instance_attr_constants[name][attrname] = decode_reference(ref)
paul@0 447
            line = f.readline().rstrip()
paul@0 448
paul@0 449
    def _get_name_references(self, f):
paul@0 450
        f.readline() # "name references:"
paul@0 451
        line = f.readline().rstrip()
paul@0 452
        while line:
paul@37 453
            name, ref = self._get_fields(line)
paul@37 454
            self.importer.all_name_references[name] = self.name_references[name] = decode_reference(ref)
paul@0 455
            line = f.readline().rstrip()
paul@0 456
paul@0 457
    def _get_initialised_names(self, f):
paul@0 458
        f.readline() # "initialised names:"
paul@0 459
        line = f.readline().rstrip()
paul@0 460
        while line:
paul@0 461
            name, version, value = self._get_fields(line, 3)
paul@0 462
            init_item(self.initialised_names, name, dict)
paul@0 463
            self.initialised_names[name][int(version)] = decode_reference(value)
paul@0 464
            line = f.readline().rstrip()
paul@0 465
paul@0 466
    def _get_aliased_names(self, f):
paul@0 467
        f.readline() # "aliased names:"
paul@0 468
        line = f.readline().rstrip()
paul@0 469
        while line:
paul@0 470
            name, version, original_name, attrnames, number = self._get_fields(line, 5)
paul@0 471
            init_item(self.aliased_names, name, dict)
paul@0 472
            if number == "{}": number = None
paul@0 473
            else: number = int(number)
paul@0 474
            self.aliased_names[name][int(version)] = (original_name, attrnames != "{}" and attrnames or None, number)
paul@0 475
            line = f.readline().rstrip()
paul@0 476
paul@0 477
    def _get_function_parameters(self, f):
paul@0 478
        f.readline() # "function parameters:"
paul@0 479
        line = f.readline().rstrip()
paul@0 480
        while line:
paul@0 481
            function, names = self._get_fields(line)
paul@0 482
            self.importer.function_parameters[function] = \
paul@49 483
                self.function_parameters[function] = names != "{}" and names.split(", ") or []
paul@0 484
            line = f.readline().rstrip()
paul@0 485
paul@0 486
    def _get_function_defaults(self, f):
paul@0 487
        f.readline() # "function default parameters:"
paul@0 488
        line = f.readline().rstrip()
paul@0 489
        while line:
paul@0 490
            function, defaults = self._get_fields(line)
paul@0 491
            self.importer.function_defaults[function] = \
paul@0 492
                self.function_defaults[function] = l = []
paul@0 493
            if defaults != "{}":
paul@0 494
                for value in defaults.split(", "):
paul@0 495
                    name, default = value.split("=")
paul@0 496
                    default = decode_reference(default)
paul@0 497
                    l.append((name, default))
paul@0 498
            line = f.readline().rstrip()
paul@0 499
paul@0 500
    def _get_function_locals(self, f):
paul@0 501
        f.readline() # "function locals:"
paul@0 502
        line = f.readline().rstrip()
paul@0 503
        while line:
paul@0 504
            function, name, value = self._get_fields(line, 3)
paul@0 505
            init_item(self.function_locals, function, dict)
paul@115 506
            init_item(self.importer.function_locals, function, dict)
paul@0 507
            if name != "{}":
paul@115 508
                self.importer.function_locals[function][name] = \
paul@109 509
                    self.function_locals[function][name] = decode_reference(value)
paul@0 510
            line = f.readline().rstrip()
paul@0 511
paul@0 512
    def _get_function_targets(self, f):
paul@0 513
        f.readline() # "function targets:"
paul@0 514
        line = f.readline().rstrip()
paul@0 515
        while line:
paul@0 516
            function, n = self._get_fields(line)
paul@0 517
            self.importer.function_targets[function] = \
paul@0 518
                self.function_targets[function] = int(n)
paul@0 519
            line = f.readline().rstrip()
paul@0 520
paul@0 521
    def _get_function_arguments(self, f):
paul@0 522
        f.readline() # "function arguments:"
paul@0 523
        line = f.readline().rstrip()
paul@0 524
        while line:
paul@0 525
            function, n = self._get_fields(line)
paul@0 526
            self.importer.function_arguments[function] = \
paul@0 527
                self.function_arguments[function] = int(n)
paul@0 528
            line = f.readline().rstrip()
paul@0 529
paul@0 530
    def _get_attribute_usage(self, f):
paul@0 531
        f.readline() # "attribute usage:"
paul@0 532
        line = f.readline().rstrip()
paul@0 533
        while line:
paul@0 534
            unit, value = self._get_fields(line)
paul@0 535
            init_item(self.attr_usage, unit, dict)
paul@0 536
            self.usage_from_cache(value, self.attr_usage[unit])
paul@0 537
            line = f.readline().rstrip()
paul@0 538
paul@0 539
    def _get_attr_accesses(self, f):
paul@0 540
        f.readline() # "attribute accesses:"
paul@0 541
        line = f.readline().rstrip()
paul@0 542
        while line:
paul@0 543
            name, value = self._get_fields(line)
paul@0 544
            self.attr_accesses[name] = set(value.split(", "))
paul@0 545
            line = f.readline().rstrip()
paul@0 546
paul@0 547
    def _get_const_accesses(self, f):
paul@0 548
        f.readline() # "constant accesses:"
paul@0 549
        line = f.readline().rstrip()
paul@0 550
        while line:
paul@0 551
            name, original_name, attrnames, objpath, ref, remaining = self._get_fields(line, 6)
paul@0 552
            if attrnames == "{}": attrnames = None
paul@0 553
            init_item(self.const_accesses, name, dict)
paul@0 554
            self.const_accesses[name][(original_name, attrnames)] = (objpath, decode_reference(ref), remaining != "{}" and remaining or "")
paul@0 555
            line = f.readline().rstrip()
paul@0 556
paul@0 557
    def _get_attr_accessors(self, f):
paul@0 558
        f.readline() # "attribute access usage:"
paul@0 559
        line = f.readline().rstrip()
paul@0 560
        while line:
paul@0 561
            objpath, name, attrname, value = self._get_fields(line, 4)
paul@0 562
            if attrname == "{}": attrname = None
paul@0 563
            access = name, attrname
paul@0 564
            init_item(self.attr_accessors, objpath, dict)
paul@0 565
            init_item(self.attr_accessors[objpath], access, list)
paul@0 566
            positions = map(int, value.split(", "))
paul@0 567
            self.attr_accessors[objpath][access].append(positions)
paul@0 568
            line = f.readline().rstrip()
paul@0 569
paul@0 570
    def _get_attr_access_modifiers(self, f):
paul@0 571
        f.readline() # "attribute access modifiers:"
paul@0 572
        line = f.readline().rstrip()
paul@0 573
        while line:
paul@0 574
            objpath, name, attrnames, value = self._get_fields(line, 4)
paul@0 575
            if name == "{}": name = None
paul@0 576
            if attrnames == "{}": attrnames = None
paul@0 577
            access = name, attrnames
paul@0 578
            init_item(self.attr_access_modifiers, objpath, dict)
paul@0 579
            init_item(self.attr_access_modifiers[objpath], access, list)
paul@0 580
            modifiers = [decode_modifier_term(s) for s in value]
paul@0 581
            self.attr_access_modifiers[objpath][access] = modifiers
paul@0 582
            line = f.readline().rstrip()
paul@0 583
paul@0 584
    def _get_constant_literals(self, f):
paul@0 585
        f.readline() # "constant literals:"
paul@0 586
        line = f.readline().rstrip()
paul@0 587
        last_path = None
paul@0 588
        n = None
paul@0 589
        while line:
paul@0 590
            path, constant = self._get_fields(line)
paul@0 591
            if path != last_path:
paul@0 592
                n = 0
paul@0 593
                last_path = path
paul@0 594
            else:
paul@0 595
                n += 1
paul@0 596
            init_item(self.constants, path, dict)
paul@0 597
            self.constants[path][eval(constant)] = n
paul@0 598
            line = f.readline().rstrip()
paul@0 599
paul@0 600
    def _get_constant_values(self, f):
paul@0 601
        f.readline() # "constant values:"
paul@0 602
        line = f.readline().rstrip()
paul@0 603
        while line:
paul@0 604
            name, value_type, value = self._get_fields(line, 3)
paul@0 605
            self.constant_values[name] = eval(value), value_type
paul@0 606
            line = f.readline().rstrip()
paul@0 607
paul@0 608
    # Generic parsing methods.
paul@0 609
paul@0 610
    def from_lines(self, f, d):
paul@0 611
paul@0 612
        "Read lines from 'f', populating 'd'."
paul@0 613
paul@0 614
        f.readline() # section heading
paul@0 615
        line = f.readline().rstrip()
paul@0 616
        while line:
paul@0 617
            name, value = self._get_fields(line)
paul@0 618
            d[name] = set(value and value.split(", ") or [])
paul@0 619
            line = f.readline().rstrip()
paul@0 620
paul@0 621
    def usage_from_cache(self, value, mapping):
paul@0 622
paul@0 623
        """
paul@0 624
        Interpret the given 'value' containing name and usage information,
paul@0 625
        storing the information in the given 'mapping'.
paul@0 626
        """
paul@0 627
paul@0 628
        local, usage = self._get_fields(value)
paul@0 629
        init_item(mapping, local, list)
paul@0 630
        self._usage_from_cache(mapping[local], usage)
paul@0 631
paul@0 632
    def _usage_from_cache(self, d, usage):
paul@0 633
paul@0 634
        # Interpret descriptions of each version of the name.
paul@0 635
paul@0 636
        all_usages = set()
paul@0 637
        for attrnames in usage.split("; "):
paul@0 638
            if attrnames == "{}":
paul@0 639
                all_attrnames = ()
paul@0 640
            else:
paul@88 641
                all_attrnames = decode_usage(attrnames)
paul@88 642
            all_usages.add(all_attrnames)
paul@0 643
paul@0 644
        d.append(all_usages)
paul@0 645
paul@0 646
    def _get_fields(self, s, n=2):
paul@0 647
        result = s.split(" ", n-1)
paul@0 648
        if len(result) == n:
paul@0 649
            return result
paul@0 650
        else:
paul@0 651
            return tuple(result) + tuple([""] * (n - len(result)))
paul@0 652
paul@0 653
class CacheWritingModule:
paul@0 654
paul@0 655
    """
paul@0 656
    A mix-in providing cache-writing support, to be combined with BasicModule.
paul@0 657
    """
paul@0 658
paul@0 659
    def to_cache(self, filename):
paul@0 660
paul@0 661
        """
paul@0 662
        Write a cached representation of the inspected module with the following
paul@0 663
        format to the file having the given 'filename':
paul@0 664
paul@0 665
        filename
paul@0 666
        (empty line)
paul@18 667
        "imports:"
paul@18 668
        required module names
paul@18 669
        possibly required module names
paul@0 670
        "members:"
paul@0 671
        zero or more: qualified name " " reference
paul@0 672
        (empty line)
paul@0 673
        "class relationships:"
paul@0 674
        zero or more: qualified class name " " base class references
paul@0 675
        (empty line)
paul@0 676
        "instance attributes:"
paul@0 677
        zero or more: qualified class name " " instance attribute names
paul@0 678
        (empty line)
paul@0 679
        "instance attribute constants:"
paul@0 680
        zero or more: qualified class name " " attribute name " " reference
paul@0 681
        (empty line)
paul@0 682
        "names used:"
paul@0 683
        zero or more: qualified class/function/module name " " names
paul@0 684
        (empty line)
paul@0 685
        "names missing:"
paul@0 686
        zero or more: qualified class/function/module name " " names
paul@0 687
        (empty line)
paul@0 688
        "name references:"
paul@0 689
        zero or more: qualified name " " reference
paul@0 690
        (empty line)
paul@0 691
        "initialised names:"
paul@0 692
        zero or more: qualified name " " definition version " " reference
paul@0 693
        (empty line)
paul@0 694
        "aliased names:"
paul@0 695
        zero or more: qualified name " " definition version " " original name " " attribute names " " access number
paul@0 696
        (empty line)
paul@0 697
        "function parameters:"
paul@0 698
        zero or more: qualified function name " " parameter names
paul@0 699
        (empty line)
paul@0 700
        "function default parameters:"
paul@0 701
        zero or more: qualified function name " " parameter names with defaults
paul@0 702
        (empty line)
paul@0 703
        "function locals:"
paul@0 704
        zero or more: qualified function name " " local variable name " " reference
paul@0 705
        (empty line)
paul@0 706
        "scope globals:"
paul@0 707
        zero or more: qualified function name " " global variable names
paul@0 708
        (empty line)
paul@0 709
        "function targets:"
paul@0 710
        zero or more: qualified function name " " maximum number of targets allocated
paul@0 711
        (empty line)
paul@0 712
        "function arguments:"
paul@0 713
        zero or more: qualified function name " " maximum number of arguments allocated
paul@0 714
        (empty line)
paul@0 715
        "attribute usage:"
paul@0 716
        zero or more: qualified scope name " " local/global/qualified variable name " " usages
paul@0 717
        (empty line)
paul@0 718
        "attribute accesses:"
paul@0 719
        zero or more: qualified scope name " " attribute-chains
paul@0 720
        (empty line)
paul@0 721
        "constant accesses:"
paul@0 722
        zero or more: qualified function name " " attribute-chain " " reference " " remaining attribute-chain
paul@0 723
        (empty line)
paul@0 724
        "attribute access usage:"
paul@0 725
        zero or more: qualified function name " " local/global variable name " " attribute name " " definition versions
paul@0 726
        (empty line)
paul@0 727
        "attribute access modifiers:"
paul@0 728
        zero or more: qualified function name " " local/global variable name " " attribute name " " access modifiers
paul@0 729
        "constant literals:"
paul@0 730
        zero or more: qualified scope name " " constant literal
paul@0 731
        "constant values:"
paul@0 732
        zero or more: qualified name " " value type " " constant literal
paul@0 733
paul@0 734
        All collections of names are separated by ", " characters.
paul@0 735
paul@0 736
        References can be "<var>", a module name, or one of "<class>" or
paul@0 737
        "<function>" followed optionally by a ":" character and a qualified
paul@0 738
        name.
paul@0 739
paul@0 740
        Parameter names with defaults are separated by ", " characters, with
paul@0 741
        each name followed by "=" and then followed by a reference. If "{}" is
paul@0 742
        indicated, no defaults are defined for the function. Similarly, function
paul@0 743
        locals may be indicated as "{}" meaning that there are no locals.
paul@0 744
paul@0 745
        All usages (attribute usage sets) are separated by "; " characters, with
paul@0 746
        the special string "{}" representing an empty set.
paul@0 747
paul@0 748
        Each usage is a collection of names separated by ", " characters, with
paul@106 749
        invoked attribute names suffixed with a "!" character.
paul@0 750
paul@0 751
        Definition versions are separated by ", " characters and indicate the
paul@0 752
        name definition version associated with the access.
paul@0 753
paul@0 754
        Access modifiers are separated by ", " characters and indicate features
paul@0 755
        of each access, with multiple accesses described on a single line.
paul@0 756
        """
paul@0 757
paul@0 758
        f = open(filename, "w")
paul@0 759
        try:
paul@0 760
            print >>f, self.filename
paul@0 761
paul@0 762
            print >>f
paul@18 763
            print >>f, "imports:"
paul@18 764
            required = list(self.required)
paul@18 765
            required.sort()
paul@18 766
            print >>f, required and ", ".join(required) or "{}"
paul@18 767
            imports = list(self.imports)
paul@18 768
            imports.sort()
paul@18 769
            print >>f, imports and ", ".join(imports) or "{}"
paul@18 770
paul@18 771
            print >>f
paul@0 772
            print >>f, "members:"
paul@0 773
            objects = self.objects.keys()
paul@0 774
            objects.sort()
paul@0 775
            for name in objects:
paul@0 776
                print >>f, name, self.objects[name]
paul@0 777
paul@0 778
            print >>f
paul@0 779
            print >>f, "class relationships:"
paul@0 780
            classes = self.classes.keys()
paul@0 781
            classes.sort()
paul@0 782
            for class_ in classes:
paul@0 783
                bases = self.classes[class_]
paul@0 784
                if bases:
paul@0 785
                    print >>f, class_, ", ".join(map(str, bases))
paul@0 786
                else:
paul@0 787
                    print >>f, class_
paul@0 788
paul@0 789
            self.to_lines(f, "instance attributes:", self.instance_attrs)
paul@0 790
paul@0 791
            print >>f
paul@0 792
            print >>f, "instance attribute constants:"
paul@0 793
            classes = self.instance_attr_constants.items()
paul@0 794
            classes.sort()
paul@0 795
            for name, attrs in classes:
paul@0 796
                attrs = attrs.items()
paul@0 797
                attrs.sort()
paul@0 798
                for attrname, ref in attrs:
paul@0 799
                    print >>f, name, attrname, ref
paul@0 800
paul@0 801
            self.to_lines(f, "names used:", self.names_used)
paul@0 802
paul@0 803
            print >>f
paul@0 804
            print >>f, "name references:"
paul@0 805
            refs = self.name_references.items()
paul@0 806
            refs.sort()
paul@0 807
            for name, ref in refs:
paul@0 808
                print >>f, name, ref
paul@0 809
paul@0 810
            print >>f
paul@0 811
            print >>f, "initialised names:"
paul@0 812
            assignments = self.initialised_names.items()
paul@0 813
            assignments.sort()
paul@0 814
            for name, refs in assignments:
paul@0 815
                versions = refs.items()
paul@0 816
                versions.sort()
paul@0 817
                for version, ref in versions:
paul@0 818
                    print >>f, name, version, ref
paul@0 819
paul@0 820
            print >>f
paul@0 821
            print >>f, "aliased names:"
paul@0 822
            assignments = self.aliased_names.items()
paul@0 823
            assignments.sort()
paul@0 824
            for name, aliases in assignments:
paul@0 825
                versions = aliases.items()
paul@0 826
                versions.sort()
paul@0 827
                for version, alias in versions:
paul@0 828
                    original_name, attrnames, number = alias
paul@0 829
                    print >>f, name, version, original_name, attrnames or "{}", number is None and "{}" or number
paul@0 830
paul@0 831
            print >>f
paul@0 832
            print >>f, "function parameters:"
paul@0 833
            functions = self.function_parameters.keys()
paul@0 834
            functions.sort()
paul@0 835
            for function in functions:
paul@49 836
                parameters = self.function_parameters[function]
paul@49 837
                if parameters:
paul@49 838
                    print >>f, function, ", ".join(parameters)
paul@49 839
                else:
paul@49 840
                    print >>f, function, "{}"
paul@0 841
paul@0 842
            print >>f
paul@0 843
            print >>f, "function default parameters:"
paul@0 844
            functions = self.function_defaults.keys()
paul@0 845
            functions.sort()
paul@0 846
            for function in functions:
paul@0 847
                parameters = self.function_defaults[function]
paul@0 848
                if parameters:
paul@0 849
                    print >>f, function, ", ".join([("%s=%s" % (name, default)) for (name, default) in parameters])
paul@0 850
                else:
paul@0 851
                    print >>f, function, "{}"
paul@0 852
paul@0 853
            print >>f
paul@0 854
            print >>f, "function locals:"
paul@0 855
            functions = self.function_locals.keys()
paul@0 856
            functions.sort()
paul@0 857
            for function in functions:
paul@0 858
                names = self.function_locals[function].items()
paul@0 859
                if names:
paul@0 860
                    names.sort()
paul@0 861
                    for name, value in names:
paul@0 862
                        print >>f, function, name, value
paul@0 863
                else:
paul@0 864
                    print >>f, function, "{}"
paul@0 865
paul@0 866
            self.to_lines(f, "scope globals:", self.scope_globals)
paul@0 867
paul@0 868
            print >>f
paul@0 869
            print >>f, "function targets:"
paul@0 870
            functions = self.function_targets.keys()
paul@0 871
            functions.sort()
paul@0 872
            for function in functions:
paul@0 873
                print >>f, function, self.function_targets[function]
paul@0 874
paul@0 875
            print >>f
paul@0 876
            print >>f, "function arguments:"
paul@0 877
            functions = self.function_arguments.keys()
paul@0 878
            functions.sort()
paul@0 879
            for function in functions:
paul@0 880
                print >>f, function, self.function_arguments[function]
paul@0 881
paul@0 882
            print >>f
paul@0 883
            print >>f, "attribute usage:"
paul@0 884
            units = self.attr_usage.keys()
paul@0 885
            units.sort()
paul@0 886
            for unit in units:
paul@0 887
                d = self.attr_usage[unit]
paul@0 888
                self.usage_to_cache(d, f, unit)
paul@0 889
paul@0 890
            print >>f
paul@0 891
            print >>f, "attribute accesses:"
paul@0 892
            paths = self.attr_accesses.keys()
paul@0 893
            paths.sort()
paul@0 894
            for path in paths:
paul@0 895
                accesses = list(self.attr_accesses[path])
paul@0 896
                accesses.sort()
paul@0 897
                print >>f, path, ", ".join(accesses)
paul@0 898
paul@0 899
            print >>f
paul@0 900
            print >>f, "constant accesses:"
paul@0 901
            paths = self.const_accesses.keys()
paul@0 902
            paths.sort()
paul@0 903
            for path in paths:
paul@0 904
                accesses = self.const_accesses[path].items()
paul@0 905
                accesses.sort()
paul@0 906
                for (original_name, attrnames), (objpath, ref, remaining_attrnames) in accesses:
paul@0 907
                    print >>f, path, original_name, attrnames, objpath, ref, remaining_attrnames or "{}"
paul@0 908
paul@0 909
            print >>f
paul@0 910
            print >>f, "attribute access usage:"
paul@0 911
            paths = self.attr_accessors.keys()
paul@0 912
            paths.sort()
paul@0 913
            for path in paths:
paul@0 914
                all_accesses = self.attr_accessors[path].items()
paul@0 915
                all_accesses.sort()
paul@0 916
                for (name, attrname), accesses in all_accesses:
paul@0 917
                    for positions in accesses:
paul@0 918
                        positions = map(str, positions)
paul@0 919
                        print >>f, path, name, attrname or "{}", ", ".join(positions)
paul@0 920
paul@0 921
            print >>f
paul@0 922
            print >>f, "attribute access modifiers:"
paul@0 923
            paths = self.attr_access_modifiers.keys()
paul@0 924
            paths.sort()
paul@0 925
            for path in paths:
paul@0 926
                all_accesses = self.attr_access_modifiers[path].items()
paul@0 927
                all_accesses.sort()
paul@0 928
                for (name, attrnames), modifiers in all_accesses:
paul@0 929
                    print >>f, path, name or "{}", attrnames or "{}", encode_modifiers(modifiers)
paul@0 930
paul@0 931
            print >>f
paul@0 932
            print >>f, "constant literals:"
paul@0 933
            paths = self.constants.keys()
paul@0 934
            paths.sort()
paul@0 935
            for path in paths:
paul@0 936
                constants = [(v, k) for (k, v) in self.constants[path].items()]
paul@0 937
                constants.sort()
paul@0 938
                for n, constant in constants:
paul@0 939
                    print >>f, path, repr(constant)
paul@0 940
paul@0 941
            print >>f
paul@0 942
            print >>f, "constant values:"
paul@0 943
            names = self.constant_values.keys()
paul@0 944
            names.sort()
paul@0 945
            for name in names:
paul@0 946
                value, value_type = self.constant_values[name]
paul@0 947
                print >>f, name, value_type, repr(value)
paul@0 948
paul@0 949
        finally:
paul@0 950
            f.close()
paul@0 951
paul@0 952
    def to_lines(self, f, heading, d):
paul@0 953
paul@0 954
        "Write lines to 'f' with the given 'heading', using 'd'."
paul@0 955
paul@0 956
        print >>f
paul@0 957
        print >>f, heading
paul@0 958
        keys = d.keys()
paul@0 959
        keys.sort()
paul@0 960
        for key in keys:
paul@0 961
            attrs = list(d[key])
paul@0 962
            if attrs:
paul@0 963
                attrs.sort()
paul@0 964
                print >>f, key, ", ".join(attrs)
paul@0 965
paul@0 966
    def usage_to_cache(self, details, f, prefix):
paul@0 967
paul@0 968
        "Write the given namespace usage details to the cache."
paul@0 969
paul@0 970
        names = list(details.keys())
paul@0 971
        if names:
paul@0 972
            names.sort()
paul@0 973
            for name in names:
paul@0 974
                if details[name]:
paul@0 975
paul@0 976
                    # Produce descriptions for each version of the name.
paul@0 977
paul@0 978
                    for version in details[name]:
paul@0 979
                        all_usages = []
paul@0 980
                        for usage in version:
paul@0 981
                            all_usages.append(encode_usage(usage))
paul@0 982
paul@0 983
                        print >>f, "%s %s %s" % (prefix, name, "; ".join(all_usages))
paul@0 984
paul@0 985
# vim: tabstop=4 expandtab shiftwidth=4