| # Content (c) 2002, 2004, 2006, 2007 David Turner <david@freetype.org> |
| # |
| # This file contains routines used to parse the content of documentation |
| # comment blocks and build more structured objects out of them. |
| # |
| |
| from sources import * |
| from utils import * |
| import string, re |
| |
| |
| # this regular expression is used to detect code sequences. these |
| # are simply code fragments embedded in '{' and '}' like in: |
| # |
| # { |
| # x = y + z; |
| # if ( zookoo == 2 ) |
| # { |
| # foobar(); |
| # } |
| # } |
| # |
| # note that indentation of the starting and ending accolades must be |
| # exactly the same. the code sequence can contain accolades at greater |
| # indentation |
| # |
| re_code_start = re.compile( r"(\s*){\s*$" ) |
| re_code_end = re.compile( r"(\s*)}\s*$" ) |
| |
| |
| # this regular expression is used to isolate identifiers from |
| # other text |
| # |
| re_identifier = re.compile( r'(\w*)' ) |
| |
| |
| ############################################################################# |
| # |
| # The DocCode class is used to store source code lines. |
| # |
| # 'self.lines' contains a set of source code lines that will be dumped as |
| # HTML in a <PRE> tag. |
| # |
| # The object is filled line by line by the parser; it strips the leading |
| # "margin" space from each input line before storing it in 'self.lines'. |
| # |
| class DocCode: |
| |
| def __init__( self, margin, lines ): |
| self.lines = [] |
| self.words = None |
| |
| # remove margin spaces |
| for l in lines: |
| if string.strip( l[:margin] ) == "": |
| l = l[margin:] |
| self.lines.append( l ) |
| |
| def dump( self, prefix = "", width=60 ): |
| lines = self.dump_lines( 0, width ) |
| for l in lines: |
| print prefix + l |
| |
| def dump_lines( self, margin=0, width=60 ): |
| result = [] |
| for l in self.lines: |
| result.append( " "*margin + l ) |
| return result |
| |
| |
| |
| ############################################################################# |
| # |
| # The DocPara class is used to store "normal" text paragraph. |
| # |
| # 'self.words' contains the list of words that make up the paragraph |
| # |
| class DocPara: |
| |
| def __init__( self, lines ): |
| self.lines = None |
| self.words = [] |
| for l in lines: |
| l = string.strip(l) |
| self.words.extend( string.split( l ) ) |
| |
| def dump( self, prefix = "", width = 60 ): |
| lines = self.dump_lines( 0, width ) |
| for l in lines: |
| print prefix + l |
| |
| def dump_lines( self, margin=0, width = 60 ): |
| cur = "" # current line |
| col = 0 # current width |
| result = [] |
| |
| for word in self.words: |
| ln = len(word) |
| if col > 0: |
| ln = ln+1 |
| |
| if col + ln > width: |
| result.append( " "*margin + cur ) |
| cur = word |
| col = len(word) |
| else: |
| if col > 0: |
| cur = cur + " " |
| cur = cur + word |
| col = col + ln |
| |
| if col > 0: |
| result.append( " "*margin + cur ) |
| |
| return result |
| |
| |
| |
| |
| ############################################################################# |
| # |
| # The DocField class is used to store a list containing either DocPara or |
| # DocCode objects. Each DocField also has an optional "name" which is used |
| # when the object corresponds to a field or value definition |
| # |
| class DocField: |
| |
| def __init__( self, name, lines ): |
| |
| self.name = name # can be None for normal paragraphs/sources |
| self.items = [] # list of items |
| |
| mode_none = 0 # start parsing mode |
| mode_code = 1 # parsing code sequences |
| mode_para = 3 # parsing normal paragraph |
| |
| margin = -1 # current code sequence indentation |
| cur_lines = [] |
| |
| # now analyze the markup lines to see if they contain paragraphs, |
| # code sequences or fields definitions |
| # |
| start = 0 |
| mode = mode_none |
| for l in lines: |
| |
| # are we parsing a code sequence ? |
| if mode == mode_code: |
| |
| m = re_code_end.match( l ) |
| if m and len(m.group(1)) <= margin: |
| # that's it, we finised the code sequence |
| code = DocCode( 0, cur_lines ) |
| self.items.append( code ) |
| margin = -1 |
| cur_lines = [] |
| mode = mode_none |
| else: |
| # nope, continue the code sequence |
| cur_lines.append( l[margin:] ) |
| else: |
| # start of code sequence ? |
| m = re_code_start.match( l ) |
| if m: |
| # save current lines |
| if cur_lines: |
| para = DocPara( cur_lines ) |
| self.items.append( para ) |
| cur_lines = [] |
| |
| # switch to code extraction mode |
| margin = len(m.group(1)) |
| mode = mode_code |
| |
| else: |
| if not string.split( l ) and cur_lines: |
| # if the line is empty, we end the current paragraph, |
| # if any |
| para = DocPara( cur_lines ) |
| self.items.append( para ) |
| cur_lines = [] |
| else: |
| # otherwise, simply add the line to the current |
| # paragraph |
| cur_lines.append( l ) |
| |
| if mode == mode_code: |
| # unexpected end of code sequence |
| code = DocCode( margin, cur_lines ) |
| self.items.append( code ) |
| |
| elif cur_lines: |
| para = DocPara( cur_lines ) |
| self.items.append( para ) |
| |
| def dump( self, prefix = "" ): |
| if self.field: |
| print prefix + self.field + " ::" |
| prefix = prefix + "----" |
| |
| first = 1 |
| for p in self.items: |
| if not first: |
| print "" |
| p.dump( prefix ) |
| first = 0 |
| |
| def dump_lines( self, margin=0, width=60 ): |
| result = [] |
| nl = None |
| for p in self.items: |
| if nl: |
| result.append( "" ) |
| |
| result.extend( p.dump_lines( margin, width ) ) |
| nl = 1 |
| |
| return result |
| |
| # this regular expression is used to detect field definitions |
| # |
| re_field = re.compile( r"\s*(\w*|\w(\w|\.)*\w)\s*::" ) |
| |
| |
| |
| class DocMarkup: |
| |
| def __init__( self, tag, lines ): |
| self.tag = string.lower(tag) |
| self.fields = [] |
| |
| cur_lines = [] |
| field = None |
| mode = 0 |
| |
| for l in lines: |
| m = re_field.match( l ) |
| if m: |
| # we detected the start of a new field definition |
| |
| # first, save the current one |
| if cur_lines: |
| f = DocField( field, cur_lines ) |
| self.fields.append( f ) |
| cur_lines = [] |
| field = None |
| |
| field = m.group(1) # record field name |
| ln = len(m.group(0)) |
| l = " "*ln + l[ln:] |
| cur_lines = [ l ] |
| else: |
| cur_lines.append( l ) |
| |
| if field or cur_lines: |
| f = DocField( field, cur_lines ) |
| self.fields.append( f ) |
| |
| def get_name( self ): |
| try: |
| return self.fields[0].items[0].words[0] |
| |
| except: |
| return None |
| |
| def get_start( self ): |
| try: |
| result = "" |
| for word in self.fields[0].items[0].words: |
| result = result + " " + word |
| return result[1:] |
| |
| except: |
| return "ERROR" |
| |
| def dump( self, margin ): |
| print " "*margin + "<" + self.tag + ">" |
| for f in self.fields: |
| f.dump( " " ) |
| print " "*margin + "</" + self.tag + ">" |
| |
| |
| |
| |
| class DocChapter: |
| |
| def __init__( self, block ): |
| self.block = block |
| self.sections = [] |
| if block: |
| self.name = block.name |
| self.title = block.get_markup_words( "title" ) |
| self.order = block.get_markup_words( "sections" ) |
| else: |
| self.name = "Other" |
| self.title = string.split( "Miscellaneous" ) |
| self.order = [] |
| |
| |
| |
| class DocSection: |
| |
| def __init__( self, name = "Other" ): |
| self.name = name |
| self.blocks = {} |
| self.block_names = [] # ordered block names in section |
| self.defs = [] |
| self.abstract = "" |
| self.description = "" |
| self.order = [] |
| self.title = "ERROR" |
| self.chapter = None |
| |
| def add_def( self, block ): |
| self.defs.append( block ) |
| |
| def add_block( self, block ): |
| self.block_names.append( block.name ) |
| self.blocks[ block.name ] = block |
| |
| def process( self ): |
| # lookup one block that contains a valid section description |
| for block in self.defs: |
| title = block.get_markup_text( "title" ) |
| if title: |
| self.title = title |
| self.abstract = block.get_markup_words( "abstract" ) |
| self.description = block.get_markup_items( "description" ) |
| self.order = block.get_markup_words( "order" ) |
| return |
| |
| def reorder( self ): |
| |
| self.block_names = sort_order_list( self.block_names, self.order ) |
| |
| |
| class ContentProcessor: |
| |
| def __init__( self ): |
| """initialize a block content processor""" |
| self.reset() |
| |
| self.sections = {} # dictionary of documentation sections |
| self.section = None # current documentation section |
| |
| self.chapters = [] # list of chapters |
| |
| def set_section( self, section_name ): |
| """set current section during parsing""" |
| if not self.sections.has_key( section_name ): |
| section = DocSection( section_name ) |
| self.sections[ section_name ] = section |
| self.section = section |
| else: |
| self.section = self.sections[ section_name ] |
| |
| def add_chapter( self, block ): |
| chapter = DocChapter( block ) |
| self.chapters.append( chapter ) |
| |
| |
| def reset( self ): |
| """reset the content processor for a new block""" |
| self.markups = [] |
| self.markup = None |
| self.markup_lines = [] |
| |
| def add_markup( self ): |
| """add a new markup section""" |
| if self.markup and self.markup_lines: |
| |
| # get rid of last line of markup if it's empty |
| marks = self.markup_lines |
| if len(marks) > 0 and not string.strip(marks[-1]): |
| self.markup_lines = marks[:-1] |
| |
| m = DocMarkup( self.markup, self.markup_lines ) |
| |
| self.markups.append( m ) |
| |
| self.markup = None |
| self.markup_lines = [] |
| |
| |
| def process_content( self, content ): |
| """process a block content and return a list of DocMarkup objects |
| corresponding to it""" |
| markup = None |
| markup_lines = [] |
| first = 1 |
| |
| for line in content: |
| found = None |
| for t in re_markup_tags: |
| m = t.match( line ) |
| if m: |
| found = string.lower(m.group(1)) |
| prefix = len(m.group(0)) |
| line = " "*prefix + line[prefix:] # remove markup from line |
| break |
| |
| # is it the start of a new markup section ? |
| if found: |
| first = 0 |
| self.add_markup() # add current markup content |
| self.markup = found |
| if len(string.strip( line )) > 0: |
| self.markup_lines.append( line ) |
| elif first == 0: |
| self.markup_lines.append( line ) |
| |
| self.add_markup() |
| |
| return self.markups |
| |
| |
| def parse_sources( self, source_processor ): |
| blocks = source_processor.blocks |
| count = len(blocks) |
| for n in range(count): |
| |
| source = blocks[n] |
| if source.content: |
| # this is a documentation comment, we need to catch |
| # all following normal blocks in the "follow" list |
| # |
| follow = [] |
| m = n+1 |
| while m < count and not blocks[m].content: |
| follow.append( blocks[m] ) |
| m = m+1 |
| |
| doc_block = DocBlock( source, follow, self ) |
| |
| |
| def finish( self ): |
| |
| # process all sections to extract their abstract, description |
| # and ordered list of items |
| # |
| for sec in self.sections.values(): |
| sec.process() |
| |
| # process chapters to check that all sections are correctly |
| # listed there |
| for chap in self.chapters: |
| for sec in chap.order: |
| if self.sections.has_key(sec): |
| section = self.sections[ sec ] |
| section.chapter = chap |
| section.reorder() |
| chap.sections.append( section ) |
| else: |
| sys.stderr.write( "WARNING: chapter '" + |
| chap.name + "' in " + chap.block.location() + \ |
| " lists unknown section '" + sec + "'\n" ) |
| |
| # check that all sections are in a chapter |
| # |
| others = [] |
| for sec in self.sections.values(): |
| if not sec.chapter: |
| others.append(sec) |
| |
| # create a new special chapter for all remaining sections |
| # when necessary |
| # |
| if others: |
| chap = DocChapter( None ) |
| chap.sections = others |
| self.chapters.append( chap ) |
| |
| |
| |
| class DocBlock: |
| |
| def __init__( self, source, follow, processor ): |
| |
| processor.reset() |
| |
| self.source = source |
| self.code = [] |
| self.type = "ERRTYPE" |
| self.name = "ERRNAME" |
| self.section = processor.section |
| self.markups = processor.process_content( source.content ) |
| |
| # compute block type from first markup tag |
| try: |
| self.type = self.markups[0].tag |
| except: |
| pass |
| |
| |
| # compute block name from first markup paragraph |
| try: |
| markup = self.markups[0] |
| para = markup.fields[0].items[0] |
| name = para.words[0] |
| m = re_identifier.match( name ) |
| if m: |
| name = m.group(1) |
| self.name = name |
| except: |
| pass |
| |
| # detect new section starts |
| if self.type == "section": |
| processor.set_section( self.name ) |
| processor.section.add_def( self ) |
| |
| # detect new chapter |
| elif self.type == "chapter": |
| processor.add_chapter( self ) |
| |
| else: |
| processor.section.add_block( self ) |
| |
| # now, compute the source lines relevant to this documentation |
| # block. We keep normal comments in for obvious reasons (??) |
| source = [] |
| for b in follow: |
| if b.format: |
| break |
| for l in b.lines: |
| # we use "/* */" as a separator |
| if re_source_sep.match( l ): |
| break |
| source.append( l ) |
| |
| # now strip the leading and trailing empty lines from the sources |
| start = 0 |
| end = len( source )-1 |
| |
| while start < end and not string.strip( source[start] ): |
| start = start + 1 |
| |
| while start < end and not string.strip( source[end] ): |
| end = end - 1 |
| |
| source = source[start:end+1] |
| |
| self.code = source |
| |
| |
| def location( self ): |
| return self.source.location() |
| |
| |
| |
| def get_markup( self, tag_name ): |
| """return the DocMarkup corresponding to a given tag in a block""" |
| for m in self.markups: |
| if m.tag == string.lower(tag_name): |
| return m |
| return None |
| |
| |
| def get_markup_name( self, tag_name ): |
| """return the name of a given primary markup in a block""" |
| try: |
| m = self.get_markup( tag_name ) |
| return m.get_name() |
| except: |
| return None |
| |
| |
| def get_markup_words( self, tag_name ): |
| try: |
| m = self.get_markup( tag_name ) |
| return m.fields[0].items[0].words |
| except: |
| return [] |
| |
| |
| def get_markup_text( self, tag_name ): |
| result = self.get_markup_words( tag_name ) |
| return string.join( result ) |
| |
| |
| def get_markup_items( self, tag_name ): |
| try: |
| m = self.get_markup( tag_name ) |
| return m.fields[0].items |
| except: |
| return None |
| |
| # eof |
