xen-vtx-unstable

view docs/pythfilter.py @ 6766:64d6de1ea743

"Added."
author ewan@linford.intra
date Sun Sep 11 21:40:15 2005 +0100 (2005-09-11)
parents
children e939d5c5e646
line source
1 #!/usr/bin/env python
3 # pythfilter.py v1.5.5, written by Matthias Baas (baas@ira.uka.de)
5 # Doxygen filter which can be used to document Python source code.
6 # Classes (incl. methods) and functions can be documented.
7 # Every comment that begins with ## is literally turned into an
8 # Doxygen comment. Consecutive comment lines are turned into
9 # comment blocks (-> /** ... */).
10 # All the stuff is put inside a namespace with the same name as
11 # the source file.
13 # Conversions:
14 # ============
15 # ##-blocks -> /** ... */
16 # "class name(base): ..." -> "class name : public base {...}"
17 # "def name(params): ..." -> "name(params) {...}"
19 # Changelog:
20 # 21.01.2003: Raw (r"") or unicode (u"") doc string will now be properly
21 # handled. (thanks to Richard Laager for the patch)
22 # 22.12.2003: Fixed a bug where no function names would be output for "def"
23 # blocks that were not in a class.
24 # (thanks to Richard Laager for the patch)
25 # 12.12.2003: Implemented code to handle static and class methods with
26 # this logic: Methods with "self" as the first argument are
27 # non-static. Methods with "cls" are Python class methods,
28 # which translate into static methods for Doxygen. Other
29 # methods are assumed to be static methods. As should be
30 # obvious, this logic doesn't take into account if the method
31 # is actually setup as a classmethod() or a staticmethod(),
32 # just if it follows the normal conventions.
33 # (thanks to Richard Laager for the patch)
34 # 11.12.2003: Corrected #includes to use os.path.sep instead of ".". Corrected
35 # namespace code to use "::" instead of ".".
36 # (thanks to Richard Laager for the patch)
37 # 11.12.2003: Methods beginning with two underscores that end with
38 # something other than two underscores are considered private
39 # and are handled accordingly.
40 # (thanks to Richard Laager for the patch)
41 # 03.12.2003: The first parameter of class methods (self) is removed from
42 # the documentation.
43 # 03.11.2003: The module docstring will be used as namespace documentation
44 # (thanks to Joe Bronkema for the patch)
45 # 08.07.2003: Namespaces get a default documentation so that the namespace
46 # and its contents will show up in the generated documentation.
47 # 05.02.2003: Directories will be delted during synchronization.
48 # 31.01.2003: -f option & filtering entire directory trees.
49 # 10.08.2002: In base classes the '.' will be replaced by '::'
50 # 18.07.2002: * and ** will be translated into arguments
51 # 18.07.2002: Argument lists may contain default values using constructors.
52 # 18.06.2002: Support for ## public:
53 # 21.01.2002: from ... import will be translated to "using namespace ...;"
54 # TODO: "from ... import *" vs "from ... import names"
55 # TODO: Using normal imports: name.name -> name::name
56 # 20.01.2002: #includes will be placed in front of the namespace
58 ######################################################################
60 # The program is written as a state machine with the following states:
61 #
62 # - OUTSIDE The current position is outside any comment,
63 # class definition or function.
64 #
65 # - BUILD_COMMENT Begins with first "##".
66 # Ends with the first token that is no "##"
67 # at the same column as before.
68 #
69 # - BUILD_CLASS_DECL Begins with "class".
70 # Ends with ":"
71 # - BUILD_CLASS_BODY Begins just after BUILD_CLASS_DECL.
72 # The first following token (which is no comment)
73 # determines indentation depth.
74 # Ends with a token that has a smaller indendation.
75 #
76 # - BUILD_DEF_DECL Begins with "def".
77 # Ends with ":".
78 # - BUILD_DEF_BODY Begins just after BUILD_DEF_DECL.
79 # The first following token (which is no comment)
80 # determines indentation depth.
81 # Ends with a token that has a smaller indendation.
83 import getopt
84 import glob
85 import os.path
86 import re
87 import shutil
88 import string
89 import sys
90 import token
91 import tokenize
93 from stat import *
95 OUTSIDE = 0
96 BUILD_COMMENT = 1
97 BUILD_CLASS_DECL = 2
98 BUILD_CLASS_BODY = 3
99 BUILD_DEF_DECL = 4
100 BUILD_DEF_BODY = 5
101 IMPORT = 6
102 IMPORT_OP = 7
103 IMPORT_APPEND = 8
105 # Output file stream
106 outfile = sys.stdout
108 # Output buffer
109 outbuffer = []
111 out_row = 1
112 out_col = 0
114 # Variables used by rec_name_n_param()
115 name = ""
116 param = ""
117 doc_string = ""
118 record_state = 0
119 bracket_counter = 0
121 # Tuple: (row,column)
122 class_spos = (0,0)
123 def_spos = (0,0)
124 import_spos = (0,0)
126 # Which import was used? ("import" or "from")
127 import_token = ""
129 # Comment block buffer
130 comment_block = []
131 comment_finished = 0
133 # Imported modules
134 modules = []
136 # Program state
137 stateStack = [OUTSIDE]
139 # Keep track of whether module has a docstring
140 module_has_docstring = False
142 # Keep track of member protection
143 protection_level = "public"
144 private_member = False
146 # Keep track of the module namespace
147 namespace = ""
149 ######################################################################
150 # Output string s. '\n' may only be at the end of the string (not
151 # somewhere in the middle).
152 #
153 # In: s - String
154 # spos - Startpos
155 ######################################################################
156 def output(s,spos, immediate=0):
157 global outbuffer, out_row, out_col, outfile
159 os = string.rjust(s,spos[1]-out_col+len(s))
161 if immediate:
162 outfile.write(os)
163 else:
164 outbuffer.append(os)
166 assert -1 == string.find(s[0:-2], "\n"), s
168 if (s[-1:]=="\n"):
169 out_row = out_row+1
170 out_col = 0
171 else:
172 out_col = spos[1]+len(s)
175 ######################################################################
176 # Records a name and parameters. The name is either a class name or
177 # a function name. Then the parameter is either the base class or
178 # the function parameters.
179 # The name is stored in the global variable "name", the parameters
180 # in "param".
181 # The variable "record_state" holds the current state of this internal
182 # state machine.
183 # The recording is started by calling start_recording().
184 #
185 # In: type, tok
186 ######################################################################
187 def rec_name_n_param(type, tok):
188 global record_state,name,param,doc_string,bracket_counter
189 s = record_state
190 # State 0: Do nothing.
191 if (s==0):
192 return
193 # State 1: Remember name.
194 elif (s==1):
195 name = tok
196 record_state = 2
197 # State 2: Wait for opening bracket or colon
198 elif (s==2):
199 if (tok=='('):
200 bracket_counter = 1
201 record_state=3
202 if (tok==':'): record_state=4
203 # State 3: Store parameter (or base class) and wait for an ending bracket
204 elif (s==3):
205 if (tok=='*' or tok=='**'):
206 tok=''
207 if (tok=='('):
208 bracket_counter = bracket_counter+1
209 if (tok==')'):
210 bracket_counter = bracket_counter-1
211 if bracket_counter==0:
212 record_state=4
213 else:
214 param=param+tok
215 # State 4: Look for doc string
216 elif (s==4):
217 if (type==token.NEWLINE or type==token.INDENT or type==token.SLASHEQUAL):
218 return
219 elif (tok==":"):
220 return
221 elif (type==token.STRING):
222 while tok[:1]=='r' or tok[:1]=='u':
223 tok=tok[1:]
224 while tok[:1]=='"':
225 tok=tok[1:]
226 while tok[-1:]=='"':
227 tok=tok[:-1]
228 doc_string=tok
229 record_state=0
231 ######################################################################
232 # Starts the recording of a name & param part.
233 # The function rec_name_n_param() has to be fed with tokens. After
234 # the necessary tokens are fed the name and parameters can be found
235 # in the global variables "name" und "param".
236 ######################################################################
237 def start_recording():
238 global record_state,param,name, doc_string
239 record_state=1
240 name=""
241 param=""
242 doc_string=""
244 ######################################################################
245 # Test if recording is finished
246 ######################################################################
247 def is_recording_finished():
248 global record_state
249 return record_state==0
251 ######################################################################
252 ## Gather comment block
253 ######################################################################
254 def gather_comment(type,tok,spos):
255 global comment_block,comment_finished
256 if (type!=tokenize.COMMENT):
257 comment_finished = 1
258 else:
259 # Output old comment block if a new one is started.
260 if (comment_finished):
261 print_comment(spos)
262 comment_finished=0
263 if (tok[0:2]=="##" and tok[0:3]!="###"):
264 append_comment_lines(tok[2:])
266 ######################################################################
267 ## Output comment block and empty buffer.
268 ######################################################################
269 def print_comment(spos):
270 global comment_block,comment_finished
271 if (comment_block!=[]):
272 output("/** ",spos)
273 for c in comment_block:
274 output(c,spos)
275 output("*/\n",spos)
276 comment_block = []
277 comment_finished = 0
279 ######################################################################
280 def set_state(s):
281 global stateStack
282 stateStack[len(stateStack)-1]=s
284 ######################################################################
285 def get_state():
286 global stateStack
287 return stateStack[len(stateStack)-1]
289 ######################################################################
290 def push_state(s):
291 global stateStack
292 stateStack.append(s)
294 ######################################################################
295 def pop_state():
296 global stateStack
297 stateStack.pop()
300 ######################################################################
301 def tok_eater(type, tok, spos, epos, line):
302 global stateStack,name,param,class_spos,def_spos,import_spos
303 global doc_string, modules, import_token, module_has_docstring
304 global protection_level, private_member
305 global out_row
307 while out_row + 1 < spos[0]:
308 output("\n", (0, 0))
310 rec_name_n_param(type,tok)
311 if (string.replace(string.strip(tok)," ","")=="##private:"):
312 protection_level = "private"
313 output("private:\n",spos)
314 elif (string.replace(string.strip(tok)," ","")=="##protected:"):
315 protection_level = "protected"
316 output("protected:\n",spos)
317 elif (string.replace(string.strip(tok)," ","")=="##public:"):
318 protection_level = "public"
319 output("public:\n",spos)
320 else:
321 gather_comment(type,tok,spos)
323 state = get_state()
325 # sys.stderr.write("%d: %s\n"%(state, tok))
327 # OUTSIDE
328 if (state==OUTSIDE):
329 if (tok=="class"):
330 start_recording()
331 class_spos = spos
332 push_state(BUILD_CLASS_DECL)
333 elif (tok=="def"):
334 start_recording()
335 def_spos = spos
336 push_state(BUILD_DEF_DECL)
337 elif (tok=="import") or (tok=="from"):
338 import_token = tok
339 import_spos = spos
340 modules = []
341 push_state(IMPORT)
342 elif (spos[1] == 0 and tok[:3] == '"""'):
343 # Capture module docstring as namespace documentation
344 module_has_docstring = True
345 append_comment_lines("\\namespace %s\n" % namespace)
346 append_comment_lines(tok[3:-3])
347 print_comment(spos)
349 # IMPORT
350 elif (state==IMPORT):
351 if (type==token.NAME):
352 modules.append(tok)
353 set_state(IMPORT_OP)
354 # IMPORT_OP
355 elif (state==IMPORT_OP):
356 if (tok=="."):
357 set_state(IMPORT_APPEND)
358 elif (tok==","):
359 set_state(IMPORT)
360 else:
361 for m in modules:
362 output('#include "'+m.replace('.',os.path.sep)+'.py"\n', import_spos, immediate=1)
363 if import_token=="from":
364 output('using namespace '+m.replace('.', '::')+';\n', import_spos)
365 pop_state()
366 # IMPORT_APPEND
367 elif (state==IMPORT_APPEND):
368 if (type==token.NAME):
369 modules[len(modules)-1]+="."+tok
370 set_state(IMPORT_OP)
371 # BUILD_CLASS_DECL
372 elif (state==BUILD_CLASS_DECL):
373 if (is_recording_finished()):
374 s = "class "+name
375 if (param!=""): s = s+" : public "+param.replace('.','::')
376 if (doc_string!=""):
377 append_comment_lines(doc_string)
378 print_comment(class_spos)
379 output(s+"\n",class_spos)
380 output("{\n",(class_spos[0]+1,class_spos[1]))
381 protection_level = "public"
382 output(" public:\n",(class_spos[0]+2,class_spos[1]))
383 set_state(BUILD_CLASS_BODY)
384 # BUILD_CLASS_BODY
385 elif (state==BUILD_CLASS_BODY):
386 if (type!=token.INDENT and type!=token.NEWLINE and type!=40 and
387 type!=tokenize.NL and type!=tokenize.COMMENT and
388 (spos[1]<=class_spos[1])):
389 output("}; // end of class\n",(out_row+1,class_spos[1]))
390 pop_state()
391 elif (tok=="def"):
392 start_recording()
393 def_spos = spos
394 push_state(BUILD_DEF_DECL)
395 # BUILD_DEF_DECL
396 elif (state==BUILD_DEF_DECL):
397 if (is_recording_finished()):
398 param = param.replace("\n", " ")
399 param = param.replace("=", " = ")
400 params = param.split(",")
401 if BUILD_CLASS_BODY in stateStack:
402 if len(name) > 1 \
403 and name[0:2] == '__' \
404 and name[len(name)-2:len(name)] != '__' \
405 and protection_level != 'private':
406 private_member = True
407 output(" private:\n",(def_spos[0]+2,def_spos[1]))
409 if (doc_string != ""):
410 append_comment_lines(doc_string)
412 print_comment(def_spos)
414 output_function_decl(name, params)
415 # output("{\n",(def_spos[0]+1,def_spos[1]))
416 set_state(BUILD_DEF_BODY)
417 # BUILD_DEF_BODY
418 elif (state==BUILD_DEF_BODY):
419 if (type!=token.INDENT and type!=token.NEWLINE \
420 and type!=40 and type!=tokenize.NL \
421 and (spos[1]<=def_spos[1])):
422 # output("} // end of method/function\n",(out_row+1,def_spos[1]))
423 if private_member and protection_level != 'private':
424 private_member = False
425 output(" " + protection_level + ":\n",(def_spos[0]+2,def_spos[1]))
426 pop_state()
427 # else:
428 # output(tok,spos)
431 def output_function_decl(name, params):
432 global def_spos
434 # Do we document a class method? then remove the 'self' parameter
435 if params[0] == 'self':
436 preamble = ''
437 params = params[1:]
438 else:
439 preamble = 'static '
440 if params[0] == 'cls':
441 params = params[1:]
443 param_string = string.join(params, ", Type ")
445 if param_string == '':
446 param_string = '(' + param_string + ');\n'
447 else:
448 param_string = '(Type ' + param_string + ');\n'
450 output(preamble, def_spos)
451 output(name, def_spos)
452 output(param_string, def_spos)
455 def append_comment_lines(lines):
456 map(append_comment_line, doc_string.split('\n'))
458 paramRE = re.compile(r'(@param \w+):')
460 def append_comment_line(line):
461 global paramRE
463 comment_block.append(paramRE.sub(r'\1', line) + '\n')
465 def dump(filename):
466 f = open(filename)
467 r = f.readlines()
468 for s in r:
469 sys.stdout.write(s)
471 def filter(filename):
472 global name, module_has_docstring
474 path,name = os.path.split(filename)
475 root,ext = os.path.splitext(name)
477 output("namespace "+root+" {\n",(0,0))
479 # set module name for tok_eater to use if there's a module doc string
480 name = root
482 # sys.stderr.write('Filtering "'+filename+'"...')
483 f = open(filename)
484 tokenize.tokenize(f.readline, tok_eater)
485 f.close()
486 print_comment((0,0))
488 output("\n",(0,0))
489 output("} // end of namespace\n",(0,0))
491 if not module_has_docstring:
492 # Put in default namespace documentation
493 output('/** \\namespace '+root+' \n',(0,0))
494 output(' \\brief Module "%s" */\n'%(root),(0,0))
496 for s in outbuffer:
497 outfile.write(s)
500 def filterFile(filename, out=sys.stdout):
501 global outfile
503 outfile = out
505 try:
506 root,ext = os.path.splitext(filename)
508 if ext==".py":
509 filter(filename)
510 else:
511 dump(filename)
513 # sys.stderr.write("OK\n")
514 except IOError,e:
515 sys.stderr.write(e[1]+"\n")
518 ######################################################################
520 # preparePath
521 def preparePath(path):
522 """Prepare a path.
524 Checks if the path exists and creates it if it does not exist.
525 """
526 if not os.path.exists(path):
527 parent = os.path.dirname(path)
528 if parent!="":
529 preparePath(parent)
530 os.mkdir(path)
532 # isNewer
533 def isNewer(file1,file2):
534 """Check if file1 is newer than file2.
536 file1 must be an existing file.
537 """
538 if not os.path.exists(file2):
539 return True
540 return os.stat(file1)[ST_MTIME]>os.stat(file2)[ST_MTIME]
542 # convert
543 def convert(srcpath, destpath):
544 """Convert a Python source tree into a C+ stub tree.
546 All *.py files in srcpath (including sub-directories) are filtered
547 and written to destpath. If destpath exists, only the files
548 that have been modified are filtered again. Files that were deleted
549 from srcpath are also deleted in destpath if they are still present.
550 The function returns the number of processed *.py files.
551 """
552 count=0
553 sp = os.path.join(srcpath,"*")
554 sfiles = glob.glob(sp)
555 dp = os.path.join(destpath,"*")
556 dfiles = glob.glob(dp)
557 leftovers={}
558 for df in dfiles:
559 leftovers[os.path.basename(df)]=1
561 for srcfile in sfiles:
562 basename = os.path.basename(srcfile)
563 if basename in leftovers:
564 del leftovers[basename]
566 # Is it a subdirectory?
567 if os.path.isdir(srcfile):
568 sdir = os.path.join(srcpath,basename)
569 ddir = os.path.join(destpath,basename)
570 count+=convert(sdir, ddir)
571 continue
572 # Check the extension (only *.py will be converted)
573 root, ext = os.path.splitext(srcfile)
574 if ext.lower()!=".py":
575 continue
577 destfile = os.path.join(destpath,basename)
578 if destfile==srcfile:
579 print "WARNING: Input and output names are identical!"
580 sys.exit(1)
582 count+=1
583 # sys.stdout.write("%s\015"%(srcfile))
585 if isNewer(srcfile, destfile):
586 preparePath(os.path.dirname(destfile))
587 # out=open(destfile,"w")
588 # filterFile(srcfile, out)
589 # out.close()
590 os.system("python %s -f %s>%s"%(sys.argv[0],srcfile,destfile))
592 # Delete obsolete files in destpath
593 for df in leftovers:
594 dname=os.path.join(destpath,df)
595 if os.path.isdir(dname):
596 try:
597 shutil.rmtree(dname)
598 except:
599 print "Can't remove obsolete directory '%s'"%dname
600 else:
601 try:
602 os.remove(dname)
603 except:
604 print "Can't remove obsolete file '%s'"%dname
606 return count
609 ######################################################################
610 ######################################################################
611 ######################################################################
613 filter_file = False
615 try:
616 opts, args = getopt.getopt(sys.argv[1:], "hf", ["help"])
617 except getopt.GetoptError,e:
618 print e
619 sys.exit(1)
621 for o,a in opts:
622 if o=="-f":
623 filter_file = True
625 if filter_file:
626 # Filter the specified file and print the result to stdout
627 filename = string.join(args)
628 filterFile(filename)
629 else:
631 if len(args)!=2:
632 sys.stderr.write("%s options input output\n"%(os.path.basename(sys.argv[0])))
633 sys.exit(1)
635 # Filter an entire Python source tree
636 print '"%s" -> "%s"\n'%(args[0],args[1])
637 c=convert(args[0],args[1])
638 print "%d files"%(c)