1 | # -*- coding: utf-8 -*- |
---|
2 | |
---|
3 | |
---|
4 | from __future__ import with_statement |
---|
5 | |
---|
6 | import sys, os, re, datetime, urllib, optparse, contextlib, subprocess, codecs |
---|
7 | |
---|
8 | from ll import sisyphus, url, ul4c |
---|
9 | |
---|
10 | from pycoco import xmlns |
---|
11 | |
---|
12 | |
---|
13 | encodingdeclaration = re.compile(r"coding[:=]\s*([-\w.]+)") |
---|
14 | |
---|
15 | |
---|
16 | class File(object): |
---|
17 | def __init__(self, name): |
---|
18 | self.name = name |
---|
19 | self.lines = [] # list of lines with tuples (# of executions, line) |
---|
20 | |
---|
21 | def __repr__(self): |
---|
22 | return "<File name=%r at 0x%x>" % (self.name, id(self)) |
---|
23 | |
---|
24 | |
---|
25 | class Python_GenerateCodeCoverage(sisyphus.Job): |
---|
26 | argdescription = "Generate code coverage info for the Python source code" |
---|
27 | projectname = "Python" |
---|
28 | jobname = "GenerateCodeCoverage" |
---|
29 | maxtime = 6* 60 * 60 # 6 hours |
---|
30 | |
---|
31 | def __init__(self): |
---|
32 | self.url = url.URL("http://svn.python.org/snapshots/python3k.tar.bz2") |
---|
33 | self.outputdir = url.Dir("~/pycoco") |
---|
34 | |
---|
35 | self.configurecmd = "./configure --enable-unicode=ucs4 --with-pydebug" |
---|
36 | |
---|
37 | self.gcovcmd = os.environ.get("COV", "gcov") |
---|
38 | self.makefile = "python/Makefile" |
---|
39 | |
---|
40 | self.buildlog = [] # the output of configuring and building Python |
---|
41 | self.testlog = [] # the output of running the test suite |
---|
42 | |
---|
43 | def cmd(self, cmd): |
---|
44 | self.log(">>> %s" % cmd) |
---|
45 | pipe = subprocess.Popen(cmd, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout |
---|
46 | lines = [] |
---|
47 | for line in pipe: |
---|
48 | self.log("... " + line) |
---|
49 | lines.append(line) |
---|
50 | return lines |
---|
51 | |
---|
52 | def files(self, base): |
---|
53 | with self.prefix("files: "): |
---|
54 | self.log("### finding files") |
---|
55 | allfiles = [] |
---|
56 | for (root, dirs, files) in os.walk(base): |
---|
57 | for file in files: |
---|
58 | if file.endswith(".py") or file.endswith(".c"): |
---|
59 | allfiles.append(File(os.path.join(root, file))) |
---|
60 | self.log("### found %d files" % len(allfiles)) |
---|
61 | return allfiles |
---|
62 | |
---|
63 | def download(self): |
---|
64 | with self.prefix("download: "): |
---|
65 | self.log("### downloading %s to %s" % (self.url, self.url.file)) |
---|
66 | urllib.urlretrieve(str(self.url), str(self.url.file)) |
---|
67 | |
---|
68 | def unpack(self): |
---|
69 | with self.prefix("unpack: "): |
---|
70 | self.log("### unpacking %s" % self.url.file) |
---|
71 | self.cmd("tar xvjf %s" % self.url.file) |
---|
72 | lines = list(open("python/.timestamp", "r")) |
---|
73 | self.timestamp = datetime.datetime.fromtimestamp(int(lines[0])) |
---|
74 | self.revision = lines[2] |
---|
75 | |
---|
76 | def configure(self): |
---|
77 | with self.prefix("configure: "): |
---|
78 | self.log("### configuring") |
---|
79 | lines = self.cmd("cd python; %s" % self.configurecmd) |
---|
80 | self.buildlog.extend(lines) |
---|
81 | |
---|
82 | def make(self): |
---|
83 | with self.prefix("make: "): |
---|
84 | self.log("### running make") |
---|
85 | self.buildlog.extend(self.cmd("cd python && make coverage")) |
---|
86 | |
---|
87 | def test(self): |
---|
88 | with self.prefix("test: "): |
---|
89 | self.log("### running test") |
---|
90 | lines = self.cmd("cd python && ./python -mtest.regrtest -T -N -uurlfetch,largefile,network,decimal") |
---|
91 | self.testlog.extend(lines) |
---|
92 | |
---|
93 | def cleanup(self): |
---|
94 | with self.prefix("cleanup: "): |
---|
95 | self.log("### cleaning up files from previous run") |
---|
96 | self.cmd("rm -rf python") |
---|
97 | self.cmd("rm %s" % self.url.file) |
---|
98 | |
---|
99 | def coveruncovered(self, file): |
---|
100 | with self.prefix("cover: "): |
---|
101 | self.log("### faking coverage info for uncovered file %r" % file.name) |
---|
102 | with open(file.name, "r") as f: |
---|
103 | file.lines = [(None, line.rstrip("\n")) for line in f] |
---|
104 | |
---|
105 | def coverpy(self, file): |
---|
106 | with self.prefix("cover: "): |
---|
107 | coverfilename = os.path.splitext(file.name)[0] + ".cover" |
---|
108 | self.log("### fetching coverage info for Python file %r from %r" % (file.name, coverfilename)) |
---|
109 | try: |
---|
110 | with open(coverfilename, "r") as f: |
---|
111 | for line in f: |
---|
112 | line = line.rstrip("\n") |
---|
113 | prefix, line = line[:7], line[7:] |
---|
114 | prefix = prefix.strip() |
---|
115 | if prefix == "." or prefix == "": |
---|
116 | file.lines.append((-1, line)) |
---|
117 | elif prefix == ">"*len(prefix): |
---|
118 | file.lines.append((0, line)) |
---|
119 | else: |
---|
120 | file.lines.append((int(prefix.rstrip(":")), line)) |
---|
121 | except IOError, exc: |
---|
122 | self.log.exc(exc) |
---|
123 | self.coveruncovered(file) |
---|
124 | |
---|
125 | def coverc(self, file): |
---|
126 | with self.prefix("cover: "): |
---|
127 | self.log("### fetching coverage info for C file %r" % file.name) |
---|
128 | dirname = os.path.dirname(file.name).split("/", 1)[-1] |
---|
129 | basename = os.path.basename(file.name) |
---|
130 | self.cmd("cd python && %s %s -o %s" % (self.gcovcmd, basename, dirname)) |
---|
131 | try: |
---|
132 | with open("python/%s.gcov" % basename, "r") as f: |
---|
133 | for line in f: |
---|
134 | line = line.rstrip("\n") |
---|
135 | if line.count(":") < 2: |
---|
136 | continue |
---|
137 | (count, lineno, line) = line.split(":", 2) |
---|
138 | count = count.strip() |
---|
139 | lineno = lineno.strip() |
---|
140 | if lineno == "0": # just the header |
---|
141 | continue |
---|
142 | if count == "-": # not executable |
---|
143 | file.lines.append((-1, line)) |
---|
144 | elif count == "#####": # not executed |
---|
145 | file.lines.append((0, line)) |
---|
146 | else: |
---|
147 | file.lines.append((int(count), line)) |
---|
148 | except IOError, exc: |
---|
149 | self.log.exc(exc) |
---|
150 | self.coveruncovered(file) |
---|
151 | |
---|
152 | def makehtml(self, files): |
---|
153 | with self.prefix("html: "): |
---|
154 | # Generate main page |
---|
155 | self.log("### generating index page") |
---|
156 | template = ul4c.compile(xmlns.page(xmlns.filelist(), onload="files_prepare()").conv().string()) |
---|
157 | s = template.renders( |
---|
158 | filename=None, |
---|
159 | now=datetime.datetime.now(), |
---|
160 | timestamp=self.timestamp, |
---|
161 | revision=self.revision, |
---|
162 | crumbs=[ |
---|
163 | dict(title="Core Development", href="http://www.python.org/dev/"), |
---|
164 | dict(title="Code coverage", href=None), |
---|
165 | ], |
---|
166 | files=[ |
---|
167 | dict( |
---|
168 | name=file.name.split("/", 1)[-1], |
---|
169 | lines=len(file.lines), |
---|
170 | coverablelines=sum(line[0]>=0 for line in file.lines), |
---|
171 | coveredlines=sum(line[0]>0 for line in file.lines), |
---|
172 | ) for file in files |
---|
173 | ], |
---|
174 | ) |
---|
175 | u = self.outputdir/"index.html" |
---|
176 | with contextlib.closing(u.openwrite()) as f: |
---|
177 | f.write(s.encode("utf-8")) |
---|
178 | |
---|
179 | # Generate page for each source file |
---|
180 | template = ul4c.compile(xmlns.page(xmlns.filecontent()).conv().string()) |
---|
181 | for (i, file) in enumerate(files): |
---|
182 | filename = file.name.split("/", 1)[-1] |
---|
183 | self.log("### generating HTML %d/%d for %s" % (i+1, len(files), filename)) |
---|
184 | s = template.renders( |
---|
185 | filename=filename, |
---|
186 | crumbs=[ |
---|
187 | dict(title="Core Development", href="http://www.python.org/dev/"), |
---|
188 | dict(title="Code coverage", href="/index.html"), |
---|
189 | dict(title=filename, href=None), |
---|
190 | ], |
---|
191 | lines=( |
---|
192 | dict(count=count, content=content.decode("latin-1").expandtabs(8)) for (count, content) in file.lines |
---|
193 | ), |
---|
194 | ) |
---|
195 | u = self.outputdir/(filename + ".html") |
---|
196 | with contextlib.closing(u.openwrite()) as f: |
---|
197 | f.write(s.encode("utf-8")) |
---|
198 | |
---|
199 | # Copy CSS/JS/GIF files |
---|
200 | for filename in ("coverage.css", "coverage_sortfilelist.css", "coverage.js", "spc.gif"): |
---|
201 | self.log("### copying %s" % filename) |
---|
202 | try: |
---|
203 | import pkg_resources |
---|
204 | except ImportError: |
---|
205 | data = open(os.path.join(os.path.dirname(__file__), filename), "rb").read() |
---|
206 | else: |
---|
207 | data = pkg_resources.resource_string(__name__, filename) |
---|
208 | with contextlib.closing((self.outputdir/filename).openwrite()) as f: |
---|
209 | f.write(data) |
---|
210 | |
---|
211 | self.log("### creating buildlog.txt") |
---|
212 | with contextlib.closing((self.outputdir/"buildlog.txt").openwrite()) as f: |
---|
213 | f.write("".join(self.buildlog)) |
---|
214 | |
---|
215 | self.log("### creating testlog.txt") |
---|
216 | with contextlib.closing((self.outputdir/"testlog.txt").openwrite()) as f: |
---|
217 | f.write("".join(self.testlog)) |
---|
218 | |
---|
219 | def execute(self): |
---|
220 | self.cleanup() |
---|
221 | self.download() |
---|
222 | self.unpack() |
---|
223 | self.configure() |
---|
224 | files = self.files("python") |
---|
225 | self.make() |
---|
226 | self.test() |
---|
227 | for file in files: |
---|
228 | if file.name.endswith(".py"): |
---|
229 | self.coverpy(file) |
---|
230 | elif file.name.endswith(".c"): |
---|
231 | self.coverc(file) |
---|
232 | self.makehtml(files) |
---|
233 | return "done with project Python (%s; %d files)" % (self.timestamp.strftime("%Y-%m-%d %H:%M:%S"), len(files)) |
---|
234 | |
---|
235 | def argparser(self): |
---|
236 | p = sisyphus.Job.argparser(self) |
---|
237 | p.add_argument("-u", "--url", dest="url", help="URL of the Python tarball", default=str(self.url), type=url.URL) |
---|
238 | p.add_argument("-d", "--outputdir", dest="outputdir", help="Directory where to put the HTML files", default=str(self.outputdir), type=url.Dir) |
---|
239 | return p |
---|
240 | |
---|
241 | def parseargs(self, args=None): |
---|
242 | args = sisyphus.Job.parseargs(self, args) |
---|
243 | self.url = args.url |
---|
244 | self.outputdir = args.outputdir |
---|
245 | return args |
---|
246 | |
---|
247 | def detectencoding(self, filename): |
---|
248 | with open(filename, "rb") as f: |
---|
249 | for (i, line) in enumerate(f): |
---|
250 | match = encodingdeclaration.search(line) |
---|
251 | if match is not None: |
---|
252 | return match.group(1) |
---|
253 | break |
---|
254 | if i >= 2: |
---|
255 | break |
---|
256 | return "utf-8" |
---|
257 | |
---|
258 | |
---|
259 | def main(args=None): |
---|
260 | sisyphus.executewithargs(Python_GenerateCodeCoverage(), args) |
---|
261 | return 0 |
---|
262 | |
---|
263 | |
---|
264 | if __name__=="__main__": |
---|
265 | sys.exit(main()) |
---|