root/livinglogic.python.xist/src/ll/make.py @ 5334:a6236d8d4a7f

Revision 5334:a6236d8d4a7f, 53.2 KB (checked in by Walter Doerwald <walter@…>, 7 years ago)

Simplify message formatting.

  • Property exe set to *
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4## Copyright 2002-2013 by LivingLogic AG, Bayreuth/Germany.
5## Copyright 2002-2013 by Walter Dörwald
6##
7## All Rights Reserved
8##
9## See ll/xist/__init__.py for the license
10
11
12"""
13:mod:`ll.make` provides tools for building projects.
14
15Like make it allows you to specify dependencies between files and actions to be
16executed when files don't exist or are out of date with respect to one of their
17sources. But unlike make you can do this in an object oriented way and targets
18are not only limited to files.
19
20Relevant classes are:
21
22*   :class:`Project`, which is the container for all actions in a project and
23
24*   :class:`Action` (and subclasses), which are used to transform input data
25    and read and write files (or other entities like database records).
26
27A simple script that copies a file :file:`foo.txt` to :file:`bar.txt`
28reencoding it from ``"latin-1"`` to ``"utf-8"`` in the process looks like
29this::
30
31    from ll import make
32
33    class MyProject(make.Project):
34        name = "Acme.MyProject"
35
36        def create(self):
37            make.Project.create(self)
38            source = self.add(make.FileAction("foo.txt"))
39            temp = source.callattr("decode", "iso-8859-1")
40            temp = temp.callattr("encode", "utf-8")
41            target = self.add(make.FileAction("bar.txt", temp))
42            self.writecreatedone()
43
44    p = MyProject()
45    p.create()
46
47    if __name__ == "__main__":
48        p.build("bar.txt")
49"""
50
51
52import sys, os, os.path, argparse, warnings, re, datetime, io, errno, tempfile, operator, types, pickle, gc, contextlib, gzip
53
54from ll import misc, url, astyle
55
56
57__docformat__ = "reStructuredText"
58
59
60###
61### Constants and helpers
62###
63
64nodata = misc.Const("nodata") # marker object for "no new data available"
65
66bigbang = datetime.datetime(1900, 1, 1) # there can be no timestamp before this one
67bigcrunch = datetime.datetime(3000, 1, 1) # there can be no timestamp after this one
68
69
70def filechanged(key):
71    """
72    Get the last modified date (or :const:`bigbang`, if the file doesn't exist).
73    """
74    try:
75        return key.mdate()
76    except (IOError, OSError):
77        return bigbang
78
79
80class Level(object):
81    """
82    Stores information about the recursive execution of :class:`Action`\s.
83    """
84    __slots__ = ("action", "since", "reportable", "reported")
85
86    def __init__(self, action, since, reportable, reported=False):
87        self.action = action
88        self.since = since
89        self.reportable = reportable
90        self.reported = reported
91
92    def __repr__(self):
93        return "<{}.{} object action={!r} since={!r} reportable={!r} reported={} at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, self.action, self.since, self.reportable, self.reported, id(self))
94
95
96def report(func):
97    """
98    Standard decorator for :meth:`Action.get` methods.
99
100    This decorator handles proper reporting of nested action calls. If it isn't
101    used, only the output of calls to :meth:`Project.writestep` will be visible
102    to the user.
103    """
104    def reporter(self, project, since):
105        reported = False
106        reportable = project.showaction is not None and isinstance(self, project.showaction)
107        if reportable:
108            if project.showidle:
109                args = ["Starting ", project.straction(self)]
110                if project.showtimestamps:
111                    args.append(" since ")
112                    args.append(project.strdatetime(since))
113                project.writestack(*args)
114                reported = True
115        level = Level(self, since, reportable, reported)
116        project.stack.append(level)
117        t1 = datetime.datetime.utcnow()
118        try:
119            data = func(self, project, since)
120        except Exception as exc:
121            project.actionsfailed += 1
122            if project.ignoreerrors: # ignore changes in failed subgraphs
123                data = nodata # Return "everything is up to date" in this case
124                error = exc.__class__
125            else:
126                raise
127        else:
128            project.actionscalled += 1
129            error = None
130        finally:
131            project.stack.pop(-1)
132        t2 = datetime.datetime.utcnow()
133        if level.reportable or error is not None:
134            if (not project.showidle and data is not nodata) or error is not None:
135                project._writependinglevels() # Only outputs something if the action hasn't called writestep()
136            if level.reported:
137                if error is not None:
138                    text1 = "Failed"
139                    text2 = " after "
140                else:
141                    text1 = "Finished"
142                    text2 = " in "
143                args = [text1, " ", project.straction(self)]
144                if project.showtime:
145                    args.append(text2)
146                    args.append(project.strtimedelta(t2-t1))
147                if project.showtimestamps:
148                    args.append(" (changed ")
149                    args.append(project.strdatetime(self.changed))
150                    args.append(")")
151                if project.showdata:
152                    args.append(": ")
153                    if error is not None:
154                        fmt = "{0.__module__}.{0.__name__}" if error.__module__ != "exceptions" else "{0.__name__}"
155                        text = fmt.format(error)
156                        args.append(s4error(text))
157                    else:
158                        args.append(project.strdata(data))
159                project.writestack(*args)
160        return data
161    reporter.__dict__.update(func.__dict__)
162    reporter.__doc__ = func.__doc__
163    reporter.__name__ = func.__name__
164    return reporter
165
166
167###
168### exceptions & warnings
169###
170
171class RedefinedTargetWarning(Warning):
172    """
173    Warning that will be issued when a target is added to a project and a target
174    with the same key already exists.
175    """
176
177    def __init__(self, key):
178        self.key = key
179
180    def __str__(self):
181        return "target with key={!r} redefined".format(self.key)
182
183
184class UndefinedTargetError(KeyError):
185    """
186    Exception that will be raised when a target with the specified key doesn't
187    exist within the project.
188    """
189
190    def __init__(self, key):
191        self.key = key
192
193    def __str__(self):
194        return "target {!r} undefined".format(self.key)
195
196
197###
198### Actions
199###
200
201def getoutputs(project, since, input):
202    """
203    Recursively iterate through the object :var:`input` (if it's a
204    :class:`tuple`, :class:`list` or :class:`dict`) and return a tuple
205    containing:
206
207    *   An object (:var:`data`) of the same structure as :var:`input`, where every
208        action object encountered is replaced with the output of that action;
209
210    *   A timestamp (:var:`changed`) which the newest timestamp among all the
211        change timestamps of the actions encountered.
212
213    If none of the actions has any data newer than :var:`since` (i.e. none of
214    the actions produced any new data) :var:`data` will be :const:`nodata`.
215    """
216    if isinstance(input, Action):
217        return (input.get(project, since), input.changed)
218    elif isinstance(input, (list, tuple)):
219        resultdata = []
220        havedata = False
221        resultchanged = bigbang
222        for item in input:
223            (data, changed) = getoutputs(project, since, item)
224            resultchanged = max(resultchanged, changed)
225            if data is not nodata and not havedata: # The first real output
226                since = bigbang # force inputs to produce data for the rest of the loop
227                resultdata = [getoutputs(project, since, item)[0] for item in input[:len(resultdata)]] # refetch data from previous inputs
228                havedata = True
229            resultdata.append(data)
230        if since is bigbang and not input:
231            resultdata = input.__class__()
232        elif not havedata:
233            resultdata = nodata
234        elif isinstance(input, tuple):
235            resultdata = tuple(resultdata)
236        return (resultdata, resultchanged)
237    elif isinstance(input, dict):
238        resultdata = {}
239        havedata = False
240        resultchanged = bigbang
241        for (key, value) in input.items():
242            (data, changed) = getoutputs(project, since, value)
243            resultchanged = max(resultchanged, changed)
244            if data is not nodata and not havedata: # The first real output
245                since = bigbang # force inputs to produce data for the rest of the loop
246                resultdata = dict((key, getoutputs(project, since, input[key])[0]) for key in resultdata) # refetch data from previous inputs
247                havedata = True
248            resultdata[key] = data
249        if since is bigbang and not input:
250            resultdata = {}
251        elif not havedata:
252            resultdata = nodata
253        return (resultdata, resultchanged)
254    else:
255        return (input if since is bigbang else nodata, bigbang)
256
257
258def _ipipe_type(obj):
259    try:
260        return obj.type
261    except AttributeError:
262        return "{}.{}".format(obj.__class__.__module__, obj.__class__.__name__)
263_ipipe_type.__xname__ = "type"
264
265
266def _ipipe_key(obj):
267    return obj.getkey()
268_ipipe_key.__xname__ = "key"
269
270
271class Action(object):
272    """
273    An :class:`Action` is responsible for transforming input data into output
274    data. It may have no, one or many inputs which themselves may be other actions.
275    It fetches, combines and transforms the output data of those actions and
276    returns its own output data.
277    """
278
279    key = None
280
281    def __init__(self):
282        """
283        Create a new :class:`Action` instance.
284        """
285        self.changed = bigbang
286
287    @report
288    def get(self, project, since):
289        """
290        This method (i.e. the implementations in subclasses) is the workhorse of
291        :mod:`ll.make`. :meth:`get` must return the output data of the action if
292        this data has changed since :var:`since` (which is a
293        :class:`datetime.datetime` object in UTC). If the data hasn't changed
294        since :var:`since` the special object :const:`nodata` must be returned.
295
296        In both cases the action must make sure that the data is internally
297        consistent, i.e. if the input data is the output data of other actions
298        :var:`self` has to ensure that those other actions update their data too,
299        independent from the fact whether :meth:`get` will return new data or not.
300
301        Two special values can be passed for :var:`since`:
302
303        :const:`bigbang`
304            This timestamp is older than any timestamp that can appear in real
305            life. Since all data is newer than this, :meth:`get` must always
306            return output data.
307
308        :const:`bigcrunch`
309            This timestamp is newer than any timestamp that can appear in real
310            life. Since there can be no data newer than this, :meth:`get` can
311            only return output data in this case if ensuring internal consistency
312            resulted in new data.
313
314        In all cases :meth:`get` must set the instance attribute :attr:`changed`
315        to the timestamp of the last change to the data before returning. In most
316        cases this if the newest :attr:`changed` timestamp of the input actions.
317        """
318        input = (self.getargs(), self.getkwargs())
319        (data, self.changed) = getoutputs(project, since, input)
320        if data is not nodata:
321            data = self.execute(project, *data[0], **data[1])
322        return data
323
324    @misc.notimplemented
325    def execute(self, project, *args, **kwargs):
326        """
327        Execute the action: transform the input data in :var:`args` and
328        :var:`kwargs` and return the resulting output data. This method must be
329        implemented in subclasses.
330        """
331
332    def getkey(self):
333        """
334        Get the nearest key from :var:`self` or its inputs. This is used by
335        :class:`ModuleAction` for the filename.
336        """
337        return self.key
338
339    def getargs(self):
340        return ()
341
342    def getkwargs(self):
343        return {}
344
345    def call(self, *args, **kwargs):
346        """
347        Return a :class:`CallAction` for calling :var:`self`\s output with
348        positional arguments from :var:`args` and keyword arguments from
349        :var:`kwargs`.
350        """
351        return CallAction(self, *args, **kwargs)
352
353    def getattr(self, attrname):
354        """
355        Return a :class:`GetAttrAction` for getting :var:`self`\s attribute
356        named :var:`attrname`.
357        """
358        return GetAttrAction(self, attrname)
359
360    def callattr(self, attrname, *args, **kwargs):
361        """
362        Return a :class:`CallAttrAction` for calling :var:`self`\s attribute
363        named :var:`attrname` with positional arguments from :var:`args` and
364        keyword arguments from :var:`kwargs`.
365        """
366        return CallAttrAction(self, attrname, *args, **kwargs)
367
368    def __repr__(self):
369        def format(arg):
370            if isinstance(arg, Action):
371                return " from {}.{}".format(arg.__class__.__module__, arg.__class__.__name__)
372            elif isinstance(arg, tuple):
373                return "=(?)"
374            elif isinstance(arg, list):
375                return "=[?]"
376            elif isinstance(arg, dict):
377                return "={?}"
378            else:
379                return "={!r}".format(arg)
380
381        output = ["arg {}{}".format(i, format(arg)) for (i, arg) in enumerate(self.getargs())]
382        for (argname, arg) in self.getkwargs().items():
383            output.append("arg {}{}".format(argname, format(arg)))
384
385        if output:
386            output = " with {}".format(", ".join(output))
387        else:
388            output = ""
389        return "<{}.{} object{} at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, output, id(self))
390
391    @misc.notimplemented
392    def __iter__(self):
393        """
394        Return an iterator over the input actions of :var:`self`.
395        """
396
397    def iterallinputs(self):
398        """
399        Return an iterator over all input actions of :var:`self`
400        (i.e. recursively).
401        """
402        for input in self:
403            yield input
404            yield from input.iterallinputs()
405
406    def findpaths(self, input):
407        """
408        Find dependency paths leading from :var:`self` to the other action
409        :var:`input`. I.e. if :var:`self` depends directly or indirectly on
410        :var:`input`, this generator will produce all paths ``p`` where
411        ``p[0] is self`` and ``p[-1] is input`` and ``p[i+1] in p[i]`` for all
412        ``i`` in ``range(len(p)-1)``.
413        """
414        if input is self:
415            yield [self]
416        else:
417            for myinput in self:
418                if isinstance(myinput, Action):
419                    for path in myinput.findpaths(input):
420                        yield [self] + path
421                else:
422                    if myinput == input:
423                        yield [self, myinput]
424
425
426class ObjectAction(Action):
427    """
428    An :class:`ObjectAction` returns an object.
429    """
430    def __init__(self, object=None):
431        Action.__init__(self)
432        self.object = object
433
434    @report
435    def get(self, project, since):
436        (data, self.changed) = getoutputs(project, since, self.object)
437        return data
438
439
440class TransformAction(Action):
441    """
442    A :class:`TransformAction` depends on exactly one input action and transforms
443    the input data into output data.
444    """
445    def __init__(self, input=None):
446        Action.__init__(self)
447        self.input = input
448
449    def getkey(self):
450        return self.input.getkey()
451
452    def __iter__(self):
453        yield self.input
454
455    def getkwargs(self):
456        return dict(data=self.input)
457
458
459class CollectAction(TransformAction):
460    """
461    A :class:`CollectAction` is a :class:`TransformAction` that simply outputs its
462    input data unmodified, but updates a number of other actions in the process.
463    """
464    def __init__(self, input=None, *otherinputs):
465        TransformAction.__init__(self, input)
466        self.otherinputs = list(otherinputs)
467
468    def addinputs(self, *otherinputs):
469        """
470        Register all actions in :var:`otherinputs` as additional actions that have
471        to be updated before :var:`self` is updated.
472        """
473        self.otherinputs.extend(otherinputs)
474        return self
475
476    def __iter__(self):
477        yield from TransformAction.__iter__(self)
478        yield from self.otherinputs
479
480    @report
481    def get(self, project, since):
482        # We don't need the data itself, so don't use getoutputs(), which would collect all inputs in a list.
483        havedata = False
484        changedinputs = bigbang
485        for item in self.otherinputs:
486            (data, changed) = getoutputs(project, since, item)
487            changedinputs = max(changedinputs, changed)
488            if data is not nodata: # The first real output
489                havedata = True
490        if havedata:
491            since = bigbang
492        (data, changedinput) = getoutputs(project, since, self.input)
493        self.changed = max(changedinputs, changedinput)
494        return data
495
496    def __repr__(self):
497        return "<{}.{} object at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, id(self))
498
499
500class PhonyAction(Action):
501    """
502    A :class:`PhonyAction` doesn't do anything. It may depend on any number of
503    additonal input actions which will be updated when this action gets updated.
504    If there's new data from any of these actions, a :class:`PhonyAction` will
505    return :const:`None` (and :const:`nodata` otherwise as usual).
506    """
507    def __init__(self, *inputs, **kwargs):
508        """
509        Create a :class:`PhonyAction` object. :var:`doc` describes the action and
510        is printed by the method :meth:`Project.writephonytargets`.
511        """
512        Action.__init__(self)
513        self.doc = kwargs.get("doc")
514        self.inputs = list(inputs)
515        self.buildno = None
516
517    def addinputs(self, *inputs):
518        """
519        Register all actions in :var:`inputs` as additional actions that have to
520        be updated once :var:`self` is updated.
521        """
522        self.inputs.extend(inputs)
523        return self
524
525    def __iter__(self):
526        return iter(self.inputs)
527
528    @report
529    def get(self, project, since):
530        # Caching the result object of a :class:`PhonyAction` is cheap (it's either :const:`None` or :const:`nodata`),
531        # so we always do the caching as this optimizes away the traversal of a complete subgraph
532        # for subsequent calls to :meth:`get` during the same build round
533        if self.buildno != project.buildno:
534            havedata = False
535            resultchanged = bigbang
536            # We don't need the data itself, so don't use getoutputs(), which would collect all inputs in a list.
537            for item in self.inputs:
538                (data, changed) = getoutputs(project, since, item)
539                resultchanged = max(resultchanged, changed)
540                if data is not nodata: # The first real output
541                    havedata = True
542            self.buildno = project.buildno
543            self.changed = resultchanged
544            return None if havedata else nodata
545        else:
546            return None if self.changed > since else nodata
547
548    def __repr__(self):
549        s = "<{}.{} object".format(self.__class__.__module__, self.__class__.__name__)
550        if self.key is not None:
551            s += " with key={!r}".format(self.key)
552        s += " at {:#x}>".format(id(self))
553        return s
554
555
556class FileAction(TransformAction):
557    """
558    A :class:`FileAction` is used for reading and writing files (and other
559    objects providing the appropriate interface).
560    """
561    def __init__(self, key, input=None, encoding=None, errors=None):
562        """
563        Create a :class:`FileAction` object with :var:`key` as the "filename".
564        :var:`key` must be an object that provides a method :meth:`open` for
565        opening readable and writable streams to the file. :var:`input` is the
566        data written to the file (or the action producing the data). :var:`encoding`
567        is the encoding to be used from reading/writing. If :var:`encoding` is
568        :const:`None` binary i/o will be used. :var:`errors` is the codec error
569        handling name for encoding/decoding text.
570        """
571        TransformAction.__init__(self, input)
572        self.key = url.URL(key)
573        self.encoding = encoding
574        self.errors = errors
575        self.buildno = None
576
577    def getkey(self):
578        return self.key
579
580    def getkwargs(self):
581        return dict(data=self.input, encoding=self.encoding, errors=errors)
582
583    def write(self, project, data):
584        """
585        Write :var:`data` to the file and return it.
586        """
587        project.writestep(self, "Writing ", format(len(data), ","), " ", ("bytes" if isinstance(data, bytes) else "chars"), " to ", project.strkey(self.key))
588        with contextlib.closing(self.key.open(mode="wb" if self.encoding is None else "w", encoding=self.encoding, errors=self.errors)) as file:
589            file.write(data)
590            project.fileswritten += 1
591            project.byteswritten += len(data)
592
593    def read(self, project):
594        """
595        Read the content from the file and return it.
596        """
597        project.writestep(self, "Reading ", project.strkey(self.key))
598        with contextlib.closing(self.key.open(mode="rb" if self.encoding is None else "r", encoding=self.encoding, errors=self.errors)) as file:
599            data = file.read()
600            project.filesread += 1
601            project.bytesread += len(data)
602            return data
603
604    @report
605    def get(self, project, since):
606        """
607        If a :class:`FileAction` object doesn't have an input action it reads the
608        input file and returns the content if the file has changed since
609        :var:`since` (otherwise :const:`nodata` is returned).
610
611        If a :class:`FileAction` object does have an input action and the output
612        data from this input action is newer than the file ``self.key`` the data
613        will be written to the file. Otherwise (i.e. the file is up to date) the
614        data will be read from the file.
615        """
616        if self.buildno != project.buildno: # a new build round
617            self.changed = filechanged(self.key) # Get timestamp of the file (or :const:`bigbang` if it doesn't exist)
618            self.buildno = project.buildno
619
620        if self.input is not None:
621            (data, self.changed) = getoutputs(project, self.changed, self.input)
622            if data is not nodata: # We've got new data from our input =>
623                self.write(project, data) # write new data to disk
624                self.changed = filechanged(self.key) # update timestamp
625                return data
626        else: # We have no inputs (i.e. this is a "source" file)
627            if self.changed is bigbang:
628                raise ValueError("source file {!r} doesn't exist".format(self.key))
629        if self.changed > since: # We are up to date now and newer than the output action
630            return self.read(project) # return file data (to output action or client)
631        # else fail through and return :const:`nodata`
632        return nodata
633
634    def chmod(self, mode=0o644):
635        """
636        Return a :class:`ModeAction` that will change the file permissions of
637        :var:`self` to :var:`mode`.
638        """
639        return ModeAction(self, mode)
640
641    def chown(self, user=None, group=None):
642        """
643        Return an :class:`OwnerAction` that will change the user and/or group
644        ownership of :var:`self`.
645        """
646        return OwnerAction(self, user, group)
647
648    def __repr__(self):
649        return "<{}.{} object key={!r} at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, self.key, id(self))
650
651
652class MkDirAction(TransformAction):
653    """
654    This action creates the a directory (passing through its input data).
655    """
656
657    def __init__(self, key, mode=0o777):
658        """
659        Create a :class:`MkDirAction` instance. :var:`mode` (which defaults to
660        :const:`0777`) will be used as the permission bit pattern for the new
661        directory.
662        """
663        TransformAction.__init__(self)
664        self.key = key
665        self.mode = mode
666
667    def execute(self, project, data):
668        """
669        Create the directory with the permission bits specified in the
670        constructor.
671        """
672        project.writestep(self, "Making directory ", project.strkey(self.key), " with mode ", oct(self.mode))
673        self.key.makedirs(self.mode)
674
675    def __repr__(self):
676        return "<{}.{} object with mode={:#03o} at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, self.mode, id(self))
677
678
679class PipeAction(TransformAction):
680    """
681    This action pipes the input through an external shell command.
682    """
683
684    def __init__(self, input, command):
685        """
686        Create a :class:`PipeAction` instance. :var:`command` is the shell command
687        to be executed (which must read it's input from stdin and write its output
688        to stdout).
689        """
690        TransformAction.__init__(self, input)
691        self.command = command
692
693    def getkwargs(self):
694        return dict(data=self.input, command=self.command)
695
696    def execute(self, project, data, command):
697        project.writestep(self, "Calling command ", command)
698        (stdin, stdout) = os.popen2(command)
699
700        stdin.write(data)
701        stdin.close()
702        output = stdout.read()
703        stdout.close()
704        return output
705
706    def __repr__(self):
707        return "<{}.{} object with command={!r} at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, self.command, id(self))
708
709
710class CacheAction(TransformAction):
711    """
712    A :class:`CacheAction` is a :class:`TransformAction` that passes through its
713    input data, but caches it, so that it can be reused during the same build
714    round.
715    """
716    def __init__(self, input=None):
717        TransformAction.__init__(self, input)
718        self.since = bigcrunch
719        self.data = nodata
720        self.buildno = None
721
722    @report
723    def get(self, project, since):
724        if self.buildno != project.buildno or (since < self.since and self.data is nodata): # If this is a new build round or we're asked about an earlier date and didn't return data last time
725            (self.data, self.changed) = getoutputs(project, since, self.input)
726            project.writenote(self, "Caching data")
727            self.since = since
728            self.buildno = project.buildno
729        if since < self.changed or since is bigbang:
730            project.writenote(self, "Reusing cached data")
731            return self.data
732        return nodata
733
734
735class GetAttrAction(TransformAction):
736    """
737    This action gets an attribute from its input object.
738    """
739
740    def __init__(self, input=None, attrname=None):
741        TransformAction.__init__(self, input)
742        self.attrname = attrname
743
744    def __iter__(self):
745        yield from TransformAction.__iter__(self)
746        yield self.attrname
747
748    def getkwargs(self):
749        return dict(data=self.input, attrname=self.attrname)
750
751    def execute(self, project, data, attrname):
752        project.writestep(self, "Getting attribute ", attrname)
753        return getattr(data, attrname)
754
755
756class CallAction(Action):
757    """
758    This action calls a function or any other callable object with a number of
759    arguments. Both positional and keyword arguments are supported and the
760    function and the arguments can be static objects or actions.
761    """
762    def __init__(self, func, *args, **kwargs):
763        Action.__init__(self)
764        self.func = func
765        self.args = args
766        self.kwargs = kwargs
767
768    def __iter__(self):
769        yield self.func
770        yield from self.args
771        yield from self.kwargs.values()
772
773    def getargs(self):
774        return (self.func,) + self.args
775
776    def getkwargs(self):
777        return self.kwargs
778
779    def execute(self, project, func, *args, **kwargs):
780        if args:
781            if len(args) == 1:
782                argsmsg = " with 1 arg"
783            else:
784                argsmsg = " with {} args".format(len(args))
785        else:
786            argsmsg = " without args"
787        if kwargs:
788            if len(kwargs) == 1:
789                kwargsmsg = " and keyword {}".format(", ".join(kwargs))
790            else:
791                kwargsmsg = " and keywords {}".format(", ".join(kwargs))
792        else:
793            kwargsmsg = ""
794        project.writestep(self, "Calling {!r}".format(func), argsmsg, kwargsmsg)
795        return func(*args, **kwargs)
796
797
798class CallAttrAction(Action):
799    """
800    This action calls an attribute of an object with a number of arguments. Both
801    positional and keyword arguments are supported and the object, the attribute
802    name and the arguments can be static objects or actions.
803    """
804    def __init__(self, obj, attrname, *args, **kwargs):
805        Action.__init__(self)
806        self.obj = obj
807        self.attrname = attrname
808        self.args = args
809        self.kwargs = kwargs
810
811    def __iter__(self):
812        yield self.obj
813        yield self.attrname
814        yield from self.args
815        yield from self.kwargs.values()
816
817    def getargs(self):
818        return (self.obj, self.attrname) + self.args
819
820    def getkwargs(self):
821        return self.kwargs
822
823    def execute(self, project, obj, attrname, *args, **kwargs):
824        func = getattr(obj, attrname)
825        project.writestep(self, "Calling {!r}".format(func))
826        return func(*args, **kwargs)
827
828
829class CommandAction(TransformAction):
830    """
831    This action executes a system command (via :func:`os.system`) and passes
832    through the input data.
833    """
834
835    def __init__(self, command, input=None):
836        """
837        Create a new :class:`CommandAction` object. :var:`command` is the command
838        that will executed when :meth:`execute` is called.
839        """
840        TransformAction.__init__(self, input)
841        self.command = command
842
843    def execute(self, project, data):
844        project.writestep(self, "Executing command ", self.command)
845        os.system(self.command)
846
847    def __repr__(self):
848        return "<{}.{} object command={!r} at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, self.command, id(self))
849
850
851class ModeAction(TransformAction):
852    """
853    :class:`ModeAction` changes file permissions and passes through the input data.
854    """
855
856    def __init__(self, input=None, mode=0o644):
857        """
858        Create an :class:`ModeAction` object. :var:`mode` (which defaults to
859        :const:`0644`) will be use as the permission bit pattern.
860        """
861        TransformAction.__init__(self, input)
862        self.mode = mode
863
864    def __iter__(self):
865        yield from TransformAction.__iter__(self)
866        yield self.mode
867
868    def getkwargs(self):
869        return dict(data=self.input, mode=self.mode)
870
871    def execute(self, project, data, mode):
872        """
873        Change the permission bits of the file ``self.getkey()``.
874        """
875        key = self.getkey()
876        project.writestep(self, "Changing mode of ", project.strkey(key), " to {:#03o}".format(mode))
877        key.chmod(mode)
878        return data
879
880
881class OwnerAction(TransformAction):
882    """
883    :class:`OwnerAction` changes the user and/or group ownership of a file and
884    passes through the input data.
885    """
886
887    def __init__(self, input=None, user=None, group=None):
888        """
889        Create a new :class:`OwnerAction` object. :var:`user` can either be a
890        numerical user id or a user name or :const:`None`. If it is :const:`None`
891        no user ownership will be changed. The same applies to :var:`group`.
892        """
893        TransformAction.__init__(self, input)
894        self.id = id
895        self.user = user
896        self.group = group
897
898    def __iter__(self):
899        yield from TransformAction.__iter__(self)
900        yield self.user
901        yield self.group
902
903    def getkwargs(self):
904        return dict(data=self.input, user=self.user, group=self.group)
905
906    def execute(self, project, data, user, group):
907        """
908        Change the ownership of the file ``self.getkey()``.
909        """
910        key = self.getkey()
911        project.writestep(self, "Changing owner of ", project.strkey(key), " to ", user, " and group to ", group)
912        key.chown(user, group)
913        return data
914
915
916class ModuleAction(TransformAction):
917    """
918    This action will turn the input string into a Python module.
919    """
920    def __init__(self, input=None):
921        """
922        Create an :class:`ModuleAction`.
923
924        This object must have an input action (which might be a :class:`FileAction`
925        that creates the source file).
926        """
927        TransformAction.__init__(self, input)
928        self.inputs = []
929        self.changed = bigbang
930        self.data = nodata
931        self.buildno = None
932
933    def addinputs(self, *inputs):
934        """
935        Register all actions in :var:`inputs` as modules used by this module.
936        These actions must be :class:`ModuleAction` objects too.
937
938        Normally it isn't neccessary to call the method directly. Instead
939        fetch the required module inside your module like this::
940
941            from ll import make
942
943            mymodule = make.currentproject.get("mymodule.py")
944
945        This will record your module as depending on :mod:`mymodule`, so if
946        :mod:`mymodule` changes, your module will be reloaded too. For this to
947        work you need to have an :class:`ModuleAction` added to the project with
948        the key ``"mymodule.py"``.
949        """
950        self.inputs.extend(inputs)
951        return self
952
953    def __iter__(self):
954        yield from TransformAction.__iter__(self)
955        yield from self.inputs
956
957    def execute(self, project, data):
958        key = self.getkey()
959        project.writestep(self, "Importing module as ", project.strkey(key))
960
961        if key is None:
962            filename = name = "<string>"
963        elif isinstance(key, url.URL):
964            try:
965                filename = key.local()
966            except ValueError: # is not local
967                filename = str(key)
968            name = key.withoutext().file.encode("ascii", "ignore").decode("ascii")
969        else:
970            filename = name = str(key)
971
972        del self.inputs[:] # The module will be reloaded => drop all dependencies (they will be rebuilt during import)
973
974        try:
975            project.importstack.append(self)
976            mod = misc.module(data, filename, name)
977        finally:
978            project.importstack.pop()
979        return mod
980
981    @report
982    def get(self, project, since):
983        # Is this module required by another?
984        if project.importstack:
985            if self not in project.importstack[-1].inputs:
986                project.importstack[-1].addinputs(self) # Append to inputs of other module
987
988        # Is this a new build round?
989        if self.buildno != project.buildno:
990            (data, changed) = getoutputs(project, self.changed, self.input) # Get the source code
991            if data is not nodata or self.data is nodata: # The file itself has changed or this is the first call
992                needimport = True
993            else: # Only check the required inputs, if ``self.input`` has *not* changed
994                (data2, changed2) = getoutputs(project, self.changed, self.inputs)
995                needimport = data2 is not nodata
996
997            if needimport:
998                if data is nodata:
999                    (data, changed) = getoutputs(project, bigbang, self.input) # We *really* need the source
1000                self.data = self.execute(project, data) # This will (re)create dependencies
1001                gc.collect() # Make sure classes from the previous module (which have cycles via the :attr:`__mro__`) are gone
1002                # Timestamp of import is the timestamp of the newest module file
1003                self.changed = changed
1004                if self.inputs:
1005                    changed = max(changed, max(input.changed for input in self.inputs))
1006                self.changed = changed
1007            self.buildno = project.buildno
1008            if self.changed > since:
1009                return self.data
1010        # Are we newer then the specified date?
1011        elif self.changed > since:
1012            key = self.getkey()
1013            project.writenote(self, "Reusing cached module ", project.strkey(key))
1014            return self.data
1015        return nodata
1016
1017    def __repr__(self):
1018        return "<{}.{} object key={!r} at {:#x}>".format(self.__class__.__module__, self.__class__.__name__, self.getkey(), id(self))
1019
1020
1021class FOPAction(TransformAction):
1022    """
1023    This action transforms an XML string (containing XSL-FO) into PDF. For it
1024    to work `Apache FOP`__ is required. The command line is hardcoded but it's
1025    simple to overwrite the class attribute :attr:`command` in a subclass.
1026
1027    __ http://xmlgraphics.apache.org/fop/
1028    """
1029    command = "/usr/local/src/fop-0.20.5/fop.sh -q -c /usr/local/src/fop-0.20.5/conf/userconfig.xml -fo {src} -pdf {dst}"
1030
1031    def execute(self, project, data):
1032        project.writestep(self, "FOPping input")
1033        (infd, inname) = tempfile.mkstemp(suffix=".fo")
1034        (outfd, outname) = tempfile.mkstemp(suffix=".pdf")
1035        try:
1036            infile = os.fdopen(infd, "wb")
1037            os.fdopen(outfd).close()
1038            infile.write(data)
1039            infile.close()
1040            os.system(self.command.format(src=inname, dst=outname))
1041            data = open(outname, "rb").read()
1042        finally:
1043            os.remove(inname)
1044            os.remove(outname)
1045        return data
1046
1047
1048class AlwaysAction(Action):
1049    """
1050    This action always returns :const:`None` as new data.
1051    """
1052    def __init__(self):
1053        Action.__init__(self)
1054        self.changed = bigbang
1055
1056    def __iter__(self):
1057        yield None
1058
1059    @report
1060    def get(self, project, since):
1061        project.writestep(self, "Returning None")
1062        return None
1063alwaysaction = AlwaysAction() # this action can be reused as it has no inputs
1064
1065
1066class NeverAction(Action):
1067    """
1068    This action never returns new data.
1069    """
1070    def __iter__(self):
1071        yield None
1072
1073    @report
1074    def get(self, project, since):
1075        return nodata
1076neveraction = NeverAction() # this action can be reused as it has no inputs
1077
1078
1079###
1080### Colors for output
1081###
1082
1083s4indent = astyle.Style.fromenv("LL_MAKE_REPRANSI_INDENT", "black:black:bold")
1084s4key = astyle.Style.fromenv("LL_MAKE_REPRANSI_KEY", "yellow:black")
1085s4action = astyle.Style.fromenv("LL_MAKE_REPRANSI_ACTION", "green:black")
1086s4time = astyle.Style.fromenv("LL_MAKE_REPRANSI_TIME", "magenta:black")
1087s4data = astyle.Style.fromenv("LL_MAKE_REPRANSI_DATA", "cyan:black")
1088s4size = astyle.Style.fromenv("LL_MAKE_REPRANSI_SIZE", "magenta:black")
1089s4counter = astyle.Style.fromenv("LL_MAKE_REPRANSI_COUNTER", "red:black:bold")
1090s4error = astyle.Style.fromenv("LL_MAKE_REPRANSI_ERROR", "red:black:bold")
1091
1092
1093###
1094### The project class
1095###
1096
1097class Project(dict):
1098    """
1099    A :class:`Project` collects all :class:`Action` objects from a project. It
1100    is responsible for initiating the build process and for generating a report
1101    about the progress of the build process.
1102    """
1103
1104    # Will be used in log messages and notifications
1105    name = "ll.make"
1106
1107    def __init__(self):
1108        super().__init__()
1109        self.actionscalled = 0
1110        self.actionsfailed = 0
1111        self.stepsexecuted = 0
1112        self.bytesread = 0
1113        self.filesread = 0
1114        self.byteswritten = 0
1115        self.fileswritten = 0
1116        self.starttime = None
1117        self.ignoreerrors = False
1118        self.here = None # cache the current directory during builds (used for shortening URLs)
1119        self.home = None # cache the home directory during builds (used for shortening URLs)
1120        self.stack = [] # keep track of the recursion during calls to :meth:`Action.get`
1121        self.importstack = [] # keep track of recursive imports
1122        self.indent = os.environ.get("LL_MAKE_INDENT", "   ") # Indentation string to use for output of nested actions
1123        self.buildno = 0 # Build number; This gets incremented on each call to :meth:`build`. Can be used by actions to determine the start of a new build round
1124
1125        self.showsummary = self._getenvbool("LL_MAKE_SHOWSUMMARY", True)
1126        self.showaction = os.environ.get("LL_MAKE_SHOWACTION", "filephony")
1127        self.showstep = os.environ.get("LL_MAKE_SHOWSTEP", "all")
1128        self.shownote = os.environ.get("LL_MAKE_SHOWNOTE", "none")
1129        self.color = self._getenvbool("LL_MAKE_COLOR", True)
1130        self.showidle = self._getenvbool("LL_MAKE_SHOWIDLE", False)
1131        self.showregistration = os.environ.get("LL_MAKE_SHOWREGISTRATION", "phony")
1132        self.showtime = self._getenvbool("LL_MAKE_SHOWTIME", True)
1133        self.showtimestamps = self._getenvbool("LL_MAKE_SHOWTIMESTAMPS", False)
1134        self.showdata = self._getenvbool("LL_MAKE_SHOWDATA", False)
1135        self.notify = self._getenvbool("LL_MAKE_NOTIFY", False)
1136
1137    def __repr__(self):
1138        return "<{}.{} with {} targets at {:#x}>".format(self.__module__, self.__class__.__name__, len(self), id(self))
1139
1140    class showaction(misc.propclass):
1141        """
1142        This property specifies which actions should be reported during the build
1143        process. On setting, the value can be:
1144
1145        :const:`None` or ``"none"``
1146            No actions will be reported;
1147
1148        ``"file"``
1149            Only :class:`FileAction`\s will be reported;
1150
1151        ``"phony"``
1152            Only :class:`PhonyAction`\s will be reported;
1153
1154        ``"filephony"``
1155            Only :class:`FileAction`\s and :class:`PhonyAction`\s will be
1156            reported;
1157
1158        a class or tuple of classes
1159            Only actions that are instances of those classes will be reported.
1160        """
1161        def __get__(self):
1162            return self._showaction
1163        def __set__(self, value):
1164            if value == "none":
1165                self._showaction = None
1166            elif value == "file":
1167                self._showaction = FileAction
1168            elif value == "phony":
1169                self._showaction = PhonyAction
1170            elif value == "filephony":
1171                self._showaction = (PhonyAction, FileAction)
1172            elif value == "all":
1173                self._showaction = Action
1174            elif isinstance(value, Action):
1175                self._showaction = value
1176            elif value:
1177                self._showaction = Action
1178            else:
1179                self._showaction = None
1180
1181    class showstep(misc.propclass):
1182        """
1183        This property specifies for which actions tranformation steps should be
1184        reported during the build process. For allowed values on setting see
1185        :prop:`showaction`.
1186        """
1187        def __get__(self):
1188            return self._showstep
1189        def __set__(self, value):
1190            if value == "none":
1191                self._showstep = None
1192            elif value == "file":
1193                self._showstep = FileAction
1194            elif value == "phony":
1195                self._showstep = PhonyAction
1196            elif value == "filephony":
1197                self._showstep = (PhonyAction, FileAction)
1198            elif value == "all":
1199                self._showstep = Action
1200            elif isinstance(value, Action):
1201                self._showstep = value
1202            elif value:
1203                self._showstep = Action
1204            else:
1205                self._showstep = None
1206
1207    class shownote(misc.propclass):
1208        """
1209        This property specifies which for which actions tranformation notes
1210        (which are similar to step, but not that important, e.g. when an
1211        information that is already there gets reused) be reported during the
1212        build process. For allowed values on setting see :prop:`showaction`.
1213        """
1214        def __get__(self):
1215            return self._shownote
1216        def __set__(self, value):
1217            if value == "none":
1218                self._shownote = None
1219            elif value == "file":
1220                self._shownote = FileAction
1221            elif value == "phony":
1222                self._shownote = PhonyAction
1223            elif value == "filephony":
1224                self._shownote = (PhonyAction, FileAction)
1225            elif value == "all":
1226                self._shownote = Action
1227            elif isinstance(value, Action):
1228                self._shownote = value
1229            elif value:
1230                self._shownote = Action
1231            else:
1232                self._shownote = None
1233
1234    class showregistration(misc.propclass):
1235        """
1236        This property specifies for which actions registration (i.e. call to the
1237        :meth:`add` should be reported. For allowed values on setting see
1238        :prop:`showaction`.
1239        """
1240        def __get__(self):
1241            return self._showregistration
1242        def __set__(self, value):
1243            if value == "none":
1244                self._showregistration = None
1245            elif value == "file":
1246                self._showregistration = FileAction
1247            elif value == "phony":
1248                self._showregistration = PhonyAction
1249            elif value == "filephony":
1250                self._showregistration = (PhonyAction, FileAction)
1251            elif value == "all":
1252                self._showregistration = Action
1253            else:
1254                self._showregistration = value
1255
1256    def _getenvbool(self, name, default):
1257        return bool(int(os.environ.get(name, default)))
1258
1259    def strtimedelta(self, delta):
1260        """
1261        Return a nicely formatted and colored string for the
1262        :class:`datetime.timedelta` value :var:`delta`. :var:`delta`
1263        may also be :const:`None` in with case ``"0"`` will be returned.
1264        """
1265        if delta is None:
1266            text = "0"
1267        else:
1268            rest = delta.seconds
1269
1270            (rest, secs) = divmod(rest, 60)
1271            (rest, mins) = divmod(rest, 60)
1272            rest += delta.days*24
1273
1274            secs += delta.microseconds/1000000.
1275            if rest:
1276                text = "{:d}:{:02d}:{:06.3f}h".format(rest, mins, secs)
1277            elif mins:
1278                text = "{:02d}:{:06.3f}m".format(mins, secs)
1279            else:
1280                text = "{:.3f}s".format(secs)
1281        return s4time(text)
1282
1283    def strdatetime(self, dt):
1284        """
1285        Return a nicely formatted and colored string for the
1286        :class:`datetime.datetime` value :var:`dt`.
1287        """
1288        return s4time(dt.strftime("%Y-%m-%d %H:%M:%S.%f"))
1289
1290    def strcounter(self, counter):
1291        """
1292        Return a nicely formatted and colored string for the counter value
1293        :var:`counter`.
1294        """
1295        return s4counter("{}.".format(counter))
1296
1297    def strerror(self, text):
1298        """
1299        Return a nicely formatted and colored string for the error text
1300        :var:`text`.
1301        """
1302        return s4error(text)
1303
1304    def strkey(self, key):
1305        """
1306        Return a nicely formatted and colored string for the action key
1307        :var:`key`.
1308        """
1309        s = str(key)
1310        if isinstance(key, url.URL) and key.islocal():
1311            if self.here is None:
1312                self.here = url.here()
1313            if self.home is None:
1314                self.home = url.home()
1315            test = str(key.relative(self.here))
1316            if len(test) < len(s):
1317                s = test
1318            test = "~/{}".format(key.relative(self.home))
1319            if len(test) < len(s):
1320                s = test
1321        return s4key(s)
1322
1323    def straction(self, action):
1324        """
1325        Return a nicely formatted and colored string for the action
1326        :var:`action`.
1327        """
1328        name = action.__class__.__name__
1329        if name.endswith("Action"):
1330            name = name[:-6]
1331
1332        if action.key is not None:
1333            return s4action(name, "(", self.strkey(action.key), ")")
1334        else:
1335            return s4action(name)
1336
1337    def strdata(self, data):
1338        if data is nodata:
1339            return "nodata"
1340        elif isinstance(data, (int, float)):
1341            return s4data(repr(data))
1342        elif data is None:
1343            return s4data("None")
1344        elif isinstance(data, bytes):
1345            return s4data("bytes ({:,}b)".format(len(data)))
1346        elif isinstance(data, str):
1347            return s4data("chars ({:,}c)".format(len(data)))
1348        else:
1349            dataclass = data.__class__
1350            fmt = "{0.__module__}.{0.__name__} @ {1:#x}" if dataclass.__module__ != "__builtin__" else "{0.__name__} @ {1:#x}"
1351            text = fmt.format(dataclass, id(data))
1352            return s4data(text)
1353
1354    def __setitem__(self, key, target):
1355        """
1356        Add the action :var:`target` to :var:`self` as a target and register it
1357        under the key :var:`key`.
1358        """
1359        if isinstance(key, url.URL) and key.islocal():
1360            key = key.abs(scheme="file")
1361        if key in self:
1362            self.warn(RedefinedTargetWarning(key), 5)
1363        target.key = key
1364        super().__setitem__(key, target)
1365
1366    def add(self, target, key=None):
1367        """
1368        Add the action :var:`target` as a target to :var:`self`. If :var:`key`
1369        is not :const:`None`, :var:`target` will be registered under this key,
1370        otherwise it will be registered under its own key (i.e. ``target.key``).
1371        """
1372        if key is None: # Use the key from the target
1373            key = target.getkey()
1374
1375        self[key] = target
1376
1377        self.stepsexecuted += 1
1378        if self.showregistration is not None and isinstance(target, self.showregistration):
1379            self.writestacklevel(0, self.strcounter(self.stepsexecuted), " Registered ", self.strkey(key))
1380
1381        return target
1382
1383    def _candidates(self, key):
1384        """
1385        Return candidates for alternative forms of :var:`key`. This is a
1386        generator, so when the first suitable candidate is found, the rest of the
1387        candidates won't have to be created at all.
1388        """
1389        yield key
1390        key2 = key
1391        if isinstance(key, str):
1392            key2 = url.URL(key)
1393            yield key2
1394        if isinstance(key2, url.URL):
1395            key2 = key2.abs(scheme="file")
1396            yield key2
1397            key2 = key2.real(scheme="file")
1398            yield key2
1399
1400    def __getitem__(self, key):
1401        """
1402        Return the target with the key :var:`key`. If an key can't be found, it
1403        will be wrapped in a :class:`ll.url.URL` object and retried. If
1404        :var:`key` still can't be found a :exc:`UndefinedTargetError` will be
1405        raised.
1406        """
1407        for key2 in self._candidates(key):
1408            try:
1409                return super().__getitem__(key2)
1410            except KeyError:
1411                pass
1412        raise UndefinedTargetError(key)
1413
1414    def has_key(self, key):
1415        """
1416        Return whether the target with the key :var:`key` exists in the project.
1417        """
1418        return key in self
1419
1420    def __contains__(self, key):
1421        """
1422        Return whether the target with the key :var:`key` exists in the project.
1423        """
1424        return any(super(Project, self).__contains__(key2) for key2 in self._candidates(key))
1425
1426    def create(self):
1427        """
1428        Create all dependencies for the project. Overwrite in subclasses.
1429
1430        This method should only be called once, otherwise you'll get lots of
1431        :exc:`RedefinedTargetWarning`\s. But you can call :meth:`clear`
1432        to remove all targets before calling :meth:`create`. You can also
1433        use the method :meth:`recreate` for that.
1434        """
1435        self.stepsexecuted = 0
1436        self.starttime = datetime.datetime.utcnow()
1437        self.writeln("Creating targets...")
1438
1439    def recreate(self):
1440        """
1441        Calls :meth:`clear` and :meth:`create` to recreate all project
1442        dependencies.
1443        """
1444        self.clear()
1445        self.create()
1446
1447    def argparser(self):
1448        """
1449        Return an :mod:`argparse` parser for parsing the command line arguments.
1450        This can be overwritten in subclasses to add more arguments.
1451        """
1452
1453        def action2name(action):
1454            if action is None:
1455                return "none"
1456            elif action is Action:
1457                return "all"
1458            elif issubclass(FileAction, action) and issubclass(PhonyAction, action):
1459                return "filephony"
1460            elif issubclass(FileAction, action):
1461                return "file"
1462            elif issubclass(PhonyAction, action):
1463                return "phony"
1464            else:
1465                return "all"
1466
1467        actions = ("all", "file", "phony", "filephony", "none")
1468        p = argparse.ArgumentParser(description="build one or more targets", epilog="For more info see http://www.livinglogic.de/Python/make/")
1469        p.add_argument("targets", metavar="target", help="Target to be built", nargs="*")
1470        p.add_argument("-x", "--ignoreerrors", dest="ignoreerrors", help="Ignore errors? (default: %(default)s)", action=misc.FlagAction, default=self.ignoreerrors)
1471        p.add_argument("-c", "--color", dest="color", help="Use colored output? (default: %(default)s)", action=misc.FlagAction, default=self.color)
1472        p.add_argument("-y", "--notify", dest="notify", help="Issue notification after the build? (default: %(default)s)", action=misc.FlagAction, default=self.notify)
1473        p.add_argument("-a", "--showaction", dest="showaction", help="Show actions? (default: %(default)s)", choices=actions, default=action2name(self.showaction))
1474        p.add_argument("-s", "--showstep", dest="showstep", help="Show steps? (default: %(default)s)", choices=actions, default=action2name(self.showstep))
1475        p.add_argument("-n", "--shownote", dest="shownote", help="Show notes? (default: %(default)s)", choices=actions, default=action2name(self.shownote))
1476        p.add_argument("-r", "--showregistration", dest="showregistration", help="Show registration? (default: %(default)s)", choices=actions, default=action2name(self.showregistration))
1477        p.add_argument("-i", "--showidle", dest="showidle", help="Show actions that didn't produce data? (default: %(default)s)", action=misc.FlagAction, default=self.showidle)
1478        p.add_argument("-d", "--showdata", dest="showdata", help="Show data? (default: %(default)s)", action=misc.FlagAction, default=self.showdata)
1479        return p
1480
1481    def parseargs(self, args=None):
1482        """
1483        Use the parser returned by :meth:`argparser` to parse the argument
1484        sequence :var:`args`, modify :var:`self` accordingly and return
1485        the result of the parsers :meth:`parse_args` call.
1486        """
1487        p = self.argparser()
1488        args = p.parse_args(args)
1489        self.ignoreerrors = args.ignoreerrors
1490        self.color = args.color
1491        self.notify = args.notify
1492        self.showaction = args.showaction
1493        self.showstep = args.showstep
1494        self.shownote = args.shownote
1495        self.showregistration = args.showregistration
1496        self.showidle = args.showidle
1497        self.showdata = args.showdata
1498        return args
1499
1500    def _get(self, target, since):
1501        """
1502        :var:`target` must be an action registered in :var:`self` (or the id of
1503        one). For this target the :meth:`Action.get` will be called with
1504        :var:`since` as the argument.
1505        """
1506        global currentproject
1507
1508        if not isinstance(target, Action):
1509            target = self[target]
1510
1511        oldproject = currentproject
1512        try:
1513            currentproject = self
1514            data = target.get(self, since)
1515        finally:
1516            currentproject = oldproject
1517        return data
1518
1519    def get(self, target):
1520        """
1521        Get up-to-date output data from the target :var:`target` (which must be
1522        an action registered with :var:`self` (or the id of one). During the call
1523        the global variable ``currentproject`` will be set to :var:`self`.
1524        """
1525        return self._get(target, bigbang)
1526
1527    def build(self, *targets):
1528        """
1529        Rebuild all targets in :var:`targets`. Items in :var:`targets` must be
1530        actions registered with :var:`self` (or their ids).
1531        """
1532        global currentproject
1533
1534        self.starttime = datetime.datetime.utcnow()
1535
1536        format = "{:,}".format
1537
1538        with url.Context():
1539            self.stack = []
1540            self.importstack = []
1541            self.actionscalled = 0
1542            self.actionsfailed = 0
1543            self.stepsexecuted = 0
1544            self.bytesread = 0
1545            self.filesread = 0
1546            self.byteswritten = 0
1547            self.fileswritten = 0
1548
1549            self.buildno += 1 # increment build number so that actions that stored the old one can detect a new build round
1550
1551            success = False
1552            try:
1553                if self.notify:
1554                    self.notifystart()
1555                for target in targets:
1556                    data = self._get(target, bigcrunch)
1557
1558                if self.showsummary:
1559                    args = []
1560                    self.write(
1561                        "built project ",
1562                        s4action(self.name),
1563                        ": ",
1564                        s4data(format(len(self))),
1565                        " registered targets; ",
1566                        s4data(format(self.actionscalled)),
1567                        " actions called; ",
1568                        s4data(format(self.stepsexecuted)),
1569                        " steps executed; ",
1570                        s4data(format(self.filesread)),
1571                        " files/",
1572                        s4data(format(self.bytesread)),
1573                        " bytes read; ",
1574                        s4data(format(self.fileswritten)),
1575                        " files/",
1576                        s4data(format(self.byteswritten)),
1577                        " bytes written; ",
1578                        s4data(format(self.actionsfailed)),
1579                        " actions failed",
1580                    )
1581                    if self.showtime:
1582                        self.write(" [t+", self.strtimedelta(datetime.datetime.utcnow()-self.starttime), "]")
1583                    self.writeln()
1584                success = True
1585            finally:
1586                if self.notify:
1587                    self.notifyfinish(datetime.datetime.utcnow()-self.starttime, success)
1588
1589    def buildwithargs(self, args=None):
1590        """
1591        For calling make scripts from the command line. :var:`args` defaults to
1592        ``sys.argv``. Any positional arguments in the command line will be treated
1593        as target ids. If there are no positional arguments, a list of all
1594        registered :class:`PhonyAction` objects will be output.
1595        """
1596        args = self.parseargs(args)
1597
1598        if args.targets:
1599            self.build(*args.targets)
1600        else:
1601            self.writeln("Available phony targets are:")
1602            self.writephonytargets()
1603
1604    def write(self, *texts):
1605        """
1606        All screen output is done through this method. This makes it possible
1607        to redirect the output (e.g. to logfiles) in subclasses.
1608        """
1609        astyle.stderr.write(*texts)
1610
1611    def writeln(self, *texts):
1612        """
1613        All screen output is done through this method. This makes it possible to
1614        redirect the output (e.g. to logfiles) in subclasses.
1615        """
1616        astyle.stderr.writeln(*texts)
1617        astyle.stderr.flush()
1618
1619    def writeerror(self, *texts):
1620        """
1621        Output an error.
1622        """
1623        self.write(*texts)
1624
1625    def notifystart(self):
1626        cmd = [
1627            "/Applications/terminal-notifier.app/Contents/MacOS/terminal-notifier",
1628            "-remove",
1629            misc.sysinfo.scriptname,
1630        ]
1631
1632        import subprocess
1633        with open("/dev/null", "wb") as f:
1634            status = subprocess.call(cmd, stdout=f)
1635
1636    def notifyfinish(self, duration, success):
1637        msgs = []
1638        if self.stepsexecuted:
1639            msgs.append("{:,} steps".format(self.stepsexecuted))
1640        if self.fileswritten:
1641            msgs.append("{:,} files".format(self.fileswritten))
1642        if self.byteswritten:
1643            msgs.append("{:,} bytes".format(self.byteswritten))
1644        if not msgs:
1645            msgs.append("nothing to do")
1646
1647        cmd = [
1648            "/Applications/terminal-notifier.app/Contents/MacOS/terminal-notifier",
1649            "-title",
1650            self.name,
1651            "-subtitle",
1652            "{} after {}".format("finished" if success else "failed", self.strtimedelta(duration)),
1653            "-message",
1654            "; ".join(msgs),
1655            "-group",
1656            misc.sysinfo.scriptname,
1657        ]
1658
1659        import subprocess
1660        with open("/dev/null", "wb") as f:
1661            status = subprocess.call(cmd, stdout=f)
1662
1663    def warn(self, warning, stacklevel):
1664        """
1665        Issue a warning through the Python warnings framework
1666        """
1667        warnings.warn(warning, stacklevel=stacklevel)
1668
1669    def writestacklevel(self, level, *texts):
1670        """
1671        Output :var:`texts` indented :var:`level` levels.
1672        """
1673        self.write(s4indent(level*self.indent), *texts)
1674        if self.showtime and self.starttime is not None:
1675            self.write(" [t+", self.strtimedelta(datetime.datetime.utcnow() - self.starttime), "]")
1676        self.writeln()
1677
1678    def writestack(self, *texts):
1679        """
1680        Output :var:`texts` indented properly for the current nesting of
1681        action execution.
1682        """
1683        count = misc.count(level for level in self.stack if level.reportable)
1684        self.writestacklevel(count, *texts)
1685
1686    def _writependinglevels(self):
1687        for (i, level) in enumerate(level for level in self.stack if level.reportable):
1688            if not level.reported:
1689                args = ["Started  ", self.straction(level.action)]
1690                if self.showtimestamps:
1691                    args.append(" (since ")
1692                    args.append(self.strdatetime(level.since))
1693                    args.append(")")
1694                self.writestacklevel(i, *args)
1695                level.reported = True
1696
1697    def writestep(self, action, *texts):
1698        """
1699        Output :var:`texts` as the description of the data transformation
1700        done by the action :var:`arction`.
1701        """
1702        self.stepsexecuted += 1
1703        if self.showstep is not None and isinstance(action, self.showstep):
1704            if not self.showidle:
1705                self._writependinglevels()
1706            self.writestack(self.strcounter(self.stepsexecuted), " ", *texts)
1707
1708    def writenote(self, action, *texts):
1709        """
1710        Output :var:`texts` as the note for the data transformation done by
1711        the action :var:`action`.
1712        """
1713        self.stepsexecuted += 1
1714        if self.shownote is not None and isinstance(action, self.shownote):
1715            if not self.showidle:
1716                self._writependinglevels()
1717            self.writestack(self.strcounter(self.stepsexecuted), " ", *texts)
1718
1719    def writecreatedone(self):
1720        """
1721        Can be called at the end of an overwritten :meth:`create` to report
1722        the number of registered targets.
1723        """
1724        self.writestacklevel(0, "done: ", s4data(str(len(self))), " registered targets")
1725
1726    def writephonytargets(self):
1727        """
1728        Show a list of all :class:`PhonyAction` objects in the project and
1729        their documentation.
1730        """
1731        phonies = []
1732        maxlen = 0
1733        for key in self:
1734            if isinstance(key, str):
1735                maxlen = max(maxlen, len(key))
1736                phonies.append(self[key])
1737        phonies.sort(key=operator.attrgetter("key"))
1738        for phony in phonies:
1739            text = astyle.Text(self.straction(phony))
1740            if phony.doc:
1741                text.append(" ", s4indent("."*(maxlen+3-len(phony.key))), " ", phony.doc)
1742            self.writeln(text)
1743
1744    def findpaths(self, target, source):
1745        """
1746        Find dependency paths leading from the action :var:`target` to the action
1747        :var:`source`.
1748        """
1749        return target.findpaths(source)
1750
1751
1752# This will be set to the project in :meth:`build` and :meth:`get`
1753currentproject = None
Note: See TracBrowser for help on using the browser.