code:
#!/usr/bin/env python3
# Copyright (c) 2002-2008 ActiveState Software Inc.
# License: MIT License (http://www.opensource.org/licenses/mit-license.php)
# Adapted for inary with python3 by Zaryob

"""
    Preprocess a file.

    Command Line Usage:
        preprocess [<options>...] <infile>

    Options:
        -h, --help      Print this help and exit.
        -V, --version   Print the version info and exit.
        -v, --verbose   Give verbose output for errors.

        -o <outfile>    Write output to the given file instead of to stdout.
        -f, --force     Overwrite given output file. (Otherwise an IOError
                        will be raised if <outfile> already exists.
        -D <define>     Define a variable for preprocessing. <define>
                        can simply be a variable name (in which case it
                        will be true) or it can be of the form
                        <var>=<val>. An attempt will be made to convert
                        <val> to an integer so "-D FOO=0" will create a
                        false value.
        -I <dir>        Add an directory to the include path for
                        #include directives.

        -k, --keep-lines    Emit empty lines for preprocessor statement
                        lines and skipped output lines. This allows line
                        numbers to stay constant.
        -s, --substitute    Substitute defines into emitted lines. By
                        default substitution is NOT done because it
                        currently will substitute into program strings.
        -c, --content-types-path <path>
                        Specify a path to a content.types file to assist
                        with filetype determination. See the
                        `_gDefaultContentTypes` string in this file for
                        details on its format.

    Module Usage:
        from preprocess import preprocess
        preprocess(infile, outfile=sys.stdout, defines={}, force=0,
                   keepLines=0, includePath=[], substitute=0,
                   contentType=None)

    The <infile> can be marked up with special preprocessor statement lines
    of the form:
        <comment-prefix> <preprocessor-statement> <comment-suffix>
    where the <comment-prefix/suffix> are the native comment delimiters for
    that file type.

    Examples
    --------

    HTML (*.htm, *.html) or XML (*.xml, *.kpf, *.xul) files:

        <!-- #if FOO -->
        ...
        <!-- #endif -->

    Python (*.py), Perl (*.pl), Tcl (*.tcl), Ruby (*.rb), Bash (*.sh),
    or make ([Mm]akefile*) files:

        # #if defined('FAV_COLOR') and FAV_COLOR == "blue"
        ...
        # #elif FAV_COLOR == "red"
        ...
        # #else
        ...
        # #endif

    C (*.c, *.h), C++ (*.cpp, *.cxx, *.cc, *.h, *.hpp, *.hxx, *.hh),
    Java (*.java), PHP (*.php) or C# (*.cs) files:

        // #define FAV_COLOR 'blue'
        ...
        /* #ifndef FAV_COLOR */
        ...
        // #endif

    Fortran 77 (*.f) or 90/95 (*.f90) files:

        C     #if COEFF == 'var'
              ...
        C     #endif

    And other languages.

    Preprocessor Syntax
    -------------------

    - Valid statements:
        #define <var> [<value>]
        #undef <var>
        #ifdef <var>
        #ifndef <var>
        #if <expr>
        #elif <expr>
        #else
        #endif
        #error <error string>
        #include "<file>"
        #include <var>
      where <expr> is any valid Python expression.
    - The expression after #if/elif may be a Python statement. It is an
      error to refer to a variable that has not been defined by a -D
      option or by an in-content #define.
    - Special built-in methods for expressions:
        defined(varName)    Return true if given variable is defined.

    Tips
    ----

    A suggested file naming convention is to let input files to
    preprocess be of the form <basename>.p.<ext> and direct the output
    of preprocess to <basename>.<ext>, e.g.:
        preprocess -o foo.py foo.p.py
    The advantage is that other tools (esp. editors) will still
    recognize the unpreprocessed file as the original language.
"""

__version_info__ = (1, 1, 0)
__version__ = '.'.join(map(str, __version_info__))

import getopt
import os
import re
import sys

# ---- exceptions

class PreprocessError(Exception):
    def __init__(self, errmsg, file=None, lineno=None, line=None):
        self.errmsg = str(errmsg)
        self.file = file
        self.lineno = lineno
        self.line = line
        Exception.__init__(self, errmsg, file, lineno, line)

    def __str__(self):
        s = ""
        if self.file is not None:
            s += self.file + ":"
        if self.lineno is not None:
            s += str(self.lineno) + ":"
        if self.file is not None or self.lineno is not None:
            s += " "
        s += self.errmsg
        # if self.line is not None:
        #    s += ": " + self.line
        return s

# ---- global data

# Comment delimiter info.
#   A mapping of content type to a list of 2-tuples defining the line
#   prefix and suffix for a comment. Each prefix or suffix can either
#   be a string (in which case it is transformed into a pattern allowing
#   whitespace on either side) or a compiled regex.
_commentGroups = {
    "Python": [('#', '')],
    "Perl": [('#', '')],
    "PHP": [('/*', '*/'), ('//', ''), ('#', '')],
    "Ruby": [('#', '')],
    "Tcl": [('#', '')],
    "Shell": [('#', '')],
    # Allowing for CSS and JavaScript comments in XML/HTML.
    "XML": [('<!--', '-->'), ('/*', '*/'), ('//', '')],
    "HTML": [('<!--', '-->'), ('/*', '*/'), ('//', '')],
    "Makefile": [('#', '')],
    "JavaScript": [('/*', '*/'), ('//', '')],
    "CSS": [('/*', '*/')],
    "C": [('/*', '*/')],
    "C++": [('/*', '*/'), ('//', '')],
    "Java": [('/*', '*/'), ('//', '')],
    "C#": [('/*', '*/'), ('//', '')],
    "IDL": [('/*', '*/'), ('//', '')],
    "Text": [('#', '')],
    "Fortran": [(re.compile(r'^[a-zA-Z*$]\s*'), ''), ('!', '')],
    "TeX": [('%', '')],
}

# ---- internal logging facility

class _Logger:
    DEBUG, INFO, WARN, ERROR, CRITICAL = list(range(5))

    def __init__(self, name, level=None, streamOrFileName=sys.stderr):
        self._name = name
        if level is None:
            self.level = self.WARN
        else:
            self.level = level
        if type(streamOrFileName) == bytes:
            self.stream = open(streamOrFileName, 'w')
            self._opennedStream = 1
        else:
            self.stream = streamOrFileName
            self._opennedStream = 0

    def __del__(self):
        if self._opennedStream:
            self.stream.close()

    def getLevel(self):
        return self.level

    def setLevel(self, level):
        self.level = level

    def _getLevelName(self, level):
        levelNameMap = {
            self.DEBUG: "DEBUG",
            self.INFO: "INFO",
            self.WARN: "WARN",
            self.ERROR: "ERROR",
            self.CRITICAL: "CRITICAL",
        }
        return levelNameMap[level]

    def isEnabled(self, level):
        return level >= self.level

    def isDebugEnabled(self):
        return self.isEnabled(self.DEBUG)

    def isInfoEnabled(self):
        return self.isEnabled(self.INFO)

    def isWarnEnabled(self):
        return self.isEnabled(self.WARN)

    def isErrorEnabled(self):
        return self.isEnabled(self.ERROR)

    def isFatalEnabled(self):
        return self.isEnabled(self.FATAL)

    def log(self, level, msg, *args):
        if level < self.level:
            return
        message = "%s: %s: " % (self._name, self._getLevelName(level).lower())
        message = message + (msg % args) + "\n"
        self.stream.write(message)
        self.stream.flush()

    def debug(self, msg, *args):
        self.log(self.DEBUG, msg, *args)

    def info(self, msg, *args):
        self.log(self.INFO, msg, *args)

    def warn(self, msg, *args):
        self.log(self.WARN, msg, *args)

    def error(self, msg, *args):
        self.log(self.ERROR, msg, *args)

    def fatal(self, msg, *args):
        self.log(self.CRITICAL, msg, *args)

log = _Logger("preprocess", _Logger.WARN)

# ---- internal support stuff

def _evaluate(expr, defines):
    """Evaluate the given expression string with the given context.

    WARNING: This runs eval() on a user string. This is unsafe.
    """
    # interpolated = _interpolate(s, defines)
    try:
        rv = eval(expr, {'defined': lambda v: v in defines}, defines)
    except Exception as ex:
        msg = str(ex)
        if msg.startswith("name '") and msg.endswith("' is not defined"):
            # A common error (at least this is presumed:) is to have
            #   defined(FOO)   instead of   defined('FOO')
            # We should give a little as to what might be wrong.
            # msg == "name 'FOO' is not defined"  -->  varName == "FOO"
            varName = msg[len("name '"):-len("' is not defined")]
            if expr.find("defined(%s)" % varName) != -1:
                # "defined(FOO)" in expr instead of "defined('FOO')"
                msg += " (perhaps you want \"defined('%s')\" instead of " \
                       "\"defined(%s)\")" % (varName, varName)
        elif msg.startswith("invalid syntax"):
            msg = "invalid syntax: '%s'" % expr
        raise PreprocessError(msg, defines['__FILE__'], defines['__LINE__'])
    log.debug("evaluate %r -> %s (defines=%r)", expr, rv, defines)
    return rv

# ---- module API

def preprocess(infile, outfile=sys.stdout, defines=None,
               force=0, keepLines=0, includePath=None, substitute=0,
               contentType=None, contentTypesRegistry=None,
               __preprocessedFiles=None):
    """Preprocess the given file.

    "infile" is the input path.
    "outfile" is the output path or stream (default is sys.stdout).
    "defines" is a dictionary of defined variables that will be
        understood in preprocessor statements. Keys must be strings and,
        currently, only the truth value of any key's value matters.
    "force" will overwrite the given outfile if it already exists. Otherwise
        an IOError will be raise if the outfile already exists.
    "keepLines" will cause blank lines to be emitted for preprocessor lines
        and content lines that would otherwise be skipped.
    "includePath" is a list of directories to search for given #include
        directives. The directory of the file being processed is presumed.
    "substitute", if true, will allow substitution of defines into emitted
        lines. (NOTE: This substitution will happen within program strings
        as well. This may not be what you expect.)
    "contentType" can be used to specify the content type of the input
        file. It not given, it will be guessed.
    "contentTypesRegistry" is an instance of ContentTypesRegistry. If not specified
        a default registry will be created.
    "__preprocessedFiles" (for internal use only) is used to ensure files
        are not recusively preprocessed.

    Returns the modified dictionary of defines or raises PreprocessError if
    there was some problem.
    """
    if includePath is None:
        includePath = []

    if defines is None:
        defines = {}

    if __preprocessedFiles is None:
        __preprocessedFiles = []

    log.info("preprocess(infile=%r, outfile=%r, defines=%r, force=%r, "
             "keepLines=%r, includePath=%r, contentType=%r, "
             "__preprocessedFiles=%r)", infile, outfile, defines, force,
             keepLines, includePath, contentType, __preprocessedFiles)
    absInfile = os.path.normpath(os.path.abspath(infile))
    if absInfile in __preprocessedFiles:
        raise PreprocessError("detected recursive #include of '%s'" \
                              % infile)
    __preprocessedFiles.append(os.path.abspath(infile))

    # Determine the content type and comment info for the input file.
    if contentType is None:
        registry = contentTypesRegistry or getDefaultContentTypesRegistry()
        contentType = registry.getContentType(infile)
        if contentType is None:
            contentType = "Text"
            log.warn("defaulting content type for '%s' to '%s'",
                     infile, contentType)
    try:
        cgs = _commentGroups[contentType]
    except KeyError:
        raise PreprocessError("don't know comment delimiters for content " \
                              "type '%s' (file '%s')" \
                              % (contentType, infile))

    # Generate statement parsing regexes. Basic format:
    #       <comment-prefix> <preprocessor-stmt> <comment-suffix>
    #  Examples:
    #       <!-- #if foo -->
    #       ...
    #       <!-- #endif -->
    #
    #       # #if BAR
    #       ...
    #       # #else
    #       ...
    #       # #endif
    stmts = ['#\s*(?P<op>if|elif|ifdef|ifndef)\s+(?P<expr>.*?)',
             '#\s*(?P<op>else|endif)',
             '#\s*(?P<op>error)\s+(?P<error>.*?)',
             '#\s*(?P<op>define)\s+(?P<var>[^\s]*?)(\s+(?P<val>.+?))?',
             '#\s*(?P<op>undef)\s+(?P<var>[^\s]*?)',
             '#\s*(?P<op>include)\s+"(?P<fname>.*?)"',
             r'#\s*(?P<op>include)\s+(?P<var>[^\s]+?)',
             ]
    patterns = []
    for stmt in stmts:
        # The comment group prefix and suffix can either be just a
        # string or a compiled regex.
        for cprefix, csuffix in cgs:
            if hasattr(cprefix, "pattern"):
                pattern = cprefix.pattern
            else:
                pattern = r"^\s*%s\s*" % re.escape(cprefix)
            pattern += stmt
            if hasattr(csuffix, "pattern"):
                pattern += csuffix.pattern
            else:
                pattern += r"\s*%s\s*$" % re.escape(csuffix)
            patterns.append(pattern)
    stmtRes = [re.compile(p) for p in patterns]

    # Process the input file.
    # (Would be helpful if I knew anything about lexing and parsing
    # simple grammars.)
    fin = open(infile)
    lines = fin.readlines()
    fin.close()
    if type(outfile) in str:
        if force and os.path.exists(outfile):
            os.chmod(outfile, 0o777)
            os.remove(outfile)
        fout = open(outfile, 'w')
    else:
        fout = outfile

    defines['__FILE__'] = infile
    SKIP, EMIT = list(range(2))  # states
    states = [(EMIT,  # a state is (<emit-or-skip-lines-in-this-section>,
               0,  # <have-emitted-in-this-if-block>,
               0)]  # <have-seen-'else'-in-this-if-block>)
    lineNum = 0
    for line in lines:
        lineNum += 1
        log.debug("line %d: %r", lineNum, line)
        defines['__LINE__'] = lineNum

        # Is this line a preprocessor stmt line?
        # XXX Could probably speed this up by optimizing common case of
        #    line NOT being a preprocessor stmt line.
        for stmtRe in stmtRes:
            match = stmtRe.match(line)
            if match:
                break
        else:
            match = None

        if match:
            op = match.group("op")
            log.debug("%r stmt (states: %r)", op, states)
            if op == "define":
                if not (states and states[-1][0] == SKIP):
                    var, val = match.group("var", "val")
                    if val is None:
                        val = None
                    else:
                        try:
                            val = eval(val, {}, {})
                        except:
                            pass
                    defines[var] = val
            elif op == "undef":
                if not (states and states[-1][0] == SKIP):
                    var = match.group("var")
                    try:
                        del defines[var]
                    except KeyError:
                        pass
            elif op == "include":
                if not (states and states[-1][0] == SKIP):
                    if "var" in match.groupdict():
                        # This is the second include form: #include VAR
                        var = match.group("var")
                        f = defines[var]
                    else:
                        # This is the first include form: #include "path"
                        f = match.group("fname")

                    for d in [os.path.dirname(infile)] + includePath:
                        fname = os.path.normpath(os.path.join(d, f))
                        if os.path.exists(fname):
                            break
                    else:
                        raise PreprocessError("could not find #include'd file " \
                                              "\"%s\" on include path: %r" \
                                              % (f, includePath))
                    defines = preprocess(fname, fout, defines, force,
                                         keepLines, includePath, substitute,
                                         contentTypesRegistry=contentTypesRegistry,
                                         __preprocessedFiles=__preprocessedFiles)
            elif op in ("if", "ifdef", "ifndef"):
                if op == "if":
                    expr = match.group("expr")
                elif op == "ifdef":
                    expr = "defined('%s')" % match.group("expr")
                elif op == "ifndef":
                    expr = "not defined('%s')" % match.group("expr")
                try:
                    if states and states[-1][0] == SKIP:
                        # Were are nested in a SKIP-portion of an if-block.
                        states.append((SKIP, 0, 0))
                    elif _evaluate(expr, defines):
                        states.append((EMIT, 1, 0))
                    else:
                        states.append((SKIP, 0, 0))
                except KeyError:
                    raise PreprocessError("use of undefined variable in " \
                                          "#%s stmt" % op, defines['__FILE__'],
                                          defines['__LINE__'], line)
            elif op == "elif":
                expr = match.group("expr")
                try:
                    if states[-1][2]:  # already had #else in this if-block
                        raise PreprocessError("illegal #elif after #else in "
                                              "same #if block", defines['__FILE__'],
                                              defines['__LINE__'], line)
                    elif states[-1][1]:  # if have emitted in this if-block
                        states[-1] = (SKIP, 1, 0)
                    elif states[:-1] and states[-2][0] == SKIP:
                        # Were are nested in a SKIP-portion of an if-block.
                        states[-1] = (SKIP, 0, 0)
                    elif _evaluate(expr, defines):
                        states[-1] = (EMIT, 1, 0)
                    else:
                        states[-1] = (SKIP, 0, 0)
                except IndexError:
                    raise PreprocessError("#elif stmt without leading #if "
                                          "stmt", defines['__FILE__'],
                                          defines['__LINE__'], line)
            elif op == "else":
                try:
                    if states[-1][2]:  # already had #else in this if-block
                        raise PreprocessError("illegal #else after #else in "
                                              "same #if block", defines['__FILE__'],
                                              defines['__LINE__'], line)
                    elif states[-1][1]:  # if have emitted in this if-block
                        states[-1] = (SKIP, 1, 1)
                    elif states[:-1] and states[-2][0] == SKIP:
                        # Were are nested in a SKIP-portion of an if-block.
                        states[-1] = (SKIP, 0, 1)
                    else:
                        states[-1] = (EMIT, 1, 1)
                except IndexError:
                    raise PreprocessError("#else stmt without leading #if "
                                          "stmt", defines['__FILE__'],
                                          defines['__LINE__'], line)
            elif op == "endif":
                try:
                    states.pop()
                except IndexError:
                    raise PreprocessError("#endif stmt without leading #if"
                                          "stmt", defines['__FILE__'],
                                          defines['__LINE__'], line)
            elif op == "error":
                if not (states and states[-1][0] == SKIP):
                    error = match.group("error")
                    raise PreprocessError("#error: " + error, defines['__FILE__'],
                                          defines['__LINE__'], line)
            log.debug("states: %r", states)
            if keepLines:
                fout.write("\n")
        else:
            try:
                if states[-1][0] == EMIT:
                    log.debug("emit line (%s)" % states[-1][1])
                    # Substitute all defines into line.
                    # XXX Should avoid recursive substitutions. But that
                    #     would be a pain right now.
                    sline = line
                    if substitute:
                        for name in reversed(sorted(defines, key=len)):
                            value = defines[name]
                            sline = sline.replace(name, str(value))
                    fout.write(sline)
                elif keepLines:
                    log.debug("keep blank line (%s)" % states[-1][1])
                    fout.write("\n")
                else:
                    log.debug("skip line (%s)" % states[-1][1])
            except IndexError:
                raise PreprocessError("superfluous #endif before this line",
                                      defines['__FILE__'],
                                      defines['__LINE__'])
    if len(states) > 1:
        raise PreprocessError("unterminated #if block", defines['__FILE__'],
                              defines['__LINE__'])
    elif len(states) < 1:
        raise PreprocessError("superfluous #endif on or before this line",
                              defines['__FILE__'], defines['__LINE__'])

    if fout != outfile:
        fout.close()

    return defines

# ---- content-type handling

_gDefaultContentTypes = """
    # Default file types understood by "preprocess.py".
    #
    # Format is an extension of 'mime.types' file syntax.
    #   - '#' indicates a comment to the end of the line.
    #   - a line is:
    #       <filetype> [<pattern>...]
    #     where,
    #       <filetype>'s are equivalent in spirit to the names used in the Windows
    #           registry in HKCR, but some of those names suck or are inconsistent;
    #           and
    #       <pattern> is a suffix (pattern starts with a '.'), a regular expression
    #           (pattern is enclosed in '/' characters), a full filename (anything
    #           else).
    #
    # Notes on case-sensitivity:
    #
    # A suffix pattern is case-insensitive on Windows and case-sensitive
    # elsewhere.  A filename pattern is case-sensitive everywhere. A regex
    # pattern's case-sensitivity is defined by the regex. This means it is by
    # default case-sensitive, but this can be changed using Python's inline
    # regex option syntax. E.g.:
    #         Makefile            /^(?i)makefile.*$/   # case-INsensitive regex

    Python              .py
    Python              .pyw
    Perl                .pl
    Ruby                .rb
    Tcl                 .tcl
    XML                 .xml
    XML                 .kpf
    XML                 .xul
    XML                 .rdf
    XML                 .xslt
    XML                 .xsl
    XML                 .wxs
    XML                 .wxi
    HTML                .htm
    HTML                .html
    XML                 .xhtml
    Makefile            /^[Mm]akefile.*$/
    PHP                 .php
    JavaScript          .js
    CSS                 .css
    C++                 .c       # C++ because then we can use //-style comments
    C++                 .cpp
    C++                 .cxx
    C++                 .cc
    C++                 .h
    C++                 .hpp
    C++                 .hxx
    C++                 .hh
    IDL                 .idl
    Text                .txt
    Fortran             .f
    Fortran             .f90
    Shell               .sh
    Shell               .csh
    Shell               .ksh
    Shell               .zsh
    Java                .java
    C#                  .cs
    TeX                 .tex

    # Some Komodo-specific file extensions
    Python              .ksf  # Fonts & Colors scheme files
    Text                .kkf  # Keybinding schemes files
"""

class ContentTypesRegistry:
    """A class that handles determining the filetype of a given path.

    Usage:
        >>> registry = ContentTypesRegistry()
        >>> registry.getContentType("foo.py")
        "Python"
    """

    def __init__(self, contentTypesPaths=None):
        self.contentTypesPaths = contentTypesPaths
        self._load()

    def _load(self):
        from os.path import dirname, join, exists

        self.suffixMap = {}
        self.regexMap = {}
        self.filenameMap = {}

        self._loadContentType(_gDefaultContentTypes)
        localContentTypesPath = join(dirname(__file__), "content.types")
        if exists(localContentTypesPath):
            log.debug("load content types file: `%r'" % localContentTypesPath)
            self._loadContentType(open(localContentTypesPath).read())
        for path in (self.contentTypesPaths or []):
            log.debug("load content types file: `%r'" % path)
            self._loadContentType(open(path).read())

    def _loadContentType(self, content, path=None):
        """Return the registry for the given content.types file.

        The registry is three mappings:
            <suffix> -> <content type>
            <regex> -> <content type>
            <filename> -> <content type>
        """
        for line in content.splitlines(0):
            words = line.strip().split()
            for i in range(len(words)):
                if words[i][0] == '#':
                    del words[i:]
                    break
            if not words: continue
            contentType, patterns = words[0], words[1:]
            if not patterns:
                if line[-1] == '\n': line = line[:-1]
                raise PreprocessError("bogus content.types line, there must " \
                                      "be one or more patterns: '%s'" % line)
            for pattern in patterns:
                if pattern.startswith('.'):
                    if sys.platform.startswith("win"):
                        # Suffix patterns are case-insensitive on Windows.
                        pattern = pattern.lower()
                    self.suffixMap[pattern] = contentType
                elif pattern.startswith('/') and pattern.endswith('/'):
                    self.regexMap[re.compile(pattern[1:-1])] = contentType
                else:
                    self.filenameMap[pattern] = contentType

    def getContentType(self, path):
        """Return a content type for the given path.

        @param path {str} The path of file for which to guess the
            content type.
        @returns {str|None} Returns None if could not determine the
            content type.
        """
        basename = os.path.basename(path)
        contentType = None
        # Try to determine from the path.
        if not contentType and basename in self.filenameMap:
            contentType = self.filenameMap[basename]
            log.debug("Content type of '%s' is '%s' (determined from full "
                      "path).", path, contentType)
        # Try to determine from the suffix.
        if not contentType and '.' in basename:
            suffix = "." + basename.split(".")[-1]
            if sys.platform.startswith("win"):
                # Suffix patterns are case-insensitive on Windows.
                suffix = suffix.lower()
            if suffix in self.suffixMap:
                contentType = self.suffixMap[suffix]
                log.debug("Content type of '%s' is '%s' (determined from "
                          "suffix '%s').", path, contentType, suffix)
        # Try to determine from the registered set of regex patterns.
        if not contentType:
            for regex, ctype in list(self.regexMap.items()):
                if regex.search(basename):
                    contentType = ctype
                    log.debug("Content type of '%s' is '%s' (matches regex '%s')",
                              path, contentType, regex.pattern)
                    break
        # Try to determine from the file contents.
        content = open(path, 'rb').read()
        if content.startswith("<?xml"):  # cheap XML sniffing
            contentType = "XML"
        return contentType

_gDefaultContentTypesRegistry = None

def getDefaultContentTypesRegistry():
    global _gDefaultContentTypesRegistry
    if _gDefaultContentTypesRegistry is None:
        _gDefaultContentTypesRegistry = ContentTypesRegistry()
    return _gDefaultContentTypesRegistry

# ---- internal support stuff
# TODO: move other internal stuff down to this section

try:
    reversed
except NameError:
    # 'reversed' added in Python 2.4 (http://www.python.org/doc/2.4/whatsnew/node7.html)
    def reversed(seq):
        rseq = list(seq)
        rseq.reverse()
        for item in rseq:
            yield item
try:
    sorted
except NameError:
    # 'sorted' added in Python 2.4. Note that I'm only implementing enough
    # of sorted as is used in this module.
    def sorted(seq, key=None):
        identity = lambda x: x
        key_func = (key or identity)
        sseq = list(seq)
        sseq.sort(lambda self, other: (key_func(self) > key_func(other))- (key_func(self) < key_func(other)))
        for item in sseq:
            yield item

# ---- mainline

def main(argv):
    try:
        optlist, args = getopt.getopt(argv[1:], 'hVvo:D:fkI:sc:',
                                      ['help', 'version', 'verbose', 'force', 'keep-lines',
                                       'substitute', 'content-types-path='])
    except getopt.GetoptError as msg:
        sys.stderr.write("preprocess: error: %s. Your invocation was: %s\n" \
                         % (msg, argv))
        sys.stderr.write("See 'preprocess --help'.\n")
        return 1
    outfile = sys.stdout
    defines = {}
    force = 0
    keepLines = 0
    substitute = 0
    includePath = []
    contentTypesPaths = []
    for opt, optarg in optlist:
        if opt in ('-h', '--help'):
            sys.stdout.write(__doc__)
            return 0
        elif opt in ('-V', '--version'):
            sys.stdout.write("preprocess %s\n" % __version__)
            return 0
        elif opt in ('-v', '--verbose'):
            log.setLevel(log.DEBUG)
        elif opt == '-o':
            outfile = optarg
        if opt in ('-f', '--force'):
            force = 1
        elif opt == '-D':
            if optarg.find('=') != -1:
                var, val = optarg.split('=', 1)
                try:
                    val = int(val)
                except ValueError:
                    pass
            else:
                var, val = optarg, None
            defines[var] = val
        elif opt in ('-k', '--keep-lines'):
            keepLines = 1
        elif opt == '-I':
            includePath.append(optarg)
        elif opt in ('-s', '--substitute'):
            substitute = 1
        elif opt in ('-c', '--content-types-path'):
            contentTypesPaths.append(optarg)

    if len(args) != 1:
        sys.stderr.write("preprocess: error: incorrect number of " \
                         "arguments: argv=%r\n" % argv)
        return 1
    else:
        infile = args[0]

    try:
        contentTypesRegistry = ContentTypesRegistry(contentTypesPaths)
        preprocess(infile, outfile, defines, force, keepLines, includePath,
                   substitute, contentTypesRegistry=contentTypesRegistry)
    except PreprocessError as ex:
        if log.isDebugEnabled():
            import traceback
            traceback.print_exc(file=sys.stderr)
        else:
            sys.stderr.write("preprocess: error: %s\n" % str(ex))
        return 1

if __name__ == "__main__":
    __file__ = sys.argv[0]
    sys.exit(main(sys.argv))


code:
def sort_bubble(array=[],reverse=False):
    mlen=len(array)
    cout_i=0
    while cout_i<mlen:
        cout_j=0
        while cout_j<mlen-1:
            if array[cout_j-1] > array[cout_j]:
                tmp=array[cout_j-1]
                array[cout_j-1]=array[cout_j]
                array[cout_j]=tmp
            cout_j=cout_j+1
        cout_i=cout_i+1
    if reverse:
        array.reverse()
    return array

def sort_merge(x,reverse=False):
    result = []
    if len(x) < 2:
        return x
    mid = int(len(x)/2)
    y = sort_merge(x[:mid])
    z = sort_merge(x[mid:])
    while (len(y) > 0) or (len(z) > 0):
        if len(y) > 0 and len(z) > 0:
            if y[0] > z[0]:
                result.append(z[0])
                z.pop(0)
            else:
                result.append(y[0])
                y.pop(0)
        elif len(z) > 0:
            for i in z:
                result.append(i)
                z.pop(0)
        else:
            for i in y:
                result.append(i)
                y.pop(0)
    if reverse:
        result.reverse()
    return result

def sort_min_max(x,reverse=False):
    llist=[]
    hlist=[]
    while len(x) != 0:
        min=x[0]
        max=x[0]
        for i in x:
            if i<=min:
                min=i
            elif i>=max:
                max=i
        hlist.insert(0,max)
        x.remove(max)
        if min != max:
            llist.append(min)
            x.remove(min)
    array=llist+hlist
    if reverse:
        array.reverse()
    return array

def sort_auto(array=[],reverse=False):
    if len(array) <= 100:
        return sort_bubble(array,reverse)
    elif len(array) <= 1000:
        return sort_min_max(array,reverse)
    else:
        return sort_merge(array,reverse)

repo_name: SulinOS/inary

license: mit


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.

import os.path

import inary.context as ctx
import inary.errors

import gettext
__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

class MirrorError(inary.errors.Error):
    pass

class Mirrors:
    def __init__(self, config=ctx.const.mirrors_conf):
        self.mirrors = {}
        self._parse(config)

    def get_mirrors(self, name):
        if name in self.mirrors:
            return list(self.mirrors[name])

        return None

    def _add_mirror(self, name, url):
        if name in self.mirrors:
            self.mirrors[name].append(url)
        else:
            self.mirrors[name] = [url]

    def _parse(self, config):
        if os.path.exists(config):
            for line in open(config).readlines():
                if not line.startswith('#') and not line == '\n':
                    mirror = line.strip().split()
                    if len(mirror) == 2:
                        (name, url) = mirror
                        self._add_mirror(name, url)
        else:
            raise inary.errors.Error(_('Mirrors file \"{}\" does not exist. Could not resolve \"mirrors://\"').format(config))


code:
# -*- coding: utf-8 -*-
#

# Guido's cool metaclass examples. fair use. ahahah.
# I find these quite handy. Use them :)

class autoprop(type):
    def __init__(cls, name, bases, dict):
        super(autoprop, cls).__init__(name, bases, dict)
        props = {}
        for name in list(dict.keys()):
            if name.startswith("_get_") or name.startswith("_set_"):
                props[name[5:]] = 1
        for name in list(props.keys()):
            fget = getattr(cls, "_get_{}".format(name), None)
            fset = getattr(cls, "_set_{}".format(name), None)
            setattr(cls, name, property(fget, fset))

class autosuper(type):
    def __init__(cls, name, bases, dict):
        super(autosuper, cls).__init__(name, bases, dict)
        setattr(cls, "_{}__super".format(name), super(cls))

class autosuprop(autosuper, autoprop):
    pass

class autoeq(type):
    """useful for structures"""

    def __init__(cls, name, bases, dict):
        super(autoeq, cls).__init__(name, bases, dict)

        def equal(self, other):
            return self.__dict__ == other.__dict__

        cls.__eq__ = equal

class Struct(metaclass=autoeq):
    def __init__(self, **entries):
        self.__dict__.update(entries)


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#
import inary.errors

class Error(inary.errors.Error):
    pass

class Exception(inary.errors.Exception):
    pass


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.

"""package building code"""

import fnmatch
import glob
import grp
# python standard library
import os
import pwd
import re
import stat

import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

import inary.errors
import inary.data.specfile as Specfile
import inary.util as util
import inary.file
import inary.context as ctx
import inary.analyzer.dependency as dependency
import inary.api
import inary.archive
import inary.data.files as Files
import inary.fetcher
import inary.uri
import inary.data.metadata as Metadata
import inary.package
import inary.data.component as Component
import inary.actionsapi.variables
import inary.db

class Error(inary.errors.Error):
    pass

class ActionScriptException(Error):
    pass

class AbandonedFilesException(inary.errors.Error):
    pass

class ExcludedArchitectureException(Error):
    pass

# Helper Functions
def get_file_type(path, pinfo_list):
    """Return the file type of a path according to the given PathInfo
    list"""

    path = "/{}".format(re.sub("/+", "/", path))
    info = None
    glob_match = parent_match = None

    for pinfo in pinfo_list:
        if path == pinfo.path:
            info = pinfo
            break

        elif fnmatch.fnmatch(path, pinfo.path):
            glob_match = pinfo

        elif fnmatch.fnmatch(path, util.join_path(pinfo.path, "*")):
            if parent_match is None or parent_match.path < pinfo.path:
                parent_match = pinfo

    else:
        info = glob_match or parent_match

    return info.fileType, info.permanent

def check_path_collision(package, pkgList):
    """This function will check for collision of paths in a package with
    the paths of packages in pkgList. The return value will be the
    list containing the paths that collide."""
    create_static = ctx.get_option("create_static")
    create_debug = ctx.config.values.build.generatedebug
    ar_suffix = ctx.const.ar_file_suffix
    debug_suffix = ctx.const.debug_file_suffix

    collisions = []
    for pinfo in package.files:
        for pkg in pkgList:
            if pkg is package:
                continue
            for path in pkg.files:
                # if pinfo.path is a subpath of path.path like
                # the example below. path.path is marked as a
                # collide. Exp:
                # pinfo.path: /usr/share
                # path.path: /usr/share/doc

                if (create_static and path.path.endswith(ar_suffix)) or \
                        (create_debug and path.path.endswith(debug_suffix)):
                    # don't throw collision error for these files.
                    # we'll handle this in gen_files_xml..
                    continue

                if util.subpath(pinfo.path, path.path):
                    collisions.append(path.path.rstrip("/"))
                    ctx.ui.debug(_('Path \"{}\" belongs in multiple packages.').format(path.path))
    return collisions

def exclude_special_files(filepath, fileinfo, ag):
    keeplist = ag.get("KeepSpecial", [])
    patterns = {"libtool": "libtool library file",
                "python": "python.*byte-compiled",
                "perl": "Perl POD document text"}

    if "libtool" in keeplist:
        # Some upstream sources have buggy libtool and ltmain.sh with them,
        # which causes wrong path entries in *.la files. And these wrong path
        # entries sometimes triggers compile-time errors or linkage problems.
        # Instead of patching all these buggy sources and maintain these
        # patches, INARY removes wrong paths...
        if re.match(patterns["libtool"], fileinfo) and \
                not os.path.islink(filepath):
            ladata = open(filepath).read()
            new_ladata = re.sub("-L{}/\S*".format(ctx.config.tmp_dir()), "", ladata)
            new_ladata = re.sub("{}/\S*/install/".format(ctx.config.tmp_dir()), "/",
                                new_ladata)
            if new_ladata != ladata:
                open(filepath, "w").write(new_ladata)

    for name, pattern in list(patterns.items()):
        if name in keeplist:
            continue

        if fileinfo is None:
            ctx.ui.warning(_("Removing special file skipped for: \"{}\"").format(filepath))
            return
        elif re.match(pattern, fileinfo):
            ctx.ui.warning(_("Removing special \"{0}\" file: \"{1}\"").format(name, filepath))
            os.unlink(filepath)
            # Remove dir if it becomes empty (Bug #11588)
            util.rmdirs(os.path.dirname(filepath))

def strip_debug_action(filepath, fileinfo, install_dir, ag):
    excludelist = tuple(ag.get("NoStrip", []))

    # real path in .inary package
    path = '/' + util.removepathprefix(install_dir, filepath)

    if path.startswith(excludelist):
        return

    outputpath = util.join_path(os.path.dirname(install_dir),
                                ctx.const.debug_dir_suffix,
                                ctx.const.debug_files_suffix,
                                path)

    if util.strip_file(filepath, fileinfo, outputpath):
        ctx.ui.info("{0} [{1}]".format(path, "stripped"), verbose=True)

class Builder:
    """Provides the package build and creation routines"""

    # FIXME: this class and every other class must use URLs as paths!

    @staticmethod
    def from_name(name):
        repodb = inary.db.repodb.RepoDB()
        sourcedb = inary.db.sourcedb.SourceDB()
        # download package and return an installer object
        # find package in repository
        sf, reponame = sourcedb.get_spec_repo(name)
        src = sf.source
        if src:

            src_uri = inary.uri.URI(src.sourceURI)
            if src_uri.is_absolute_path():
                src_path = str(src_uri)
            else:
                repo = repodb.get_repo(reponame)
                # FIXME: don't use dirname to work on URLs
                src_path = os.path.join(
                    os.path.dirname(repo.indexuri.get_uri()),
                    str(src_uri.path()))

            ctx.ui.info(_("Source URI: {}").format(src_path), verbose=True)

            return Builder(src_path)
        else:
            raise Error(_("Source \"{}\" not found in any active repository.").format(name))

    def __init__(self, specuri):

        self.componentdb = inary.db.componentdb.ComponentDB()
        self.installdb = inary.db.installdb.InstallDB()

        # process args
        if not isinstance(specuri, inary.uri.URI):
            specuri = inary.uri.URI(specuri)

        # read spec file, we'll need it :)
        self.set_spec_file(specuri)

        if specuri.is_remote_file():
            self.specdir = self.fetch_files()
        else:
            self.specdir = os.path.dirname(self.specuri.get_uri())

        # Don't wait until creating .inary file for complaining about versioning
        # scheme errors
        self.package_rfp=None
        if self.spec.source.rfp:
            ctx.ui.info(util.colorize(_("[ !!! ] Building RFP for {}").format(self.spec.source.rfp),
                        color="purple"))
            if ctx.ui.confirm(_("Would you like to compile this RFP package?")):
                self.package_rfp=self.spec.source.rfp
            else:
                raise Error(_("Didn't permit build RFP package."))

        self.check_versioning(self.spec.getSourceVersion(),
                              self.spec.getSourceRelease())

        # Check package format
        self.target_package_format = ctx.get_option("package_format") \
                                     or inary.package.Package.default_format

        self.read_translations(self.specdir)

        self.sourceArchives = inary.archive.SourceArchives(self.spec)

        self.build_types = self.get_build_types()

        # Use an empty string for the default build
        self.set_build_type("")

        self.check_paths()

        self.actionLocals = None
        self.actionGlobals = None

        self.delta_history_search_paths = []
        self.delta_search_paths = {}

        self.new_packages = []
        self.new_debug_packages = []

        self.delta_map = {}

        self.has_ccache = False
        self.has_icecream = False

    def set_spec_file(self, specuri):
        if not specuri.is_remote_file():
            # FIXME: doesn't work for file://
            specuri = inary.uri.URI(os.path.realpath(specuri.get_uri()))
        self.specuri = specuri
        spec = Specfile.SpecFile()
        spec.read(self.specuri, ctx.config.tmp_dir())
        self.spec = spec

    def read_translations(self, specdir):
        self.spec.read_translations(util.join_path(specdir,
                                                   ctx.const.translations_file))

    def package_filename(self, package_info, release_info=None,
                         distro_id=None, with_extension=True):

        if release_info is None:
            release_info = self.spec.history[0]

        if distro_id is None:
            distro_id = ctx.config.values.general.distribution_id

        fn = "-".join((package_info.name,
                       release_info.version,
                       release_info.release,
                       distro_id,
                       package_info.architecture))

        if self.package_rfp:
            package_extension = ".rfp" + ctx.const.package_suffix
        else:
            package_extension = ctx.const.package_suffix
        if with_extension:
            fn += package_extension

        return fn

    # directory accessor functions

    # pkg_x_dir: per package directory for storing info type x

    def pkg_dir(self):
        """package build directory"""
        packageDir = self.spec.source.name + '-' + \
                     self.spec.getSourceVersion() + '-' + \
                     self.spec.getSourceRelease()
        return util.join_path(ctx.config.dest_dir(),
                              ctx.config.tmp_dir(),
                              packageDir)

    def pkg_work_dir(self):
        suffix = "-{}".format(self.build_type) if self.build_type else ""
        return self.pkg_dir() + ctx.const.work_dir_suffix + suffix

    def pkg_debug_dir(self):
        return self.pkg_dir() + ctx.const.debug_dir_suffix

    def pkg_install_dir(self):
        return self.pkg_dir() + ctx.const.install_dir_suffix

    def set_state(self, state):
        stateFile = util.join_path(self.pkg_work_dir(), "inaryBuildState")
        open(stateFile, "w").write(state)

    def get_state(self):
        stateFile = util.join_path(self.pkg_work_dir(), "inaryBuildState")
        if not os.path.exists(stateFile):  # no state
            return None
        return open(stateFile).read()

    def build(self):
        """Build the package in one shot."""
        architecture = ctx.config.values.general.architecture
        if architecture in self.spec.source.excludeArch:
            raise ExcludedArchitectureException(
                _("pspec.xml avoids this package from building for \"{}\"").format(
                    architecture))

        ctx.ui.status(_("Building source package: \"{}\"").format(
            self.spec.source.name))
        # check if all patch files exists, if there are missing no need
        # to unpack!
        self.check_patches()

        self.check_build_dependencies()
        self.fetch_component()
        self.fetch_source_archives()

        util.clean_dir(self.pkg_install_dir())
        for build_type in self.build_types:
            self.set_build_type(build_type)
            self.unpack_source_archives()

            if self.has_ccache:
                ctx.ui.info(_("ccache detected..."))
            if self.has_icecream:
                ctx.ui.info(_("IceCream detected. Make sure your daemon "
                              "is up and running..."))
        for build_type in self.build_types:
            self.run_setup_action()
        for build_type in self.build_types:
            self.run_build_action()
        for build_type in self.build_types:
            if ctx.get_option('debug') and not ctx.get_option('ignore_check'):
                self.run_check_action()
        for build_type in self.build_types:
            self.run_install_action()

        # after all, we are ready to build/prepare the packages
        self.build_packages()

    def get_build_types(self):
        ignored_build_types = \
            ctx.config.values.build.ignored_build_types.split(",")
        build_types = []
        packages = []

        for package in self.spec.packages:
            if package.buildType:
                if package.buildType in ignored_build_types:
                    continue

                if package.buildType not in build_types:
                    build_types.append(package.buildType)

            packages.append(package)

        self.spec.packages = packages
        build_types.append("")
        return build_types

    def set_build_type(self, build_type):
        if build_type:
            ctx.ui.action(_("Rebuilding source for build type: {}").format(build_type))

        self.build_type = build_type
        self.set_environment_vars()
        self.load_action_script()

    def set_environment_vars(self):
        """Sets the environment variables for actions API to use"""

        # Each time a builder is created we must reset
        # environment. See bug #2575
        os.environ.clear()
        inary.actionsapi.variables.initVariables()

        env = {"PKG_DIR": self.pkg_dir(),
               "WORK_DIR": self.pkg_work_dir(),
               "HOME": self.pkg_work_dir(),
               "INSTALL_DIR": self.pkg_install_dir(),
               "INARY_BUILD_TYPE": self.build_type,
               "SRC_NAME": self.spec.source.name,
               "SRC_VERSION": self.spec.getSourceVersion(),
               "SRC_RELEASE": self.spec.getSourceRelease(),
               "PYTHONDONTWRITEBYTECODE": '1'}
        if self.build_type == "emul32":
            env["CC"] = "{} -m32".format(os.getenv("CC"))
            env["CXX"] = "{} -m32".format(os.getenv("CXX"))
            env["CFLAGS"] = os.getenv("CFLAGS").replace("-fPIC", "")
            env["CXXFLAGS"] = os.getenv("CXXFLAGS").replace("-fPIC", "")
            env["PKG_CONFIG_PATH"] = "/usr/lib32/pkgconfig"
        if self.build_type == "clang":
            env['CC'] = "clang"
            env['CXX'] = "clang++"
        if self.build_type == "clang32":
            env['CC'] = "clang -m32"
            env['CXX'] = "clang++ -m32"
        os.environ.update(env)

        # First check icecream, if not found use ccache
        # TODO: Add support for using both of them
        if ctx.config.values.build.buildhelper == "icecream":
            if os.path.exists("/opt/icecream/bin/gcc"):
                self.has_icecream = True
                os.environ["PATH"] = "/opt/icecream/bin:%(PATH)s" % os.environ

        elif ctx.config.values.build.buildhelper == "ccache":
            if os.path.exists("/usr/lib/ccache/bin/gcc"):
                self.has_ccache = True

                os.environ["PATH"] = "/usr/lib/ccache/bin:%(PATH)s" % os.environ
                # Force ccache to use /root/.ccache instead of $HOME/.ccache
                # as $HOME can be modified through actions.py
                os.environ["CCACHE_DIR"] = "/root/.ccache"

    def fetch_files(self):
        self.specdiruri = os.path.dirname(self.specuri.get_uri())
        pkgname = os.path.basename(self.specdiruri)
        self.destdir = util.join_path(ctx.config.tmp_dir(), pkgname)
        # self.location = os.path.dirname(self.url.uri)

        self.fetch_actionsfile()
        self.check_build_dependencies()
        self.fetch_translationsfile()
        self.fetch_patches()
        self.fetch_additionalFiles()
        self.fetch_scomfiles()

        return self.destdir

    def fetch_pspecfile(self):
        pspecuri = util.join_path(self.specdiruri, ctx.const.pspec_file)
        self.download(pspecuri, self.destdir)

    def fetch_actionsfile(self):
        actionsuri = util.join_path(self.specdiruri, ctx.const.actions_file)
        self.download(actionsuri, self.destdir)

    def fetch_translationsfile(self):
        translationsuri = util.join_path(self.specdiruri,
                                         ctx.const.translations_file)
        try:
            self.download(translationsuri, self.destdir)
        except inary.fetcher.FetchError:
            # translations.xml is not mandatory for INARY
            pass

    def fetch_patches(self):
        for patch in self.spec.source.patches:
            dir_name = os.path.dirname(patch.filename)
            patchuri = util.join_path(self.specdiruri,
                                      ctx.const.files_dir,
                                      patch.filename)
            self.download(patchuri, util.join_path(self.destdir,
                                                   ctx.const.files_dir,
                                                   dir_name))

    def fetch_additionalFiles(self):
        for pkg in self.spec.packages + [self.spec.source]:
            for afile in pkg.additionalFiles:
                file_name = os.path.basename(afile.filename)
                dir_name = os.path.dirname(afile.filename)
                afileuri = util.join_path(self.specdiruri,
                                          ctx.const.files_dir, dir_name, file_name)
                self.download(afileuri, util.join_path(self.destdir,
                                                       ctx.const.files_dir,
                                                       dir_name))

    def fetch_scomfiles(self):
        for package in self.spec.packages:
            for pscom in package.providesScom:
                scomuri = util.join_path(self.specdiruri,
                                         ctx.const.scom_dir, pscom.script)
                self.download(scomuri, util.join_path(self.specdir,
                                                      ctx.const.scom_dir))
                ctx.ui.info("Scom Script Fetched {}".format(pscom.script))

    @staticmethod
    def download(uri, transferdir):
        # fix auth info and download
        uri = inary.file.File.make_uri(uri)
        inary.file.File.download(uri, transferdir)

    def fetch_component(self):
        if self.spec.source.partOf:
            return

        diruri = util.parenturi(self.specuri.get_uri())
        parentdir = util.parenturi(diruri)
        url = util.join_path(parentdir, 'component.xml')
        progress = ctx.ui.Progress
        if inary.uri.URI(url).is_remote_file():
            try:
                inary.fetcher.fetch_url(url, self.pkg_work_dir(), progress)
            except inary.fetcher.FetchError:
                ctx.ui.warning(_("Cannot find component.xml for \"{}\" in remote "
                                 "directory, Source is now part of "
                                 "unknown component.").format(self.spec.source.name))
                self.spec.source.partOf = 'unknown'
                return
            path = util.join_path(self.pkg_work_dir(), 'component.xml')
        else:
            if not os.path.exists(url):
                ctx.ui.warning(_("Cannot find component.xml for \"{}\" in upper "
                                 "directory, Source is now part of "
                                 "unknown component.").format(self.spec.source.name))
                self.spec.source.partOf = 'unknown'
                return
            path = url
        comp = Component.CompatComponent()
        comp.read(path)
        self.spec.source.partOf = comp.name

    def fetch_source_archives(self):
        ctx.ui.status(_("Building source package: \"{}\" [ Fetching Step ]").format(self.spec.source.name),
                      push_screen=False)
        self.sourceArchives.fetch()

    def unpack_source_archives(self):
        ctx.ui.status(_("Building source package: \"{}\" [ Unpacking Step ]").format(self.spec.source.name),
                      push_screen=False)
        ctx.ui.action(util.colorize(">>> ", 'purple') + _("Unpacking archive(s)..."))
        self.sourceArchives.unpack(self.pkg_work_dir())

        # Grab AdditionalFiles
        self.copy_additional_source_files()

        # apply the patches and prepare a source directory for build.
        if self.apply_patches():
            ctx.ui.info(_(" -> unpacked ({})").format(self.pkg_work_dir()))
            self.set_state("unpack")

    def run_setup_action(self):
        #  Run configure, build and install phase
        ctx.ui.status(_("Building source package: \"{}\" [ SetupAction Step ]").format(self.spec.source.name),
                      push_screen=False)
        ctx.ui.action(util.colorize(">>> ", 'yellow') + _("Setting up source..."))
        if self.run_action_function(ctx.const.setup_func):
            self.set_state("setupaction")

    def run_build_action(self):
        ctx.ui.status(_("Building source package: \"{}\" [ BuildAction Step ]").format(self.spec.source.name),
                      push_screen=False)
        ctx.ui.action(util.colorize(">>> ", 'green') + _("Building source..."))
        if self.run_action_function(ctx.const.build_func):
            self.set_state("buildaction")

    def run_check_action(self):
        ctx.ui.status(_("Building source package: \"{}\" [ CheckAction Step ]").format(self.spec.source.name),
                      push_screen=False)
        ctx.ui.action(util.colorize(">>> ", 'blue') + _("Testing package..."))
        self.run_action_function(ctx.const.check_func)

    def run_install_action(self):
        ctx.ui.status(_("Building source package: \"{}\" [ InstallAction Step ]").format(self.spec.source.name),
                      push_screen=False)
        ctx.ui.action(util.colorize(">>> ", 'cyan') + _("Installing..."))

        # install function is mandatory!
        if self.run_action_function(ctx.const.install_func, True):
            self.set_state("installaction")

    def get_abandoned_files(self):
        # return the files those are not collected from the install dir

        install_dir = self.pkg_install_dir()
        abandoned_files = []
        all_paths_in_packages = []
        skip_paths = []

        for package in self.spec.packages:
            for path in package.files:
                path = util.join_path(install_dir, path.path)
                all_paths_in_packages.append(path)

        def is_included(path1, path2):
            """Return True if path2 includes path1"""
            return path1 == path2 \
                   or fnmatch.fnmatch(path1, path2) \
                   or (not os.path.isfile(path2) and fnmatch.fnmatch(path1, util.join_path(path2, "*")))

        for root, dirs, files in os.walk(install_dir):
            if not dirs and not files:
                for _path in all_paths_in_packages:
                    if is_included(root, _path):
                        break
                else:
                    abandoned_files.append(root)

            if root in all_paths_in_packages:
                skip_paths.append(root)
                continue

            skip = False
            for skip_path in skip_paths:
                if root.startswith(skip_path):
                    skip = True
                    break
            if skip: continue

            for file_ in files:
                fpath = util.join_path(root, file_)
                for _path in all_paths_in_packages:
                    if is_included(fpath, _path):
                        if os.path.isfile(_path):
                            all_paths_in_packages.pop(all_paths_in_packages.index(_path))
                        break
                else:
                    abandoned_files.append(fpath)

        len_install_dir = len(install_dir)
        return [x[len_install_dir:] for x in abandoned_files]

    def copy_additional_source_files(self):
        # store additional files
        for afile in self.spec.source.additionalFiles:
            src = os.path.join(self.specdir, ctx.const.files_dir, afile.filename)
            dest = os.path.join(self.pkg_src_dir(), afile.target)
            util.copy_file(src, dest)
            if afile.permission:
                # mode is octal!
                os.chmod(dest, int(afile.permission, 8))

    def compile_action_script(self):
        """Compiles the action script and returns a code object"""

        fname = util.join_path(self.specdir, ctx.const.actions_file)
        try:
            buf = open(fname).read()
            return compile(buf, fname, "exec")
        except IOError as e:
            raise Error(_("Unable to read Actions Script ({0}): {1}").format(
                fname, e))
        except SyntaxError as e:
            raise Error(_("SyntaxError in Actions Script ({0}): {1}").format(
                fname, e))

    def load_action_script(self):
        """Compiles and executes the action script"""

        compiled_script = self.compile_action_script()

        try:
            localSymbols = globalSymbols = {}
            exec(compiled_script, localSymbols, globalSymbols)
        except Exception as e:
            raise (e)

        self.actionLocals = localSymbols
        self.actionGlobals = globalSymbols

    def compile_scom_script(self):
        """Compiles scom scripts to check syntax errors"""
        for package in self.spec.packages:
            for pscom in package.providesScom:
                fname = util.join_path(self.specdir, ctx.const.scom_dir,
                                       pscom.script)

                try:
                    if(os.path.splitext(fname)[1]=="py"):
                        buf = open(fname).read()
                        compile(buf, "error", "exec")
                except IOError as e:
                    raise Error(_("Unable to read SCOM script ({0}): {1}").format(
                        fname, e))
                except SyntaxError as e:
                    raise Error(_("SyntaxError in SCOM file ({0}): {1}").format(
                        fname, e))

    def pkg_src_dir(self):
        """Returns the real path of WorkDir for an unpacked archive."""

        dirname = self.actionGlobals.get("WorkDir")
        if dirname:
            return util.join_path(self.pkg_work_dir(), dirname)

        dirname = self.spec.source.name + "-" + self.spec.getSourceVersion()
        src_dir = util.join_path(self.pkg_work_dir(), dirname)

        if not os.path.exists(src_dir):
            archive = self.spec.source.archive[0]

            # For binary types, WorkDir is usually "."
            if archive.type == "binary":
                return self.pkg_work_dir()

            basename = os.path.basename(archive.uri)
            dirname = os.path.splitext(basename)[0]
            src_dir = util.join_path(self.pkg_work_dir(), dirname)

            while not os.path.exists(src_dir):
                src_dir, ext = os.path.splitext(src_dir)
                if not ext:
                    break
            if not os.path.exists(src_dir):
                src_dir = util.join_path(self.pkg_work_dir(), [d for d in os.walk(self.pkg_work_dir()).__next__()[1] if
                                                               not d.startswith(".")][0])
                if self.get_state() == "unpack": ctx.ui.info("Using {} as  WorkDir".format(src_dir), verbose=True)

        return src_dir

    @staticmethod
    def log_sandbox_violation(operation, path, canonical_path):
        ctx.ui.error(_("Sandbox violation: {0} ({1} -> {2})").format(operation,
                                                                     path,
                                                                     canonical_path))

    def run_action_function(self, func, mandatory=False):
        """Calls the corresponding function in actions.py.

        If mandatory parameter is True, and function is not present in
        actionLocals inary.build.Error will be raised."""
        # we'll need our working directory after actionscript
        # finished its work in the archive source directory.
        curDir = os.getcwd()
        src_dir = self.pkg_src_dir()
        if os.path.exists(src_dir):
            os.chdir(src_dir)
        else:
            raise Error(_("ERROR: WorkDir ({}) does not exist\n").format(src_dir))

        if func in self.actionLocals:
            if ctx.get_option('ignore_sandbox') or \
                    not ctx.config.values.build.enablesandbox or \
                    "emul32" in self.build_type:
                self.actionLocals[func]()
            else:
                import catbox

                ctx.ui.info(_("Sandbox enabled build..."))

                # Configure allowed paths from sandbox.conf
                valid_paths = [bytes(self.pkg_dir().encode('utf-8'))]
                conf_file = ctx.const.sandbox_conf
                if os.path.exists(conf_file):
                    for line in open(conf_file):
                        line = line.strip()
                        if len(line) > 0 and not line.startswith("#"):
                            if line.startswith("~"):
                                line = os.environ["HOME"] + line[1:]
                            valid_paths.append(line.encode('utf-8'))

                # Extra path for ccache when needed
                if ctx.config.values.build.buildhelper == "ccache":
                    valid_paths.append(os.environ.get("CCACHE_DIR",
                                                      "/root/.ccache"))

                ret = catbox.run(self.actionLocals[func],
                                 valid_paths,
                                 logger=self.log_sandbox_violation)
                # Retcode can be 0 while there is a sanbox violation, so only
                # look for violations to correctly handle it
                if ret.violations:
                    ctx.ui.error(_("Sandbox violation result:"))
                    for result in ret.violations:
                        ctx.ui.error("{0} ({1} -> {2})".format(result[0],
                                                               result[1],
                                                               result[2]))
                    raise Error(_("Sandbox violations!"))

                if ret.code == 1:
                    raise ActionScriptException
        else:
            if mandatory:
                raise Error(_("unable to call function from actions: \'{}\'").format(func))

        os.chdir(curDir)
        return True

    def check_paths(self):
        paths = []

        for package in self.spec.packages:
            for path_info in package.files:
                path = os.path.normpath(path_info.path)

                if not path.startswith("/"):
                    raise Error(_("Source package '{0}' defines a relative 'Path' element: "
                                  "{1}").format(self.spec.source.name, path_info.path))

                if path in paths:
                    raise Error(_("Source package '{0}' defines multiple 'Path' tags "
                                  "for {1}").format(self.spec.source.name, path_info.path))

                paths.append(path)

    @staticmethod
    def check_versioning(version, release):
        try:
            int(release)
            inary.version.make_version(version)
        except (ValueError, inary.version.InvalidVersionError):
            raise Error(_("{0}-{1} is not a valid INARY version format").format(version, release))

    def check_build_dependencies(self):
        """check and try to install build dependencies, otherwise fail."""

        build_deps = self.spec.source.buildDependencies

        for package in self.spec.packages:
            build_deps.extend(package.buildDependencies)

        if not ctx.config.values.general.ignore_safety and \
                not ctx.get_option('ignore_safety'):
            if self.componentdb.has_component('system.devel'):
                build_deps_names = set([x.package for x in build_deps])
                devel_deps_names = set(self.componentdb.get_component('system.devel').packages)
                extra_names = devel_deps_names - build_deps_names
                extra_names = [x for x in extra_names if not self.installdb.has_package(x)]
                if extra_names:
                    ctx.ui.warning(_('Safety switch: following extra packages in system.devel will be installed: ') +
                                   util.strlist(extra_names))
                    extra_deps = [dependency.Dependency(package=x) for x in extra_names]
                    build_deps.extend(extra_deps)
            else:
                ctx.ui.warning(_('Safety switch: the component system.devel cannot be found.'))

        # find out the build dependencies that are not satisfied...
        dep_unsatis = []
        for dep in build_deps:
            if not dep.satisfied_by_installed():
                dep_unsatis.append(dep)

        if dep_unsatis:
            ctx.ui.info(_("Unsatisfied Build Dependencies:") + ' '
                        + util.strlist([str(x) for x in dep_unsatis]))

            def fail():
                raise Error(_('Cannot build package due to unsatisfied build dependencies.'))

            if not ctx.config.get_option('ignore_dependency'):
                for dep in dep_unsatis:
                    if not dep.satisfied_by_repo():
                        raise Error(_('Build dependency \"{}\" cannot be satisfied.').format(str(dep)))
                if ctx.ui.confirm(
                        _('Would you like to install the unsatisfied build dependencies?')):
                    ctx.ui.info(_('Installing build dependencies.'))
                    if not inary.api.install([dep.package for dep in dep_unsatis], reinstall=True):
                        fail()
                else:
                    fail()
            else:
                ctx.ui.warning(_('Ignoring build dependencies.'))

    def check_patches(self):
        """check existence of patch files and their sizes."""

        files_dir = os.path.abspath(util.join_path(self.specdir,
                                                   ctx.const.files_dir))
        for patch in self.spec.source.patches:
            patchFile = util.join_path(files_dir, patch.filename)
            if not os.access(patchFile, os.F_OK):
                raise Error(_("Patch file is missing: \"{}\"\n").format(patch.filename))
            if os.stat(patchFile).st_size == 0:
                ctx.ui.warning(_('Patch file is empty: \"{}\"').format(patch.filename))

    def apply_patches(self):
        files_dir = os.path.abspath(util.join_path(self.specdir,
                                                   ctx.const.files_dir))

        for patch in self.spec.source.patches:
            patchFile = util.join_path(files_dir, patch.filename)
            relativePath = patch.filename
            reverseApply = patch.reverse and patch.reverse.lower() == "true"
            if patch.compressionType:
                patchFile = util.uncompress(patchFile,
                                            compressType=patch.compressionType,
                                            targetDir=ctx.config.tmp_dir())
                relativePath = relativePath.rsplit(".{}".format(patch.compressionType, 1))[0]

            ctx.ui.action(_("Applying patch: {}").format(patch.filename))
            util.do_patch(self.pkg_src_dir(), patchFile,
                          level=patch.level,
                          name=relativePath,
                          reverse=reverseApply)
        return True

    def generate_static_package_object(self):
        ar_files = []
        for root, dirs, files in os.walk(self.pkg_install_dir()):
            for f in files:
                if f.endswith(ctx.const.ar_file_suffix) and util.is_ar_file(util.join_path(root, f)):
                    ar_files.append(util.join_path(root, f))

        if not len(ar_files):
            return None

        static_package_obj = Specfile.Package()
        static_package_obj.name = self.spec.source.name + ctx.const.static_name_suffix
        # FIXME: find a better way to deal with the summary and description constants.
        static_package_obj.summary['en'] = 'Ar files for {}'.format(self.spec.source.name)
        static_package_obj.description['en'] = 'Ar files for {}'.format(self.spec.source.name)
        static_package_obj.partOf = "static"
        for f in ar_files:
            static_package_obj.files.append(Specfile.Path(path=f[len(self.pkg_install_dir()):], fileType="library"))

        # append all generated packages to dependencies
        for p in self.spec.packages:
            static_package_obj.packageDependencies.append(
                inary.analyzer.dependency.Dependency(package=p.name))

        return static_package_obj

    def generate_debug_package_object(self, package):
        debug_package_obj = Specfile.Package()
        debug_package_obj.debug_package = True
        debug_package_obj.name = package.name + ctx.const.debug_name_suffix
        # FIXME: find a better way to deal with the summary and description constants.
        debug_package_obj.summary['en'] = 'Debug files for {}'.format(package.name)
        debug_package_obj.description['en'] = 'Debug files for {}'.format(package.name)
        debug_package_obj.partOf = "dbginfo"

        dependency = inary.analyzer.dependency.Dependency()
        dependency.package = package.name
        dependency.release = self.spec.history[0].release
        debug_package_obj.packageDependencies.append(dependency)

        for path_info in package.files:
            path = util.join_path(ctx.const.debug_files_suffix, path_info.path)
            debug_path_info = Specfile.Path(path=path, fileType="debug")
            debug_package_obj.files.append(debug_path_info)

        return debug_package_obj

    def gen_metadata_xml(self, package):
        """Generate the metadata.xml file for build source.

        metadata.xml is composed of the information from specfile plus
        some additional information."""
        metadata = Metadata.MetaData()
        metadata.from_spec(self.spec.source, package, self.spec.history)

        metadata.package.buildHost = ctx.config.values.build.build_host

        metadata.package.distribution = ctx.config.values.general.distribution
        metadata.package.distributionRelease = ctx.config.values.general.distribution_release
        metadata.package.architecture = ctx.config.values.general.architecture
        metadata.package.packageFormat = self.target_package_format

        size = 0
        for fileinfo in self.files.list:
            size += fileinfo.size

        metadata.package.installedSize = int(size)

        self.metadata = metadata

    def gen_files_xml(self, package):
        """Generates files.xml using the path definitions in specfile and
        the files produced by the build system."""

        if package.debug_package:
            install_dir = self.pkg_debug_dir()
        else:
            install_dir = self.pkg_install_dir()

        # FIXME: We need to expand globs before trying to calculate hashes
        # Not on the fly like now.

        # we'll exclude collisions in get_file_hashes. Having a
        # collisions list is not wrong, we must just handle it :).
        collisions = check_path_collision(package, self.spec.packages)
        # FIXME: material collisions after expanding globs could be
        # reported as errors

        # Use a dict to avoid duplicate entries in files.xml.
        d = {}

        def add_path(path):
            # add the files under material path
            for fpath, fhash in util.get_file_hashes(path, collisions, install_dir):
                if ctx.get_option('create_static') \
                        and fpath.endswith(ctx.const.ar_file_suffix) \
                        and not package.name.endswith(ctx.const.static_name_suffix) \
                        and util.is_ar_file(fpath):
                    # if this is an ar file, and this package is not a static package,
                    # don't include this file into the package.
                    continue
                frpath = util.removepathprefix(install_dir, fpath)  # relative path
                ftype, permanent = get_file_type(frpath, package.files)
                fsize = int(util.dir_size(fpath))
                if not os.path.islink(fpath):
                    st = os.stat(fpath)
                else:
                    st = os.lstat(fpath)
                _uid = str(st.st_uid)
                _gid = str(st.st_gid)

                for afile in package.additionalFiles:
                    # FIXME: Better way?
                    if frpath == util.removepathprefix("/", afile.target):
                        # This is an additional file, uid and gid will change
                        if afile.owner is None:
                            afile.owner = "root"
                        if afile.group is None:
                            afile.group = "root"

                        try:
                            _uid = str(pwd.getpwnam(afile.owner)[2])
                        except KeyError:
                            ctx.ui.warning(_("No user named '{}' found "
                                             "on the system").format(afile.owner))
                        try:
                            _gid = str(grp.getgrnam(afile.group)[2])
                        except KeyError:
                            ctx.ui.warning(_("No group named '{}' found "
                                             "on the system").format(afile.group))
                        break
                d[frpath] = Files.FileInfo(path=frpath, type=ftype, permanent=permanent,
                                           size=fsize, hash=fhash, uid=_uid, gid=_gid,
                                           mode=oct(stat.S_IMODE(st.st_mode)))

                if stat.S_IMODE(st.st_mode) & stat.S_ISUID:
                    ctx.ui.warning(_("/{} has suid bit set").format(frpath))

        for pinfo in package.files:
            wildcard_path = util.join_path(install_dir, pinfo.path)
            for path in glob.glob(wildcard_path):
                add_path(path)

        files = Files.Files()
        for fileinfo in d.values():
            files.append(fileinfo)

        files_xml_path = util.join_path(self.pkg_dir(), ctx.const.files_xml)
        files.write(files_xml_path)
        self.files = files

    def file_actions(self):
        install_dir = self.pkg_install_dir()
        witcher = None  # Who makes magic? ;)
        try:
            witcher = __import__("magic").detect_from_filename
        except ModuleNotFoundError:
            ctx.ui.warning(_("Module \"magic\" cannot found. Falling back with \"file\" command. \
It is dangerous. So, if you wanna create stable packages, please fix \
this issue in your workplace. Probably installing \"python3-filemagic\" \
package might be a good solution."))

        for root, dirs, files in os.walk(install_dir):
            for fn in files:
                filepath = util.join_path(root, fn)
                if witcher:
                    try:
                        fileinfo=witcher(filepath).name
                    except ValueError:
                        ctx.ui.warning(_("File \"{}\" might be a broken symlink. Check it before publishing package.".format(filepath)))
                        fileinfo="broken symlink"

                    ctx.ui.info(_("\'magic\' return of \"{0}\" is \"{1}\"").format(filepath, fileinfo), verbose=True)
                else:
                    ret, out, err = util.run_batch("file {}".format(filepath), ui_debug=False)
                    if ret:
                        ctx.ui.error(_("\'file\' command failed with return code {0} for file: \"{1}\"").format(ret, filepath) +
                                     _("Output:\n{}").format(out))

                    fileinfo = str(out)
                    ctx.ui.info(_("\'file\' command return is \"{}\"").format(out), verbose=True)

                strip_debug_action(filepath, fileinfo, install_dir, self.actionGlobals)
                exclude_special_files(filepath, fileinfo, self.actionGlobals)

    def build_packages(self):
        """Build each package defined in PSPEC file. After this process there
        will be .inary files hanging around, AS INTENDED ;)"""

        doc_ptrn = re.compile(ctx.const.doc_package_end)

        self.fetch_component()  # bug 856

        # Operations and filters for package files
        self.file_actions()

        if ctx.get_option('create_static'):
            obj = self.generate_static_package_object()
            if obj:
                self.spec.packages.append(obj)

        if ctx.config.values.build.generatedebug:
            debug_packages = []
            for package in self.spec.packages:
                if "noDebug" in package.buildFlags:
                    continue

                obj = self.generate_debug_package_object(package)
                if obj:
                    debug_packages.append(obj)

            if debug_packages:
                self.spec.packages.extend(debug_packages)

        install_dir = self.pkg_install_dir()

        # Store additional files
        c = os.getcwd()
        os.chdir(self.specdir)
        for package in self.spec.packages:
            for afile in package.additionalFiles:
                src = os.path.join(ctx.const.files_dir, afile.filename)
                dest = os.path.join(
                    install_dir + os.path.dirname(afile.target),
                    os.path.basename(afile.target))
                util.copy_file(src, dest)

                # FIXME: Check that chmodding is safe for non-root builds
                if afile.permission:
                    # mode is octal!
                    os.chmod(dest, int(afile.permission, 8))

        os.chdir(c)

        # Show the files those are not collected from the install dir
        abandoned_files = self.get_abandoned_files()
        if abandoned_files:
            ctx.ui.error(_("There are abandoned files "
                           "under the install dir ({}):").format(install_dir))

            for f in abandoned_files:
                ctx.ui.info("    - {}".format(f))

            raise AbandonedFilesException

        for package in self.spec.packages:
            # removing "farce" in specfile.py:SpecFile.override_tags
            # this block of code came here... SpecFile should never
            # ever ruin the generated PSPEC file. If build process
            # needs this, we should do it in here... (bug: #3773)
            if not package.summary:
                package.summary = self.spec.source.summary
            if not package.description:
                package.description = self.spec.source.description
            if not package.partOf:
                if package.name.endswith(ctx.const.devel_package_end):
                    if self.spec.source.partOf in ctx.const.assign_to_system_devel:
                        package.partOf = ctx.const.system_devel_component
                    else:
                        package.partOf = ctx.const.devels_component
                elif re.search(doc_ptrn, package.name):
                    package.partOf = ctx.const.docs_component
                else:
                    package.partOf = self.spec.source.partOf
            if not package.license:
                package.license = self.spec.source.license
            if not package.icon:
                package.icon = self.spec.source.icon

            self.gen_files_xml(package)

            if not self.files.list:
                if not package.debug_package:
                    ctx.ui.warning(_("Ignoring empty package: \"{}\"").format(package.name))
                continue

            ctx.ui.status(_("Building package: \"{}\"").format(package.name), push_screen=True)
            self.gen_metadata_xml(package)

            name = self.package_filename(self.metadata.package)

            outdir = ctx.get_option('output_dir')
            if outdir:
                name = util.join_path(outdir, name)

            name = os.path.normpath(name)

            if package.debug_package:
                self.new_debug_packages.append(name)
            else:
                self.new_packages.append(name)

            ctx.ui.info(_("Creating \"{}\"...").format(name))

            pkg = inary.package.Package(name, "w",
                                        format=self.target_package_format,
                                        tmp_dir=self.pkg_dir())

            # add config files to package
            os.chdir(self.specdir)
            for pscom in package.providesScom:
                fname = util.join_path(ctx.const.scom_dir,
                                       pscom.script)
                pkg.add_to_package(fname)

            # add xmls and files
            os.chdir(self.pkg_dir())
            pkg.add_files_xml(ctx.const.files_xml)

            # Sort the files in-place according to their path for an ordered
            # tarfile layout which dramatically improves the compression
            # performance of lzma.
            pkg.files.list.sort(key=lambda x: x.path)

            for finfo in pkg.files.list:
                orgname = util.join_path("install", finfo.path)
                if package.debug_package:
                    orgname = util.join_path("debug", finfo.path)
                pkg.add_to_install(orgname, finfo.path)

            if self.target_package_format == "1.2":
                archive_format = ".tar.xz"
            elif self.target_package_format == "1.1":
                archive_format = ".tar.lzma"
            else:
                # "1.0" format does not have an archive
                archive_format = ""

            self.metadata.package.installTarHash = util.sha1_file(
                "{0}/install{1}".format(self.pkg_dir(), archive_format))
            self.metadata.write(util.join_path(self.pkg_dir(), ctx.const.metadata_xml))
            pkg.add_metadata_xml(ctx.const.metadata_xml)

            os.chdir(c)

            # FIXME Remove this hack
            pkg.metadata.package.debug_package = package.debug_package

            if "noDelta" not in package.buildFlags:
                delta_packages = self.build_delta_packages(pkg)
            else:
                delta_packages = []

            self.delta_map[name] = delta_packages

            pkg.close()

        self.set_state("buildpackages")

        if ctx.config.values.general.autoclean is True:
            ctx.ui.info(_("Cleaning build directory..."))
            util.clean_dir(self.pkg_dir())
        else:
            ctx.ui.info(_("Keeping build directory"))

        # reset environment variables after build.  this one is for
        # buildfarm actually. buildfarm re-inits inary for each build
        # and left environment variables go directly into initial dict
        # making actionsapi.variables.exportFlags() useless...
        os.environ.clear()
        os.environ.update(ctx.config.environ)

    def search_old_packages_for_delta(self, release=None, max_count=0,
                                      search_paths=None):
        if search_paths is None:
            search_paths = (ctx.config.compiled_packages_dir(),
                            ctx.config.debug_packages_dir())

        if release:
            self.delta_search_paths[release] = search_paths
        elif max_count > 0:
            self.delta_history_search_paths.append((search_paths, max_count))

    def build_delta_packages(self, package):

        def find_old_package(filename, search_paths):
            for package_dir in search_paths:
                path = util.join_path(package_dir, filename)
                if os.path.exists(path):
                    return path
                else:
                    path = os.path.join(package_dir,
                                        util.parse_package_dir_path(filename),
                                        filename)
                    if os.path.exists(path):
                        return path

        old_packages = {}

        for old_release, search_paths in list(self.delta_search_paths.items()):
            if old_release in old_packages:
                continue

            update = None
            for update_tag in self.spec.history[1:]:
                if update_tag.release == old_release:
                    update = update_tag
                    break
            else:
                continue

            filename = self.package_filename(package.metadata.package, update)
            old_package = find_old_package(filename, search_paths)
            if old_package:
                old_packages[old_release] = old_package

        for search_paths, max_count in self.delta_history_search_paths:
            if max_count < 1:
                continue

            found_old_packages = {}
            for update in self.spec.history[1:]:
                if update.release in old_packages:
                    continue

                filename = self.package_filename(package.metadata.package,
                                                 update)
                old_package = find_old_package(filename, search_paths)
                if old_package:
                    found_old_packages[update.release] = old_package

                    if len(found_old_packages) == max_count:
                        break

            old_packages.update(found_old_packages)

        from inary.operations.delta import create_delta_packages_from_obj
        return create_delta_packages_from_obj(list(old_packages.values()),
                                              package,
                                              self.specdir)

# build functions...

def build(pspec):
    if pspec.endswith('.xml'):
        pb = Builder(pspec)
    else:
        pb = Builder.from_name(pspec)
    try:
        pb.build()
    except ActionScriptException as e:
        ctx.ui.error(_("Action script error caught. {}").format(e))
    finally:
        if ctx.ui.errors or ctx.ui.warnings:
            ctx.ui.warning(_("*** {} error(s), {} warning(s)").format(
                             ctx.ui.errors, ctx.ui.warnings))
    return pb

order = {"none": 0,
         "fetch": 1,
         "unpack": 2,
         "setupaction": 3,
         "buildaction": 4,
         "installaction": 5,
         "buildpackages": 6}

def __buildState_fetch(pb):
    # fetch is the first state to run.
    pb.check_patches()
    pb.fetch_source_archives()

def __buildState_unpack(pb, last):
    if order[last] < order["fetch"]:
        __buildState_fetch(pb)
    pb.unpack_source_archives()

def __buildState_setupaction(pb, last):
    if order[last] < order["unpack"]:
        __buildState_unpack(pb, last)
    pb.run_setup_action()

def __buildState_buildaction(pb, last):
    if order[last] < order["setupaction"]:
        __buildState_setupaction(pb, last)
    pb.run_build_action()

def __buildState_checkaction(pb, last):
    if order[last] < order["buildaction"]:
        __buildState_buildaction(pb, last)
    pb.run_check_action()

def __buildState_installaction(pb, last):
    if order[last] < order["buildaction"]:
        __buildState_buildaction(pb, last)
    pb.run_install_action()

def __buildState_buildpackages(pb):
    for build_type in pb.build_types:
        pb.set_build_type(build_type)
        last = pb.get_state() or "none"

        if order[last] < order["installaction"]:
            __buildState_installaction(pb, last)

    pb.build_packages()

def build_until(pspec, state):
    if pspec.endswith('.xml'):
        pb = Builder(pspec)
    else:
        pb = Builder.from_name(pspec)

    pb.compile_scom_script()

    if state == "fetch":
        __buildState_fetch(pb)
        return

    # from now on build dependencies are needed
    pb.check_build_dependencies()

    if state == "package":
        __buildState_buildpackages(pb)
        return

    for build_type in pb.build_types:
        pb.set_build_type(build_type)
        last = pb.get_state() or "none"

        __build_until(pb, state, last)

def __build_until(pb, state, last):
    ctx.ui.info(_("Last state was '{}'").format(last))

    if state == "unpack":
        __buildState_unpack(pb, last)
        return

    if state == "setup":
        __buildState_setupaction(pb, last)
        return

    if state == "build":
        __buildState_buildaction(pb, last)
        return

    if state == "check":
        __buildState_checkaction(pb, last)
        return

    if state == "install":
        __buildState_installaction(pb, last)


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.

import os

import inary.context as ctx
import inary.db
import inary.util

import gettext
__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

# These guys will change due to depmod calls.
_blessed_kernel_borks = [
    "modules.alias",
    "modules.alias.bin",
    "modules.dep",
    "modules.dep.bin",
    "modules.symbols",
    "modules.symbols.bin",
]

def ignorance_is_bliss(f):
    """ Too many complaints about things that are missing. """
    p = f
    if not p.startswith("/"):
        p = "/{}".format(f)

    pbas = os.path.basename(p)
    p = p.replace("/lib64/", "/lib/")

    # Ignore kernel depmod changes?
    if p.startswith("/lib/modules"):
        if pbas in _blessed_kernel_borks:
            return True

    if p.endswith(".pyc"):
        # ignore python compiled file changes
        return True

def file_corrupted(pfile):
    path = os.path.join(ctx.config.dest_dir(), pfile.path)
    if os.path.islink(path):
        if inary.util.sha1_data(inary.util.read_link(path)) != pfile.hash:
            return True
    else:
        try:
            if inary.util.sha1_file(path) != pfile.hash:
                return True
        except inary.util.FilePermissionDeniedError as e:
            raise e
    return False

def check_files(files, check_config=False):
    results = {
        'missing': [],
        'corrupted': [],
        'denied': [],
        'config': [],
    }

    for f in files:
        if not check_config and f.type == "config":
            continue
        if not f.hash:
            continue
        if ignorance_is_bliss(f.path):
            continue

        is_file_corrupted = False

        path = os.path.join(ctx.config.dest_dir(), f.path)
        if os.path.lexists(path):
            try:
                is_file_corrupted = file_corrupted(f)

            except inary.util.FilePermissionDeniedError:
                # Can't read file, probably because of permissions, skip
                results['denied'].append(f.path)

            else:
                if is_file_corrupted:
                    # Detect file type
                    if f.type == "config":
                        results['config'].append(f.path)
                    else:
                        results['corrupted'].append(f.path)

        else:
            # Shipped file doesn't exist on the system
            results['missing'].append(f.path)

    return results

def check_config_files(package):
    config_files = inary.db.installdb.InstallDB().get_config_files(package)
    return check_files(config_files, True)

def check_package_files(package):
    files = inary.db.installdb.InstallDB().get_files(package).list
    return check_files(files)

def check(package, config=False):
    """
    Returns a dictionary that contains a list of both corrupted and missing files
    @param package: name of the package to be checked
    @param config: _only_ check the config files of the package, default behaviour is to check all the files
    of the package but the config files
    """
    if config:
        return check_config_files(package)
    else:
        return check_package_files(package)