code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import re
import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

import inary.db.repodb
import inary.db.itembyrepo
import inary.data.component as Component
import inary.db.lazydb as lazydb
from inary.sxml import autoxml, xmlext

class ComponentDB(lazydb.LazyDB):

    def __init__(self):
        lazydb.LazyDB.__init__(self, cacheable=True)

    def init(self):
        component_nodes = {}
        component_packages = {}
        component_sources = {}

        repodb = inary.db.repodb.RepoDB()

        for repo in repodb.list_repos():
            doc = repodb.get_repo_doc(repo)
            component_nodes[repo] = self.__generate_components(doc)
            component_packages[repo] = self.__generate_packages(doc)
            component_sources[repo] = self.__generate_sources(doc)

        self.cdb = inary.db.itembyrepo.ItemByRepo(component_nodes)
        self.cpdb = inary.db.itembyrepo.ItemByRepo(component_packages)
        self.csdb = inary.db.itembyrepo.ItemByRepo(component_sources)

    @staticmethod
    def __generate_packages(doc):
        components = {}
        packages = xmlext.getTagByName(doc, "Package")
        for pkg in packages:
            name = xmlext.getNodeText(pkg, "Name")
            partof = xmlext.getNodeText(pkg, "PartOf")
            components.setdefault(partof, []).append(name)

        return components

    @staticmethod
    def __generate_sources(doc):
        components = {}
        specfile = xmlext.getTagByName(doc, "SpecFile")
        for spec in specfile:
            source = xmlext.getNode(spec, "Source")
            name = xmlext.getNodeText(source, "Name")
            partof = xmlext.getNodeText(source, "PartOf")
            components.setdefault(partof, []).append(name)

        return components

    @staticmethod
    def __generate_components(doc):
        components = {}
        component = xmlext.getTagByName(doc, "Component")
        for comp in component:
            name = xmlext.getNodeText(comp, "Name")
            components[name] = xmlext.toString(comp)

        return components

    def has_component(self, name, repo=None):
        return self.cdb.has_item(name, repo)

    def list_components(self, repo=None):
        return self.cdb.get_item_keys(repo)

    def search_component(self, terms, lang=None, repo=None):
        rename = '<LocalName xml:lang="({0}|en)">.*?{1}.*?</LocalName>'
        resum = '<Summary xml:lang="({0}|en)">.*?{1}.*?</Summary>'
        redesc = '<Description xml:lang="({0}|en)">.*?{1}.*?</Description>'

        if not lang:
            lang = autoxml.LocalText.get_lang()
        found = []
        for name, xml in self.cdb.get_items_iter(repo):
            if name not in found and terms == [term for term in terms if
                                               re.compile(rename.format(lang, term), re.I).search(xml) or \
                                               re.compile(resum.format(lang, term), re.I).search(xml) or \
                                               re.compile(redesc.format(lang, term), re.I).search(xml)]:
                found.append(name)
        return found

    # Returns the component in given repo or first found component in repo order if repo is None
    def get_component(self, component_name, repo=None):

        if not self.has_component(component_name, repo):
            raise Exception(_('Component {} not found.').format(component_name))

        component = Component.Component()
        component.parse(self.cdb.get_item(component_name, repo))

        try:
            component.packages = self.cpdb.get_item(component_name, repo)
        except Exception:  # FIXME: what exception could we catch here, replace with that.
            pass

        try:
            component.sources = self.csdb.get_item(component_name, repo)
        except Exception:  # FIXME: what exception could we catch here, replace with that.
            pass

        return component

    # Returns the component with combined packages and sources from all repos that contain this component
    def get_union_component(self, component_name):

        component = Component.Component()
        component.parse(self.cdb.get_item(component_name))

        for repo in inary.db.repodb.RepoDB().list_repos():
            try:
                component.packages.extend(self.cpdb.get_item(component_name, repo))
            except Exception:  # FIXME: what exception could we catch here, replace with that.
                pass

            try:
                component.sources.extend(self.csdb.get_item(component_name, repo))
            except Exception:  # FIXME: what exception could we catch here, replace with that.
                pass

        return component

    # Returns packages of given component from given repo or first found component's packages in repo
    # order if repo is None.
    # If walk is True than also the sub components' packages are returned
    def get_packages(self, component_name, repo=None, walk=False):

        component = self.get_component(component_name, repo)
        if not walk:
            return component.packages

        packages = []
        packages.extend(component.packages)

        sub_components = [x for x in self.list_components(repo) if x.startswith(component_name + ".")]
        for sub in sub_components:
            try:
                packages.extend(self.get_component(sub, repo).packages)
            except Exception:  # FIXME: what exception could we catch here, replace with that.
                pass

        return packages

    # Returns the component with combined packages and sources from all repos that contain this component
    # If walk is True than also the sub components' packages from all repos are returned
    def get_union_packages(self, component_name, walk=False):

        component = self.get_union_component(component_name)
        if not walk:
            return component.packages

        packages = []
        packages.extend(component.packages)

        sub_components = [x for x in self.list_components() if x.startswith(component_name + ".")]
        for sub in sub_components:
            try:
                packages.extend(self.get_union_component(sub).packages)
            except Exception:  # FIXME: what exception could we catch here, replace with that.
                pass

        return packages

    # Returns sources of given component from given repo or first found component's packages in repo
    # order if repo is None.
    # If walk is True than also the sub components' packages are returned
    def get_sources(self, component_name, repo=None, walk=False):

        component = self.get_component(component_name, repo)
        if not walk:
            return component.sources

        sources = []
        sources.extend(component.sources)

        sub_components = [x for x in self.list_components(repo) if x.startswith(component_name + ".")]
        for sub in sub_components:
            try:
                sources.extend(self.get_component(sub, repo).sources)
            except Exception:  # FIXME: what exception could we catch here, replace with that.
                pass

        return sources

    # Returns the component with combined packages and sources from all repos that contain this component
    # If walk is True than also the sub components' sources from all repos are returned
    def get_union_sources(self, component_name, repo=None, walk=False):

        component = self.get_union_component(component_name)
        if not walk:
            return component.sources

        sources = []
        sources.extend(component.sources)

        sub_components = [x for x in self.list_components(repo) if x.startswith(component_name + ".")]
        for sub in sub_components:
            try:
                sources.extend(self.get_union_component(sub).sources)
            except Exception:  # FIXME: what exception could we catch here, replace with that.
                pass

        return sources


code:
# -*- coding: utf-8 -*-
#
# Copyright (C) 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import hashlib
import os
import re
import shelve

import inary.context as ctx
import inary.db
import inary.db.lazydb as lazydb
import inary.util

import gettext
__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

# FIXME:
# We could traverse through files.xml files of the packages to find the path and
# the package - a linear search - as some well known package managers do. But the current
# file conflict mechanism of inary prevents this and needs a fast has_file function.
# So currently filesdb is the only db and we cant still get rid of rebuild-db :/

class FilesDB(lazydb.LazyDB):

    def __init__(self):
        self.filesdb = {}
        self.__check_filesdb()

    def __del__(self):
        self.close()

    def create_filesdb(self):
        ctx.ui.info(_('Creating files database...'), color='blue')
        installdb = inary.db.installdb.InstallDB()
        for pkg in installdb.list_installed():
            files = installdb.get_files(pkg)
            self.add_files(pkg, files)
            ctx.ui.info("%-80.80s\r" % _(' -> Adding files of \"{}\" package to db...').format(pkg), noln=True, color='brightpurple')
        ctx.ui.info(_('\nAdded files database...'), color='green')

    def get_file(self, path):
        key = hashlib.md5(path.encode('utf-8')).hexdigest()
        return self.filesdb.get(key), path

    def has_file(self, path):
        key = hashlib.md5(path.encode('utf-8')).hexdigest()
        if key in self.filesdb:
            return True
        else:
            return False

    def search_file(self, term):
        pkg, path = self.get_file(term)
        if pkg:
            return [(pkg, [path])]

        installdb = inary.db.installdb.InstallDB()
        found = []
        for pkg in installdb.list_installed():
            files_xml = open(os.path.join(installdb.package_path(pkg), ctx.const.files_xml)).read()
            paths = re.compile('<Path>(.*?%s.*?)</Path>' % re.escape(term), re.I).findall(files_xml)
            if paths:
                found.append((pkg, paths))
        return found

    def add_files(self, pkg, files):
        self.__check_filesdb()

        for f in files.list:
            key = hashlib.md5(f.path.encode('utf-8')).hexdigest()
            self.filesdb[key] = pkg

    def remove_files(self, files):
        for f in files:
            key = hashlib.md5(f.path.encode('utf-8')).hexdigest()
            if key in self.filesdb:
                del self.filesdb[key]

    @staticmethod
    def destroy():
        files_db = os.path.join(ctx.config.info_dir(), ctx.const.files_db)
        if os.path.exists(files_db):
            os.unlink(files_db)

    def close(self):
        if isinstance(self.filesdb, shelve.DbfilenameShelf):
            self.filesdb.close()

    def __check_filesdb(self):
        if isinstance(self.filesdb, shelve.DbfilenameShelf):
            return

        files_db = os.path.join(ctx.config.info_dir(), ctx.const.files_db)

        if not os.path.exists(files_db):
            flag = "n"

        elif os.access(files_db, os.W_OK):
            flag = "w"
        else:
            flag = "r"

        self.filesdb = shelve.open(files_db, flag)
        if flag == "n":
            self.create_filesdb()

    def update(self):
        files_db = os.path.join(ctx.config.info_dir(), ctx.const.files_db)
        os.remove(files_db)
        self.__check_filesdb()

@inary.util.locked
def rebuild_db():
    # save parameters and shutdown inary
    options = ctx.config.options
    ui = ctx.ui
    scom = ctx.scom
    from inary import _cleanup
    _cleanup()

    ctx.filesdb.close()
    ctx.filesdb.destroy()
    ctx.filesdb = inary.db.filesdb.FilesDB()
    ctx.filesdb.update()

    # reinitialize everything
    ctx.ui = ui
    ctx.config.set_options(options)
    ctx.scom = scom

repo_name: SulinOS/inary

license: mit


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

import inary.db.repodb
import inary.db.itembyrepo
import inary.data.group as Group
import inary.db.lazydb as lazydb
from inary.sxml import xmlext

class GroupNotFound(Exception):
    pass

class GroupDB(lazydb.LazyDB):

    def __init__(self):
        lazydb.LazyDB.__init__(self, cacheable=True)

    def init(self):
        group_nodes = {}
        group_components = {}

        repodb = inary.db.repodb.RepoDB()

        for repo in repodb.list_repos():
            doc = repodb.get_repo_doc(repo)
            group_nodes[repo] = self.__generate_groups(doc)
            group_components[repo] = self.__generate_components(doc)

        self.gdb = inary.db.itembyrepo.ItemByRepo(group_nodes)
        self.gcdb = inary.db.itembyrepo.ItemByRepo(group_components)

    @staticmethod
    def __generate_components(doc):
        groups = {}
        components = xmlext.getTagByName(doc, "Component")
        for comp in components:
            group = xmlext.getNodeText(comp, "Group")
            name = xmlext.getNodeText(comp, "Name")
            groups.setdefault(group, []).append(name)

        return groups

    @staticmethod
    def __generate_groups(doc):
        groups = {}
        group = xmlext.getTagByName(doc, "Group")
        for gr in group:
            name = xmlext.getNodeText(gr, "Name")
            groups[name] = xmlext.toString(gr)

        return groups

    def has_group(self, name, repo=None):
        return self.gdb.has_item(name, repo)

    def list_groups(self, repo=None):
        return self.gdb.get_item_keys(repo)

    def get_group(self, name, repo=None):

        if not self.has_group(name, repo):
            raise GroupNotFound(_('Group \"{}\" not found.').format(name))

        group = Group.Group()
        group.parse(self.gdb.get_item(name, repo))

        return group

    def get_group_components(self, name, repo=None):
        if not self.has_group(name, repo):
            raise GroupNotFound(_('Group \"{}\" not found.').format(name))

        if self.gcdb.has_item(name):
            return self.gcdb.get_item(name, repo)

        return []


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import os

import inary.context as ctx
import inary.data.history as History
import inary.db.lazydb as lazydb
import inary.util

class HistoryDB(lazydb.LazyDB):

    # noinspection PyArgumentList
    def init(self):
        self.__logs = self.__generate_history()
        self.history = History.History()

    @staticmethod
    def __generate_history():
        logs = [x for x in os.listdir(ctx.config.history_dir()) if x.endswith(".xml")]
        # logs.sort(key=lambda x,y:int(x.split("_")[0]) - int(y.split("_")[0]))
        logs.sort(key=lambda x: int(x.split("_")[0].replace("0o", "0")))
        logs.reverse()
        return logs

    def create_history(self, operation):
        self.history.create(operation)

    def add_and_update(self, pkgBefore=None, pkgAfter=None, operation=None, otype=None):
        self.add_package(pkgBefore, pkgAfter, operation, otype)
        self.update_history()

    def add_package(self, pkgBefore=None, pkgAfter=None, operation=None, otype=None):
        self.history.add(pkgBefore, pkgAfter, operation, otype)

    @staticmethod
    def load_config(operation, package):
        config_dir = os.path.join(ctx.config.history_dir(), "%03d" % operation, package)
        if os.path.exists(config_dir):
            import distutils.dir_util as dir_util
            dir_util.copy_tree(config_dir, "/")

    def save_config(self, package, config_file):
        hist_dir = os.path.join(ctx.config.history_dir(), self.history.operation.no, package)
        if os.path.isdir(config_file):
            os.makedirs(os.path.join(hist_dir, config_file))
            return

        destdir = os.path.join(hist_dir, config_file[1:])
        inary.util.copy_file_stat(config_file, destdir)

    def update_repo(self, repo, uri, operation=None):
        self.history.update_repo(repo, uri, operation)
        self.update_history()

    def update_history(self):
        self.history.update()

    def get_operation(self, operation):
        for log in self.__logs:
            if log.startswith("%03d_" % operation):
                hist = History.History(os.path.join(ctx.config.history_dir(), log))
                hist.operation.no = int(log.split("_")[0].replace("0o", "0"))
                return hist.operation
        return None

    @staticmethod
    def get_package_config_files(operation, package):
        package_path = os.path.join(ctx.config.history_dir(), "%03d/%s" % (operation, package))
        if not os.path.exists(package_path):
            return None

        configs = []
        for root, dirs, files in os.walk(package_path):
            for f in files:
                configs.append(("{0}/{1}".format(root, f)))

        return configs

    def get_config_files(self, operation):
        config_path = os.path.join(ctx.config.history_dir(), "%03d" % operation)
        if not os.path.exists(config_path):
            return None

        allconfigs = {}
        packages = os.listdir(config_path)
        for package in packages:
            allconfigs[package] = self.get_package_config_files(operation, package)

        return allconfigs

    def get_till_operation(self, operation):
        if not [x for x in self.__logs if x.startswith("%03d_" % operation)]:
            return

        for log in self.__logs:
            if log.startswith("%03d_" % operation):
                return

            hist = History.History(os.path.join(ctx.config.history_dir(), log))
            hist.operation.no = int(log.split("_")[0].replace("0o", "0"))
            yield hist.operation

    def get_last(self, count=0):
        count = count or len(self.__logs)
        for log in self.__logs[:count]:
            hist = History.History(os.path.join(ctx.config.history_dir(), log))
            hist.operation.no = int(log.split("_")[0].replace("0o", "0"))
            yield hist.operation

    def get_last_repo_update(self, last=1):
        repoupdates = [l for l in self.__logs if l.endswith("repoupdate.xml")]
        repoupdates.reverse()
        if not len(repoupdates) >= 2:
            return None

        if last != 1 and len(repoupdates) <= last:
            return None

        hist = History.History(os.path.join(ctx.config.history_dir(), repoupdates[-last]))
        return hist.operation.date


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#
#
# installation database
#

import os
import re
import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

# INARY
import inary.analyzer.dependency
import inary.context as ctx
import inary.data
import inary.data.files as Files
import inary.db.lazydb as lazydb
from inary.sxml import autoxml, xmlext
import inary.util

class InstallDBError(inary.errors.Error):
    pass

class InstallInfo:
    state_map = {'i': _('installed'), 'ip': _('installed-pending')}

    def __init__(self, state, version, release, distribution, time):
        self.state = state
        self.version = version
        self.release = release
        self.distribution = distribution
        self.time = time

    def one_liner(self):
        import time
        time_str = time.strftime("%d %b %Y %H:%M", self.time)
        s = '%2s|%15s|%6s|%8s|%12s' % (self.state, self.version, self.release,
                                       self.distribution, time_str)
        return s

    def __str__(self):
        s = _("State: {0}\nVersion: {1}, Release: {2}\n").format(
            InstallInfo.state_map[self.state], self.version, self.release)
        import time
        time_str = time.strftime("%d %b %Y %H:%M", self.time)
        s += _('Distribution: {0}, Install Time: {1}\n').format(self.distribution,
                                                                time_str)
        return s

class InstallDB(lazydb.LazyDB):

    def __init__(self):
        lazydb.LazyDB.__init__(self, cacheable=True, cachedir=ctx.config.packages_dir())
        # self.init()

    def init(self):
        self.installed_db = self.__generate_installed_pkgs()
        self.rev_deps_db = self.__generate_revdeps()
        self.installed_extra = self.__generate_installed_extra()

    @staticmethod
    def __generate_installed_extra():
        ie = []
        ie_path = os.path.join(ctx.config.info_dir(), ctx.const.installed_extra)
        if os.path.isfile(ie_path):
            with open(ie_path) as ie_file:
                ie.extend(ie_file.read().strip().split("\n"))
        return ie

    @staticmethod
    def __generate_installed_pkgs():

        def split_name(dirname):
            name, version, release = dirname.rsplit("-", 2)
            return name, version + "-" + release

        return dict(list(map(split_name, os.listdir(ctx.config.packages_dir()))))

    @staticmethod
    def __get_marked_packages(_type):
        info_path = os.path.join(ctx.config.info_dir(), _type)
        if os.path.exists(info_path):
            return open(info_path).read().split()
        return []

    def __add_to_revdeps(self, package, revdeps):
        metadata_xml = os.path.join(self.package_path(package), ctx.const.metadata_xml)
        meta_doc = xmlext.parse(metadata_xml)

        try:
            pkg = xmlext.getNode(meta_doc, "Package")
        except:
            pkg = None

        if pkg is None:
            # If package info is broken or not available, skip it.
            ctx.ui.warning(_("Installation info for package \"{}\" is broken. "
                             "Reinstall it to fix this problem.").format(package))
            del self.installed_db[package]
            return

        deps = xmlext.getNode(pkg, 'RuntimeDependencies')
        if deps:
            for dep in xmlext.getTagByName(deps, 'Dependency'):
                revdep = revdeps.setdefault(xmlext.getNodeText(dep), {})
                revdep[package] = xmlext.toString(dep)

            for anydep in xmlext.getTagByName(deps, 'AnyDependency'):
                for dep in xmlext.getTagByName(anydep, 'Dependency'):
                    revdep = revdeps.setdefault(xmlext.getNodeText(dep), {})
                    revdep[package] = xmlext.toString(anydep)

    def __generate_revdeps(self):
        revdeps = {}
        for package in self.list_installed():
            self.__add_to_revdeps(package, revdeps)
        return revdeps

    def list_installed(self):
        return list(self.installed_db.keys())

    def has_package(self, package):
        return package in self.installed_db

    def list_installed_with_build_host(self, build_host):
        build_host_re = re.compile("<BuildHost>(.*?)</BuildHost>")
        found = []
        for name in self.list_installed():
            xml = open(os.path.join(self.package_path(name), ctx.const.metadata_xml)).read()
            matched = build_host_re.search(xml)
            if matched:
                if build_host != matched.groups()[0]:
                    continue
            elif build_host:
                continue

            found.append(name)

        return found

    @staticmethod
    def __get_version(meta_doc):
        package = xmlext.getNode(meta_doc, 'Package')
        history = xmlext.getNode(package, 'History')
        update = xmlext.getNode(history, 'Update')

        version = xmlext.getNodeText(update, 'Version')
        release = xmlext.getNodeAttribute(update, 'release')

        return version, release, None

    @staticmethod
    def __get_distro_release(meta_doc):
        package = xmlext.getNode(meta_doc, 'Package')
        distro = xmlext.getNodeText(package, 'Distribution')
        release = xmlext.getNodeText(package, 'DistributionRelease')

        return distro, release

    @staticmethod
    def __get_install_tar_hash(meta_doc):
        package = xmlext.getNode(meta_doc, 'Package')
        hash = xmlext.getNodeText(package, 'InstallTarHash')
        return hash

    def get_install_tar_hash(self, package):
        metadata_xml = os.path.join(self.package_path(package), ctx.const.metadata_xml)

        meta_doc = xmlext.parse(metadata_xml)

        return self.__get_install_tar_hash(meta_doc)

    def get_version_and_distro_release(self, package):
        metadata_xml = os.path.join(self.package_path(package), ctx.const.metadata_xml)

        meta_doc = xmlext.parse(metadata_xml)

        return self.__get_version(meta_doc) + self.__get_distro_release(meta_doc)

    def get_version(self, package):
        metadata_xml = os.path.join(self.package_path(package), ctx.const.metadata_xml)

        meta_doc = xmlext.parse(metadata_xml)

        return self.__get_version(meta_doc)

    def get_files(self, package):
        files = Files.Files()
        files_xml = os.path.join(self.package_path(package), ctx.const.files_xml)
        files.read(files_xml)
        return files

    def get_config_files(self, package):
        files = self.get_files(package)
        return [x for x in files.list if x.type == 'config']

    def search_package(self, terms, lang=None, fields=None, cs=False):
        """
        fields (dict) : looks for terms in the fields which are marked as True
        If the fields is equal to None this method will search in all fields

        example :
        if fields is equal to : {'name': True, 'summary': True, 'desc': False}
        This method will return only package that contents terms in the package
        name or summary
        """
        resum = '<Summary xml:lang=.({0}|en).>.*?{1}.*?</Summary>'
        redesc = '<Description xml:lang=.({0}|en).>.*?{1}.*?</Description>'
        if not fields:
            fields = {'name': True, 'summary': True, 'desc': True}
        if not lang:
            lang = autoxml.LocalText.get_lang()
        found = []
        for name in self.list_installed():
            xml = open(os.path.join(self.package_path(name), ctx.const.metadata_xml)).read()
            if terms == [term for term in terms if (fields['name'] and
                                                    re.compile(term, re.I).search(name)) or \
                                                   (fields['summary'] and
                                                    re.compile(resum.format(lang, term), 0 if cs else re.I).search(
                                                        xml)) or \
                                                   (fields['desc'] and
                                                    re.compile(redesc.format(lang, term), 0 if cs else re.I).search(
                                                        xml))]:
                found.append(name)
        return found

    def get_isa_packages(self, isa):
        risa = '<IsA>{}</IsA>'.format(isa)
        packages = []
        for name in self.list_installed():
            xml = open(os.path.join(self.package_path(name), ctx.const.metadata_xml)).read()
            if re.compile(risa).search(xml):
                packages.append(name)
        return packages

    def get_info(self, package):
        files_xml = os.path.join(self.package_path(package), ctx.const.files_xml)
        ctime = inary.util.creation_time(files_xml)
        pkg = self.get_package(package)
        state = "i"
        if pkg.name in self.list_pending():
            state = "ip"

        info = InstallInfo(state,
                           pkg.version,
                           pkg.release,
                           pkg.distribution,
                           ctime)
        return info

    @staticmethod
    def __make_dependency(depStr):

        node = xmlext.parseString(depStr)

        dependency = inary.analyzer.dependency.Dependency()
        dependency.package = xmlext.getNodeText(node)

        if xmlext.getAttributeList(node):
            if xmlext.getNodeAttribute(node, "version"):
                dependency.__dict__["version"] = xmlext.getNodeAttribute(node, "version")
            elif xmlext.getNodeAttribute(node, "release"):
                dependency.__dict__["release"] = xmlext.getNodeAttribute(node, "release")
            else:
                pass #FIXME: ugly
        return dependency

    def __create_dependency(self, depStr):
        if "<AnyDependency>" in depStr:
            anydependency = inary.data.specfile.AnyDependency()
            for dep in re.compile('(<Dependency .*?>.*?</Dependency>)').findall(depStr):
                anydependency.dependencies.append(self.__make_dependency(dep))
            return anydependency
        else:
            return self.__make_dependency(depStr)

    def get_rev_deps(self, name):
        rev_deps = []

        package_revdeps = self.rev_deps_db.get(name)
        if package_revdeps:
            for pkg, dep in list(package_revdeps.items()):
                dependency = self.__create_dependency(dep)
                rev_deps.append((pkg, dependency))

        return rev_deps

    def get_orphaned(self):
        """
        get list of packages installed as extra dependency,
        but without reverse dependencies now.
        """
        orphaned_packages=[]
        for x in self.installed_extra:
            if not self.get_rev_deps(x):

                if x.endswith(ctx.const.doc_package_end) and ctx.config.values.general.allow_docs:
                    if inary.util.remove_suffix(ctx.const.doc_package_end, x) not in self.list_installed():
                        orphaned_packages.append(x)
                    else:
                        pass

                elif x.endswith(ctx.const.info_package_end) and ctx.config.values.general.allow_pages:
                    if inary.util.remove_suffix(ctx.const.info_package_end, x) not in self.list_installed():
                        orphaned_packages.append(x)
                    else:
                        pass

                elif x.endswith(ctx.const.debug_name_suffix) and ctx.config.values.general.allow_dbginfo:
                    if inary.util.remove_suffix(ctx.const.debug_name_suffix, x) not in self.list_installed():
                        orphaned_packages.append(x)
                    else:
                        pass

                elif x.endswith(ctx.const.static_name_suffix) and ctx.config.values.general.allow_static:
                    if inary.util.remove_suffix(ctx.const.static_name_suffix, x) not in self.list_installed():
                        orphaned_packages.append(x)
                    else:
                        pass

                else:
                    orphaned_packages.append(x)

        return orphaned_packages

    def get_no_rev_deps(self):
        """
        get installed packages list which haven't reverse dependencies.
        """
        return [x for x in self.installed_db if not self.get_rev_deps(x)]

    @staticmethod
    def pkg_dir(pkg, version, release):
        return inary.util.join_path(ctx.config.packages_dir(), pkg + '-' + version + '-' + release)

    def get_package(self, package):
        metadata = inary.data.metadata.MetaData()
        metadata_xml = os.path.join(self.package_path(package), ctx.const.metadata_xml)
        metadata.read(metadata_xml)
        return metadata.package

    def __mark_package(self, _type, package):
        packages = self.__get_marked_packages(_type)
        if package not in packages:
            packages.append(package)
            self.__write_marked_packages(_type, packages)

    def mark_pending(self, package):
        self.__mark_package(ctx.const.config_pending, package)

    def mark_needs_restart(self, package):
        self.__mark_package(ctx.const.needs_restart, package)

    def mark_needs_reboot(self, package):
        self.__mark_package(ctx.const.needs_reboot, package)

    def add_package(self, pkginfo):
        # Cleanup old revdep info
        for revdep_info in list(self.rev_deps_db.values()):
            if pkginfo.name in revdep_info:
                del revdep_info[pkginfo.name]

        self.installed_db[pkginfo.name] = "{0.version}-{0.release}".format(pkginfo)
        self.__add_to_revdeps(pkginfo.name, self.rev_deps_db)

    def remove_package(self, package_name):
        if package_name in self.installed_db:
            del self.installed_db[package_name]

        # Cleanup revdep info
        for revdep_info in list(self.rev_deps_db.values()):
            if package_name in revdep_info:
                del revdep_info[package_name]

        self.clear_pending(package_name)

    def list_pending(self):
        return self.__get_marked_packages(ctx.const.config_pending)

    def list_needs_restart(self):
        return self.__get_marked_packages(ctx.const.needs_restart)

    def list_needs_reboot(self):
        return self.__get_marked_packages(ctx.const.needs_reboot)

    @staticmethod
    def __write_marked_packages(_type, packages):
        info_file = os.path.join(ctx.config.info_dir(), _type)
        config = open(info_file, "w")
        for pkg in set(packages):
            config.write("{}\n".format(pkg))
        config.close()

    def __clear_marked_packages(self, _type, package):
        if package == "*":
            self.__write_marked_packages(_type, [])
            return
        packages = self.__get_marked_packages(_type)
        if package in packages:
            packages.remove(package)
            self.__write_marked_packages(_type, packages)

    def clear_pending(self, package):
        self.__clear_marked_packages(ctx.const.config_pending, package)

    def clear_needs_restart(self, package):
        self.__clear_marked_packages(ctx.const.needs_restart, package)

    def clear_needs_reboot(self, package):
        self.__clear_marked_packages(ctx.const.needs_reboot, package)

    def package_path(self, package):

        if package in self.installed_db:
            return os.path.join(ctx.config.packages_dir(), "{0}-{1}".format(package, self.installed_db[package]))

        raise Exception(_('Package \"{}\" is not installed.').format(package))


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import gzip
import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

import inary.db

class ItemByRepo:
    def __init__(self, dbobj, compressed=False):
        self.dbobj = dbobj
        self.compressed = compressed

    def has_repo(self, repo):
        return repo in self.dbobj

    def has_item(self, item, repo=None):
        for r in self.item_repos(repo):
            if r in self.dbobj and item in self.dbobj[r]:
                return True

        return False

    def which_repo(self, item):
        for r in inary.db.repodb.RepoDB().list_repos():
            if r in self.dbobj and item in self.dbobj[r]:
                return r

        raise Exception(_("\"{}\" not found in any repository.").format(str(item)))

    def get_item_repo(self, item, repo=None):
        for r in self.item_repos(repo):
            if r in self.dbobj and item in self.dbobj[r]:
                if self.compressed:
                    return gzip.zlib.decompress(self.dbobj[r][item]).decode('utf-8'), r
                else:
                    return self.dbobj[r][item], r

        raise Exception(_("Repo item \"{}\" not found.").format(str(item)))

    def get_item(self, item, repo=None):
        item, repo = self.get_item_repo(item, repo)
        return item

    def get_item_keys(self, repo=None):
        items = []
        for r in self.item_repos(repo):
            if not self.has_repo(r):
                raise Exception(_('Repository \"{}\" does not exist.').format(repo))

            if r in self.dbobj:
                items.extend(list(self.dbobj[r].keys()))

        return list(set(items))

    def get_list_item(self, repo=None):
        items = []
        for r in self.item_repos(repo):
            if not self.has_repo(r):
                raise Exception(_('Repository \"{}\" does not exist.').format(repo))

            if r in self.dbobj:
                items.extend(self.dbobj[r])

        return list(set(items))

    def get_items_iter(self, repo=None):
        for r in self.item_repos(repo):
            if not self.has_repo(r):
                raise Exception(_('Repository \"{}\" does not exist.').format(repo))

            if self.compressed:
                for item in list(self.dbobj[r].keys()):
                    yield str(item), gzip.zlib.decompress(self.dbobj[r][item]).decode('utf-8')
            else:
                for item in list(self.dbobj[r].keys()):
                    yield str(item), str(self.dbobj[r][item])

    def item_repos(self, repo=None):
        if repo:
            repos = [repo]
            return repos
        else:
            return inary.db.repodb.RepoDB().list_repos()


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import os
import pickle
import time

import inary.context as ctx
import inary.util as util

# lower borks for international locales. What we want is ascii lower.
lower_map = str.maketrans(util.ascii_uppercase, util.ascii_lowercase)

class Singleton(object):
    _the_instances = {}

    def __new__(type):
        if not type.__name__ in Singleton._the_instances:
            Singleton._the_instances[type.__name__] = object.__new__(type)
        return Singleton._the_instances[type.__name__]

    def _instance(self):
        return self._the_instances[type(self).__name__]

    def _delete(self):
        # FIXME: After invalidate, previously initialized db object becomes
        ctx.ui.debug("LazyDB: {0} invalidated.".format(self.__class__.__name__))
        del self._the_instances[type(self).__name__]

class LazyDB(Singleton):
    cache_version = "1.0"

    def __init__(self, cacheable=False, cachedir=None):
        if "initialized" not in self.__dict__:
            self.initialized = False
        self.cacheable = cacheable
        self.cachedir = cachedir

    def is_initialized(self):
        return self.initialized

    def __cache_file(self):
        return util.join_path(ctx.config.cache_root_dir(),
                              "{}.cache".format(self.__class__.__name__.translate(lower_map)))

    def __cache_version_file(self):
        return "{}.version".format(self.__cache_file())

    def __cache_file_version(self):
        try:
            return open(self.__cache_version_file()).read().strip()
        except IOError:
            return "2.2"

    def cache_save(self):
        if os.access(ctx.config.cache_root_dir(), os.W_OK) and self.cacheable:
            with open(self.__cache_version_file(), "w") as f:
                f.write(LazyDB.cache_version)
                f.flush()
                os.fsync(f.fileno())
            pickle.dump(self._instance().__dict__,
                        open(self.__cache_file(), 'wb'), 1)

            ctx.ui.debug("LazyDB: {0} cached.".format(self.__class__.__name__))

    def cache_valid(self):
        if not self.cachedir:
            return True
        if not os.path.exists(self.cachedir):
            return False
        if self.__cache_file_version() != LazyDB.cache_version:
            return False

        cache_modified = os.stat(self.__cache_file()).st_mtime
        cache_dir_modified = os.stat(self.cachedir).st_mtime
        return cache_modified > cache_dir_modified

    def cache_load(self):
        if os.path.exists(self.__cache_file()) and self.cache_valid():
            try:
                self._instance().__dict__ = pickle.load(open(self.__cache_file(), 'rb'))
                return True
            except (pickle.UnpicklingError, EOFError):
                if os.access(ctx.config.cache_root_dir(), os.W_OK):
                    os.unlink(self.__cache_file())
                return False
        return False

    def cache_flush(self):
        if os.path.exists(self.__cache_file()):
            os.unlink(self.__cache_file())
            ctx.ui.debug("LazyDB: {0} cached.".format(self.__class__.__name__))

    def invalidate(self):
        self._delete()

    def cache_regenerate(self):
        try:
            self.this_attr_does_not_exist()
        except AttributeError:
            pass

    def __init(self):
        if not self.cache_load():
            self.init()

    def __getattr__(self, attr):
        if not attr == "__setstate__" and not self.initialized:
            start = time.time()
            self.__init()
            end = time.time()
            ctx.ui.debug("{0} initialized in {1}.".format(self.__class__.__name__, end - start))
            self.initialized = True

        if attr not in self.__dict__:
            raise AttributeError(attr)

        return self.__dict__[attr]


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import datetime
import gzip
import re
import time

import inary.analyzer.dependency
import inary.data.metadata
import inary.db
import inary.db.itembyrepo
import inary.db.lazydb as lazydb
from inary.sxml import xmlext

import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

class PackageDB(lazydb.LazyDB):

    def __init__(self):
        lazydb.LazyDB.__init__(self, cacheable=True)
        # self.init()

    def init(self):
        self.__package_nodes = {}  # Packages
        self.__revdeps = {}  # Reverse dependencies
        self.__obsoletes = {}  # Obsoletes
        self.__replaces = {}  # Replaces

        repodb = inary.db.repodb.RepoDB()

        for repo in repodb.list_repos():
            doc = repodb.get_repo_doc(repo)
            self.__package_nodes[repo] = self.__generate_packages(doc)
            self.__revdeps[repo] = self.__generate_revdeps(doc)
            self.__obsoletes[repo] = self.__generate_obsoletes(doc)
            self.__replaces[repo] = self.__generate_replaces(doc)

        self.pdb = inary.db.itembyrepo.ItemByRepo(self.__package_nodes, compressed=True)
        self.rvdb = inary.db.itembyrepo.ItemByRepo(self.__revdeps)
        self.odb = inary.db.itembyrepo.ItemByRepo(self.__obsoletes)
        self.rpdb = inary.db.itembyrepo.ItemByRepo(self.__replaces)

    ## Generate functions look sooo ugly
    @staticmethod
    def __generate_replaces(doc):
        replaces = []
        packages = xmlext.getTagByName(doc, "Package")
        for node in packages:
            if xmlext.getNodeText(node, "Replaces"):
                replaces.append(xmlext.getNodeText(node, "Name"))
        return replaces

    @staticmethod
    def __generate_obsoletes(doc):
        distribution = xmlext.getNode(doc, "Distribution")
        obsoletes = distribution and xmlext.getNode(distribution, "Obsoletes")
        src_repo = xmlext.getNode(doc, "SpecFile") is not None

        if not obsoletes or src_repo:
            return []
        return [xmlext.getNodeText(x) for x in xmlext.getTagByName(obsoletes, "Package")]

    @staticmethod
    def __generate_packages(doc):
        pdict = {}

        for x in xmlext.getTagByName(doc, "Package"):
            name = xmlext.getNodeText(x, "Name")
            compressed_data = gzip.zlib.compress(xmlext.toString(x).encode('utf-8'))
            pdict[name] = compressed_data

        return pdict

    @staticmethod
    def __generate_revdeps(doc):
        revdeps = {}
        for node in xmlext.getTagByName(doc, "Package"):
            name = xmlext.getNodeText(node, 'Name')
            deps = xmlext.getNode(node, 'RuntimeDependencies')
            if deps:
                for dep in xmlext.getTagByName(deps, "Dependency"):
                    revdeps.setdefault(xmlext.getNodeText(dep), set()).add((name, xmlext.toString(dep)))

        return revdeps

    def has_package(self, name, repo=None):
        return self.pdb.has_item(name, repo)

    def get_package(self, name, repo=None):
        pkg, repo = self.get_package_repo(name, repo)
        return pkg

    def search_in_packages(self, packages, terms, lang=None):
        resum = '.*?{1}.*?'
        redesc = '<Description xml:lang=.({0}|en).>.*?{1}.*?</Description>'
        if not lang:
            lang = inary.sxml.autoxml.LocalText.get_lang()
        found = []
        for name in packages:
            xml = self.pdb.get_item(name)
            if terms == [term for term in terms if re.compile(term, re.I).search(name) or \
                                                   re.compile(resum.format(lang, term), re.I).search(xml) or \
                                                   re.compile(redesc.format(lang, term), re.I).search(xml)]:
                found.append(name)
        return found

    def search_package(self, terms, lang=None, repo=None, fields=None, cs=False):
        """
        fields (dict) : looks for terms in the fields which are marked as True
        If the fields is equal to None the method will search on all fields

        example :
        if fields is equal to : {'name': True, 'summary': True, 'desc': False}
        This method will return only package that contents terms in the package
        name or summary
        """
        resum = '.*?{1}.*?'
        redesc = '<Description xml:lang=.({0}|en).>.*?{1}.*?</Description>'
        if not lang:
            lang = inary.sxml.autoxml.LocalText.get_lang()
        if not fields:
            fields = {'name': True, 'summary': True, 'desc': True}
        found = []
        for name, xml in self.pdb.get_items_iter(repo):
            if terms == [term for term in terms if (fields['name'] and
                                                    re.compile(term, re.I).search(name)) or \
                                                   (fields['summary'] and
                                                    re.compile(resum.format(lang, term), 0 if cs else re.I).search(
                                                        xml)) or \
                                                   (fields['desc'] and
                                                    re.compile(redesc.format(lang, term), 0 if cs else re.I).search(
                                                        xml))]:
                found.append(name)
        return found

    @staticmethod
    def __get_version(meta_doc):
        history = xmlext.getNode(meta_doc, 'History')
        update = xmlext.getNode(history, 'Update')

        version = xmlext.getNodeText(update, 'Version')
        release = xmlext.getNodeAttribute(update, 'release')

        return version, release, None

    @staticmethod
    def __get_distro_release(meta_doc):
        distro = xmlext.getNodeText(meta_doc, 'Distribution')
        release = xmlext.getNodeText(meta_doc, 'DistributionRelease')

        return distro, release

    def get_version_and_distro_release(self, name, repo):
        if not self.has_package(name, repo):
            raise Exception(_('Package \"{}\" not found.').format(name))

        pkg_doc = xmlext.parseString(self.pdb.get_item(name, repo))

        return self.__get_version(pkg_doc) + self.__get_distro_release(pkg_doc)

    def get_version(self, name, repo):
        if not self.has_package(name, repo):
            raise Exception(_('Package \"{}\" not found.').format(name))

        pkg_doc = xmlext.parseString(self.pdb.get_item(name, repo))

        return self.__get_version(pkg_doc)

    def get_package_repo(self, name, repo=None):
        pkg, repo = self.pdb.get_item_repo(name, repo)
        package = inary.data.metadata.Package()
        package.parse(pkg)
        return package, repo

    def which_repo(self, name):
        return self.pdb.which_repo(name)

    def get_obsoletes(self, repo=None):
        return self.odb.get_list_item(repo)

    @staticmethod
    def get_isa_packages(isa):
        repodb = inary.db.repodb.RepoDB()

        packages = set()
        for repo in repodb.list_repos():
            doc = repodb.get_repo_doc(repo)
            for package in xmlext.getTagByName(doc, "Package"):
                if xmlext.getNodeText(package, "IsA"):
                    for node in xmlext.getTagByName(package, "IsA"):
                        if xmlext.getNodeText(node) == isa:
                            packages.add(xmlext.getNodeText(package, "Name"))

        return list(packages)

    def get_rev_deps(self, name, repo=None):
        try:
            rvdb = self.rvdb.get_item(name, repo)
        except:  # FIXME: what exception could we catch here, replace with that.
            return []

        rev_deps = []

        for pkg, dep in rvdb:
            node = xmlext.parseString(dep)
            dependency = inary.analyzer.dependency.Dependency()
            dependency.package = xmlext.getNodeText(node)

            if xmlext.getAttributeList(node):
                if xmlext.getNodeAttribute(node, "version"):
                    dependency.__dict__["version"] = xmlext.getNodeAttribute(node, "version")
                elif xmlext.getNodeAttribute(node, "release"):
                    dependency.__dict__["release"] = xmlext.getNodeAttribute(node, "release")
                else:
                    pass #FIXME: ugly
            rev_deps.append((pkg, dependency))

        return rev_deps

    # replacesdb holds the info about the replaced packages (ex. gaim -> pidgin)
    def get_replaces(self, repo=None):
        pairs = {}

        for pkg_name in self.rpdb.get_list_item():
            xml = self.pdb.get_item(pkg_name, repo)
            package = xmlext.parseString(str(xml))
            replaces_tag = xmlext.getNode(package, "Replaces")
            if replaces_tag:
                for node in xmlext.getTagByName(replaces_tag, "Package"):
                    r = inary.data.relation.Relation()
                    # XXX Is there a better way to do this?
                    r.decode(node, [])
                    if inary.data.replace.installed_package_replaced(r):
                        pairs.setdefault(r.package, []).append(pkg_name)

        return pairs

    def list_packages(self, repo):
        return self.pdb.get_item_keys(repo)

    def list_newest(self, repo, since=None):
        packages = []
        historydb = inary.db.historydb.HistoryDB()
        if since:
            since_date = datetime.datetime(*time.strptime(since, "%Y-%m-%d")[0:6])
        else:
            since_date = datetime.datetime(*time.strptime(historydb.get_last_repo_update(), "%Y-%m-%d")[0:6])

        for pkg in self.list_packages(repo):
            failed = False
            try:
                enter_date = datetime.datetime(*time.strptime(self.get_package(pkg).history[-1].date, "%m-%d-%Y")[0:6])
            except:
                failed = True
            if failed:
                try:
                    enter_date = datetime.datetime(
                        *time.strptime(self.get_package(pkg).history[-1].date, "%Y-%m-%d")[0:6])
                except:
                    enter_date = datetime.datetime(
                        *time.strptime(self.get_package(pkg).history[-1].date, "%Y-%d-%m")[0:6])

            if enter_date >= since_date:
                packages.append(pkg)
        return packages


code:
# -*- coding: utf-8 -*-
#
#
# Old author: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

import os

import inary.context as ctx
import inary.db.lazydb as lazydb
import inary.errors
from inary.file import File
import inary.sxml.xmlext as xmlext
import inary.uri
import inary.util as util

"""
xmlext içine tag için remove eklenene kadar böyle
"""

class RepoError(inary.errors.Error):
    pass

class IncompatibleRepoError(RepoError):
    pass

class Repo:
    def __init__(self, indexuri):
        self.indexuri = indexuri

medias = (cd, usb, remote, local) = list(range(4))

class RepoOrder:
    def __init__(self):
        self._doc = None
        self.repos = self._get_repos()

    def add(self, repo_name, repo_url, repo_type="remote"):
        repo_doc = self._get_doc()

        repo_node = xmlext.addNode(repo_doc, "Repo")

        xmlext.addText(repo_node, "Name", repo_name)

        xmlext.addText(repo_node, "Url", repo_url)

        xmlext.addText(repo_node, "Status", "active")

        xmlext.addText(repo_node, "Media", repo_type)

        self._update(repo_doc)

    def set_status(self, repo_name, status):
        repo_doc = self._get_doc()
        for r in xmlext.getTagByName(repo_doc, "Repo"):
            if xmlext.getNodeText(r, "Name") == repo_name:
                status_node = xmlext.getNode(r, "Status")
                if status_node:
                    xmlext.removeChild(status_node, r)
                    xmlext.addText(r, "Status", status)
                else:
                    xmlext.addText(r, "Status", status)

        self._update(repo_doc)

    def get_status(self, repo_name):
        repo_doc = self._get_doc()
        for r in xmlext.getTagByName(repo_doc, "Repo"):
            if xmlext.getNodeText(r, "Name") == repo_name:
                status_node = xmlext.getNode(r, "Status")
                if status_node:
                    status = xmlext.getNodeText(status_node)
                    if status in ["active", "inactive"]:
                        return status

        return "inactive"

    def remove(self, repo_name):
        repo_doc = self._get_doc()
        for r in xmlext.getChildElts(repo_doc):
            if xmlext.getNodeText(r, "Name") == repo_name:
                xmlext.removeChild(r, repo_doc)

        self._update(repo_doc)

    def get_order(self):
        order = []

        # FIXME: get media order from inary.conf
        for m in ["cd", "usb", "remote", "local"]:
            if m in self.repos:
                order.extend(self.repos[m])

        return order

    def _update(self, doc):
        repos_file = os.path.join(ctx.config.info_dir(), ctx.const.repos)
        open(repos_file, "w").write("{}\n".format(xmlext.toPretty(doc)))

        self._doc = None
        self.repos = self._get_repos()

    def _get_doc(self):
        if self._doc is None:
            repos_file = os.path.join(ctx.config.info_dir(), ctx.const.repos)
            if os.path.exists(repos_file):
                self._doc = xmlext.parse(repos_file)

            else:
                self._doc = xmlext.newDocument("REPOS")

        return self._doc

    def _get_repos(self):
        repo_doc = self._get_doc()
        order = {}

        for r in xmlext.getTagByName(repo_doc, "Repo"):
            media = xmlext.getNodeText(r, "Media")
            name = xmlext.getNodeText(r, "Name")
            status = xmlext.getNodeText(r, "Status")
            order.setdefault(media, []).append(name)

        return order

class RepoDB(lazydb.LazyDB):

    def init(self):
        self.repoorder = RepoOrder()

    def has_repo(self, name):
        return name in self.list_repos(only_active=False)

    def has_repo_url(self, url, only_active=True):
        return url in self.list_repo_urls(only_active)

    def get_repo_doc(self, repo_name):
        repo = self.get_repo(repo_name)

        index_path = repo.indexuri.get_uri()

        # FIXME Local index files should also be cached.
        if File.is_compressed(index_path) or repo.indexuri.is_remote_file():
            index = os.path.basename(index_path)
            index_path = util.join_path(ctx.config.index_dir(),
                                        repo_name, index)

            if File.is_compressed(index_path):
                index_path = os.path.splitext(index_path)[0]

        if not os.path.exists(index_path):
            ctx.ui.warning(_("{} repository needs to be updated").format(repo_name))
            return xmlext.newDocument("INARY")

        try:
            return xmlext.parse(index_path)
        except Exception as e:
            raise RepoError(_(
                "Error parsing repository index information: {} \n Index file does not exist or is malformed.").format(
                e))

    def get_repo(self, repo):
        return Repo(inary.uri.URI(self.get_repo_url(repo)))

    # FIXME: this method is a quick hack around repo_info.indexuri.get_uri()
    def get_repo_url(self, repo):
        urifile_path = util.join_path(ctx.config.index_dir(), repo, "uri")
        uri = open(urifile_path).read()
        return uri.rstrip()

    def add_repo(self, name, repo_info, at=None):
        repo_path = util.join_path(ctx.config.index_dir(), name)
        ###########
        try:
            os.makedirs(repo_path)
        except:
            pass
        # FIXME: FileExistError errno: 17
        # When addind repo there are the same as name empty dirs it should remove it
        urifile_path = util.join_path(ctx.config.index_dir(), name, "uri")
        open(urifile_path, "w").write(repo_info.indexuri.get_uri())
        self.repoorder.add(name, repo_info.indexuri.get_uri())

    def remove_repo(self, name):
        util.clean_dir(os.path.join(ctx.config.index_dir(), name))
        self.repoorder.remove(name)

    def get_source_repos(self, only_active=True):
        repos = []

        for r in self.list_repos(only_active):
            if xmlext.getNode(self.get_repo_doc(r), "SpecFile"):
                repos.append(r)

        return repos

    def get_binary_repos(self, only_active=True):
        repos = []

        for r in self.list_repos(only_active):
            if not xmlext.getNode(self.get_repo_doc(r), "SpecFile"):
                repos.append(r)

        return repos

    def list_repos(self, only_active=True):
        temp = []
        for x in self.repoorder.get_order():
            if not only_active:
                temp.append(x)
            elif self.repo_active(x):
                temp.append(x)
        return temp

    def list_repo_urls(self, only_active=True):
        repos = []
        for r in self.list_repos(only_active):
            repos.append(self.get_repo_url(r))
        return repos

    def get_repo_by_url(self, url):
        if not self.has_repo_url(url):
            return None

        for r in self.list_repos(only_active=False):
            if self.get_repo_url(r) == url:
                return r

    def activate_repo(self, name):
        self.repoorder.set_status(name, "active")

    def deactivate_repo(self, name):
        self.repoorder.set_status(name, "inactive")

    def repo_active(self, name):
        return self.repoorder.get_status(name) == "active"

    def get_distribution(self, name):
        doc = self.get_repo_doc(name)
        distro = xmlext.getNode(doc, "Distribution")
        return distro and xmlext.getNodeText(distro, "SourceName")

    def get_distribution_release(self, name):
        doc = self.get_repo_doc(name)

        distro = xmlext.getNode(doc, "Distribution")
        return distro and xmlext.getNodeText(distro, "Version")

    def check_distribution(self, name):
        if ctx.get_option('ignore_check'):
            return

        dist_name = self.get_distribution(name)
        if dist_name is None:
            return

        compatible = dist_name == ctx.config.values.general.distribution

        dist_release = self.get_distribution_release(name)
        if dist_release is not None:
            compatible &= \
                dist_release == ctx.config.values.general.distribution_release

        if not compatible:
            self.deactivate_repo(name)
            raise IncompatibleRepoError(
                _("Repository \"{}\" is not compatible with your distribution. Repository is disabled.").format(name))


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

import gzip
import re

import inary.analyzer
import inary.data.specfile as Specfile
import inary.db
import inary.db.lazydb as lazydb
from inary.sxml import autoxml, xmlext

import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

class SourceDB(lazydb.LazyDB):

    def __init__(self):
        lazydb.LazyDB.__init__(self, cacheable=True)
        # self.init()

    def init(self):
        self.__source_nodes = {}
        self.__pkgstosrc = {}
        self.__revdeps = {}

        repodb = inary.db.repodb.RepoDB()

        for repo in repodb.list_repos():
            doc = repodb.get_repo_doc(repo)
            self.__source_nodes[repo], self.__pkgstosrc[repo] = self.__generate_sources(doc)
            self.__revdeps[repo] = self.__generate_revdeps(doc)

        self.sdb = inary.db.itembyrepo.ItemByRepo(self.__source_nodes, compressed=True)
        self.psdb = inary.db.itembyrepo.ItemByRepo(self.__pkgstosrc)
        self.rvdb = inary.db.itembyrepo.ItemByRepo(self.__revdeps)

    @staticmethod
    def __generate_sources(doc):
        sources = {}
        pkgstosrc = {}

        for spec in xmlext.getTagByName(doc, "SpecFile"):
            src = xmlext.getNode(spec, "Source")
            src_name = xmlext.getNodeText(src, "Name")
            compressed_data = gzip.zlib.compress(xmlext.toString(spec).encode('utf-8'))
            sources[src_name] = compressed_data

            for package in xmlext.getTagByName(spec, "Package"):
                pkgstosrc[xmlext.getNodeText(package, "Name")] = src_name

        return sources, pkgstosrc

    @staticmethod
    def __generate_revdeps(doc):
        revdeps = {}

        for spec in xmlext.getTagByName(doc, "SpecFile"):
            src = xmlext.getNode(spec, "Source")
            name = xmlext.getNodeText(src, "Name")
            deps = xmlext.getNode(src, "BuildDependencies")
            if deps:
                for dep in xmlext.getTagByName(deps, "Dependency"):
                    revdeps.setdefault(xmlext.getNodeText(dep), set()).add((name, xmlext.toString(dep)))

        return revdeps

    def list_sources(self, repo=None):
        return self.sdb.get_item_keys(repo)

    def which_repo(self, name):
        return self.sdb.which_repo(self.pkgtosrc(name))

    def which_source_repo(self, name):
        source = self.pkgtosrc(name)
        return source, self.sdb.which_repo(source)

    def has_spec(self, name, repo=None):
        return self.sdb.has_item(name, repo)

    def get_spec(self, name, repo=None):
        spec, repo = self.get_spec_repo(name, repo)
        return spec

    def search_spec(self, terms, lang=None, repo=None, fields=None, cs=False):
        """
        fields (dict) : looks for terms in the fields which are marked as True
        If the fields is equal to None this method will search in all fields

        example :
        if fields is equal to : {'name': True, 'summary': True, 'desc': False}
        This method will return only package that contents terms in the package
        name or summary
        """
        resum = '<Summary xml:lang=.({0}|en).>.*?{1}.*?</Summary>'
        redesc = '<Description xml:lang=.({0}|en).>.*?{1}.*?</Description>'
        if not fields:
            fields = {'name': True, 'summary': True, 'desc': True}
        if not lang:
            lang = autoxml.LocalText.get_lang()
        found = []
        for name, xml in self.sdb.get_items_iter(repo):
            if terms == [term for term in terms if (fields['name'] and
                                                    re.compile(term, re.I).search(name)) or \
                                                   (fields['summary'] and
                                                    re.compile(resum.format(lang, term), 0 if cs else re.I).search(
                                                        xml)) or \
                                                   (fields['desc'] and
                                                    re.compile(redesc.format(lang, term), 0 if cs else re.I).search(
                                                        xml))]:
                found.append(name)
        return found

    def get_spec_repo(self, name, repo=None):
        src, repo = self.sdb.get_item_repo(name, repo)
        spec = Specfile.SpecFile()
        spec.parse(src)
        return spec, repo

    def pkgtosrc(self, name, repo=None):
        return self.psdb.get_item(name, repo)

    def get_rev_deps(self, name, repo=None):
        try:
            rvdb = self.rvdb.get_item(name, repo)
        except Exception:  # FIXME: what exception could we catch here, replace with that.
            return []

        rev_deps = []

        for pkg, dep in rvdb:
            node = xmlext.parseString(dep)
            dependency = inary.analyzer.dependency.Dependency()
            dependency.package = xmlext.getNodeText(node)

            if xmlext.getAttributeList(node):
                if xmlext.getNodeAttribute(node, "version"):
                    dependency.__dict__["version"] = xmlext.getNodeAttribute(node, "version")
                elif xmlext.getNodeAttribute(node, "release"):
                    dependency.__dict__["release"] = xmlext.getNodeAttribute(node, "release")
                else:
                    pass #FIXME: ugly
            rev_deps.append((pkg, dependency))

        return rev_deps


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.

# FIXME: Exception shadows builtin Exception. This is no good.
class Exception(Exception):
    """Class of exceptions that must be caught and handled within INARY"""

    def __str__(self):
        s = ''
        for x in self.args:
            if s != '':
                s += '\n'
            s += str(x)
        return str(s)

class Error(Exception):
    """Class of exceptions that lead to program termination"""
    pass

class AnotherInstanceError(Exception):
    pass

class PrivilegeError(Exception):
    pass


code:
# -*- coding: utf-8 -*-

# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

# Gettext translation library
# python standard library modules
import os
import shutil
import time
import random

import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

# Network libraries
# import ftplib
try:
    import pycurl
except ImportError:
    raise(_("PyCurl module not found. Please install python3-pycurl or check your installation."))

# inary modules
import inary
import inary.db
import inary.errors
import inary.mirrors
import inary.util as util
import inary.context as ctx
import inary.uri

from base64 import encodebytes

# For raising errors when fetching
class FetchError(inary.errors.Error):
    pass

# For raising errors when connecting to server
class RangeError(inary.errors.Error):
    pass

# For raising errors when opening files
class FileError(inary.errors.Error):
    pass

# For raising errors when connecting to server
class RemoteError(inary.errors.Error):
    pass

class UIHandler:
    def __init__(self):
        self.filename = None
        self.url = None
        self.basename = None
        self.downloaded_size = 0
        self.percent = None
        self.pkgname = None
        self.rate = 0.0
        self.size = 0
        self.eta = '--:--:--'
        self.symbol = '--/-'
        self.last_updated = 0
        self.exist_size = 0

    def start(self, archive, url, basename, size=0):
        if os.path.exists(archive):
            self.exist_size = os.path.getsize(archive)
        self.filename = basename
        self.url = url
        self.basename = basename
        self.total_size = size or 0

        self.now = lambda: time.time()
        self.t_diff = lambda: self.now() - self.s_time

        self.s_time = self.now()

    def update(self, total_to_download, total_downloaded, total_to_upload, total_uploaded):
        if self.size == total_downloaded:
            return

        self.size = total_downloaded
        self.total_size = total_to_download
        if self.total_size:
            try:
                self.percent = (self.size * 100.0) / self.total_size
            except:
                self.percent = 0
        else:
            self.percent = 0

        if int(self.now()) != int(self.last_updated) and self.size > 0:
            try:
                self.rate, self.symbol = util.human_readable_rate(
                    (self.size - self.exist_size) / (self.now() - self.s_time))
            except ZeroDivisionError:
                self.rate, self.symbol = None, None
            if self.total_size:
                self.eta = '%02d:%02d:%02d' % \
                           tuple([i for i in time.gmtime((self.t_diff() * (100 - self.percent)) / self.percent)[3:6]])

        self._update_ui()

    def end(self, read):
        pass

    def _update_ui(self):
        ctx.ui.display_progress(operation="fetching",
                                percent=self.percent,
                                filename=self.filename,
                                total_size=self.total_size or self.size,
                                downloaded_size=self.size,
                                rate=self.rate,
                                eta=self.eta,
                                symbol=self.symbol,
                                basename=self.basename)

        self.last_updated = self.now()

class Fetcher:
    """Fetcher can fetch a file from various sources using various
    protocols."""

    def __init__(self, url, destdir="/tmp", destfile=None):
        if not isinstance(url, inary.uri.URI):
            url = inary.uri.URI(url)

        if ctx.config.get_option("authinfo"):
            url.set_auth_info(ctx.config.get_option("authinfo"))

        self.url = url
        self.destdir = destdir
        self.destfile = destfile
        self.progress = None

        self.archive_file = os.path.join(destdir, destfile or url.filename())
        self.partial_file = os.path.join(self.destdir, self.url.filename()) + ctx.const.partial_suffix
        util.ensure_dirs(self.destdir)

    def fetch(self, timeout=30):
        """Return value: Fetched file's full path.."""

        if not self.url.filename():
            raise FetchError(_('Filename error'))

        if not os.access(self.destdir, os.W_OK):
            raise FetchError(_('Access denied to write to destination directory: "{}"').format(self.destdir))

        if os.path.exists(self.archive_file) and not os.access(self.archive_file, os.W_OK):
            raise FetchError(_('Access denied to destination file: "{}"').format(self.archive_file))

        c = pycurl.Curl()
        c.protocol = self.url.scheme()
        c.setopt(c.URL, self.url.get_uri())
        # Some runtime settings (user agent, bandwidth limit, timeout, redirections etc.)
        c.setopt(pycurl.MAX_RECV_SPEED_LARGE, self._get_bandwith_limit())
        c.setopt(pycurl.USERAGENT, ('Inary Fetcher/' + inary.__version__).encode("utf-8"))
        c.setopt(pycurl.AUTOREFERER, 1)
        c.setopt(pycurl.CONNECTTIMEOUT, timeout)  # This for waiting to establish connection
        # c.setopt(pycurl.TIMEOUT, timeout) # This for waiting to read data
        c.setopt(pycurl.MAXREDIRS, 50)
        c.setopt(pycurl.NOSIGNAL, True)

        if hasattr(pycurl, "AUTOREFERER"):
            c.setopt(pycurl.AUTOREFERER, 1)

        c.setopt(pycurl.LOW_SPEED_TIME, 30)
        c.setopt(pycurl.LOW_SPEED_LIMIT, 5)

        if not ctx.config.values.general.ssl_verify:
            c.setopt(pycurl.SSL_VERIFYPEER, 0)
            c.setopt(pycurl.SSL_VERIFYHOST, 0)
        else:
            # To block man-in-middle attack
             c.setopt(pycurl.SSL_VERIFYHOST, 2)
            # c.setopt(pycurl.CAINFO, "/etc/inary/certificates/sourceforge.crt")

        # Header
        # c.setopt(pycurl.HTTPHEADER, ["%s: %s" % header for header in self._get_http_headers().items()])

        handler = UIHandler()
        handler.start(self.archive_file, self.url.get_uri(), self.url.filename())

        if os.path.exists(self.partial_file):
            if self._test_range_support():
                file_id = open(self.partial_file, "ab")
                c.setopt(c.RESUME_FROM, os.path.getsize(self.partial_file))
                ctx.ui.info(_("Partial file detected. Download resuming..."), verbose=True)
            else:
                file_id = open(self.partial_file, "wb")

        else:
            file_id = open(self.partial_file, "wb")

        # Function sets
        c.setopt(pycurl.DEBUGFUNCTION, ctx.ui.debug)
        c.setopt(c.NOPROGRESS, False)
        c.setopt(c.XFERINFOFUNCTION, handler.update)

        c.setopt(pycurl.FOLLOWLOCATION, 1)
        c.setopt(c.WRITEDATA, file_id)

        try:
            c.perform()
            ctx.ui.info("\n", noln=True)  # This is not a bug. This is a new feature. ŞAka bir yana bu hata
            # pycurl yüzünden kaynaklanıyor
            file_id.close()
            ctx.ui.info(_("RESPONSE: ") + str(c.getinfo(c.RESPONSE_CODE)), verbose=True)
            ctx.ui.info(_("Downloaded from:" + str(c.getinfo(c.EFFECTIVE_URL))), verbose=True)
            c.close()

        except pycurl.error as x:
            if x.args[0]==33:
                os.remove(self.partial_file)
            raise FetchError("{}".format(x.args[1]))

        if os.stat(self.partial_file).st_size == 0:
            os.remove(self.partial_file)
            ctx.ui.error(
                FetchError(_('A problem occurred. Please check the archive address and/or permissions again.')))

        shutil.move(self.partial_file, self.archive_file)
        return self.archive_file

    def _get_http_headers(self):
        headers = []
        if self.url.auth_info() and (self.url.scheme() == "http" or self.url.scheme() == "https"):
            enc = encodebytes('{0}:{0}'.format(self.url.auth_info()).encode('utf-8'))
            headers.append(('Authorization', 'Basic {}'.format(enc)))
        return headers

    def _get_ftp_headers(self):
        headers = []
        if self.url.auth_info() and self.url.scheme() == "ftp":
            enc = encodebytes('{0}:{0}'.format(self.url.auth_info()).encode('utf-8'))
            headers.append(('Authorization', 'Basic {}'.format(enc)))
        return headers

    def _get_proxies(self):
        proxies = {}

        if ctx.config.values.general.http_proxy and self.url.scheme() == "http":
            proxies[inary.uri.URI(ctx.config.values.general.http_proxy).scheme()] = ctx.config.values.general.http_proxy

        if ctx.config.values.general.https_proxy and self.url.scheme() == "https":
            proxies[
                inary.uri.URI(ctx.config.values.general.https_proxy).scheme()] = ctx.config.values.general.https_proxy

        if ctx.config.values.general.ftp_proxy and self.url.scheme() == "ftp":
            proxies[inary.uri.URI(ctx.config.values.general.ftp_proxy).scheme()] = ctx.config.values.general.ftp_proxy

        if self.url.scheme() in proxies:
            ctx.ui.info(_("Proxy configuration has been found for '{}' protocol.").format(self.url.scheme()))

        return proxies

    @staticmethod
    def _get_bandwith_limit():
        bandwidth_limit = ctx.config.options.bandwidth_limit or ctx.config.values.general.bandwidth_limit
        if bandwidth_limit and bandwidth_limit != "0":
            ctx.ui.warning(_("Bandwidth usage is limited to {} KB/s.").format(bandwidth_limit))
            return 1024 * int(bandwidth_limit)
        else:
            return 0

    def _test_range_support(self):
        if not os.path.exists(self.partial_file):
            return False

        import urllib.request
        try:
            file_obj = urllib.request.urlopen(self.url.get_uri())
        except urllib.request.URLError:
            ctx.ui.debug(_("Remote file can not be reached. Previously downloaded part of the file will be removed."))
            os.remove(self.partial_file)
            return False

        headers = file_obj.info()
        file_obj.close()
        if headers.get("Content-Length"):
            return True
        else:
            ctx.ui.debug(_("Server doesn't support partial downloads. Previously downloaded part of the file will be over-written."))
            os.remove(self.partial_file)
            return False

# helper function
def fetch_url(url, destdir=None, progress=None, destfile=None,pkgname=''):

    if not destdir:
        destdir=ctx.config.archives_dir()
    if not progress:
        progress=ctx.ui.Progress
    if "mirrors://" in str(url):
        fetch_from_mirror(str(url),destdir,progress,destfile)
    else:
        fetch = Fetcher(url, destdir, destfile)
        fetch.progress = progress
        fetch.fetch()

def fetch_from_fallback(url, destdir=None, progress=None, destfile=None):
    archive = os.path.basename(url)
    src = os.path.join(ctx.config.values.build.fallback, archive)
    ctx.ui.warning(_('Trying fallback address: \"{}\"').format(src))
    fetch_url(src, destdir=destdir, progress=progress, destfile=destfile)

def fetch_from_locale(url, destdir=None, progress=None, destfile=None):
    if not destdir:
        destdir=ctx.config.archives_dir()
    if url.startswith("file://"):
        url = url[7:]
    if not os.access(url, os.F_OK):
        raise FetchError(_('No such file or no permission to read for {}.').format(url))
    shutil.copy(url, os.path.join(destdir, destfile or url.split("/")[-1]))

def fetch_from_mirror(url, destdir=None, progress=None, destfile=None):
    sep = url[len("mirrors://"):].split("/")
    name = sep.pop(0)
    archive = "/".join(sep)

    mirrors = inary.mirrors.Mirrors().get_mirrors(name)
    random.shuffle(mirrors) # randomize mirror list
    if not mirrors:
        raise inary.mirrors.MirrorError(_("\"{}\" mirrors are not defined.").format(name))

    for mirror in mirrors:
        try:
            mirror_url = os.path.join(mirror, archive)
            ctx.ui.debug(_('Fetching source from: \"{}\"').format(mirror_url))
            fetch_url(mirror_url, destdir=destdir, progress=progress, destfile=destfile)
            return
        except FetchError:
            pass

    raise FetchError(_('Could not fetch source from \"{}\" mirrors.').format(name))

# Operation function
def fetch(packages=None, path=os.path.curdir):
    """
    Fetches the given packages from the repository without installing, just downloads the packages.
    @param packages: list of package names -> list_of_strings
    @param path: path to where the packages will be downloaded. If not given, packages will be downloaded
    to the current working directory.
    """
    if packages is None:
        packages = []
    packagedb = inary.db.packagedb.PackageDB()
    repodb = inary.db.repodb.RepoDB()
    for name in packages:
        package, repo = packagedb.get_package_repo(name)
        ctx.ui.info(_("\"{0}\" package found in \"{1}\" repository.").format(package.name, repo))
        uri = inary.uri.URI(package.packageURI)
        output = os.path.join(path, uri.path())
        if os.path.exists(output) and package.packageHash == inary.util.sha1_file(output):
            ctx.ui.warning(_("\"{}\" package already fetched.").format(uri.path()))
            continue
        if uri.is_absolute_path():
            url = str(uri.path())
        else:
            url = os.path.join(os.path.dirname(repodb.get_repo_url(repo)), str(uri.path()))

        fetch_url(url, path, ctx.ui.Progress)


code:
# -*- coding: utf-8 -*-
#
# Main fork Pisi: Copyright (C) 2005 - 2011, Tubitak/UEKAE
#
# Copyright (C) 2016 - 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#

"""
generic file abstraction that allows us to use URIs for everything
we support only the simple read case ATM
we are just encapsulating a common pattern in our program, nothing big.
like all inary classes, it has been programmed in a non-restrictive way
"""

import os
import lzma
import shutil

import gettext

__trans = gettext.translation('inary', fallback=True)
_ = __trans.gettext

import inary.context as ctx
import inary.errors
import inary.fetcher
import inary.uri
import inary.util

class AlreadyHaveException(inary.errors.Exception):
    def __init__(self, url, localfile):
        inary.errors.Exception.__init__(self, _("URL \"{0}\" already downloaded as \"{1}\"").format(url, localfile))
        self.url = url
        self.localfile = localfile

class NoSignatureFound(inary.errors.Exception):
    def __init__(self, url):
        inary.errors.Exception.__init__(self, _("No signature found for \"{}\"").format(url))
        self.url = url

class Error(inary.errors.Error):
    pass

class InvalidSignature(inary.errors.Error):
    def __init__(self, url):
        inary.errors.Exception.__init__(self, _("GPG Signature is invalid for \"{}\"").format(url))
        self.url = url

class File:
    # Compression types
    COMPRESSION_TYPE_AUTO = 0
    COMPRESSION_TYPE_BZ2 = 1
    COMPRESSION_TYPE_XZ = 2

    (read, write) = list(range(2))  # modes
    (detached, whatelse) = list(range(2))

    __compressed_file_extensions = {".xz": COMPRESSION_TYPE_XZ,
                                    ".bz2": COMPRESSION_TYPE_BZ2}

    @staticmethod
    def make_uri(uri):
        """handle URI arg"""
        if isinstance(uri, str):
            uri = inary.uri.URI(uri)
        elif not isinstance(uri, inary.uri.URI):
            raise Error(_("uri must have type either URI or string."))
        return uri

    @staticmethod
    def choose_method(filename, compress):
        if compress == File.COMPRESSION_TYPE_AUTO:
            for ext, method in list(File.__compressed_file_extensions.items()):
                if filename.endswith(ext):
                    return method

            return None
        else:
            return compress

    @staticmethod
    def is_compressed(filename):
        return filename.endswith(tuple(File.__compressed_file_extensions))

    @staticmethod
    def decompress(localfile, compress):
        compress = File.choose_method(localfile, compress)
        if compress == File.COMPRESSION_TYPE_XZ:
            open(localfile[:-3], "w").write(lzma.LZMAFile(localfile).read().decode('utf-8'))
            localfile = localfile[:-3]
        elif compress == File.COMPRESSION_TYPE_BZ2:
            import bz2
            open(localfile[:-4], "w").write(bz2.BZ2File(localfile).read())
            localfile = localfile[:-4]
        return localfile

    @staticmethod
    def download(uri, transfer_dir="/tmp", sha1sum=False,
                 compress=None, sign=None, copylocal=False,pkgname=''):

        assert isinstance(uri, inary.uri.URI)

        inary.util.ensure_dirs(transfer_dir)

        # Check file integrity before saving?
        check_integrity = sha1sum or sign
        origfile = inary.util.join_path(transfer_dir, uri.filename())

        if sha1sum:
            sha1filename = File.download(inary.uri.URI(uri.get_uri() + '.sha1sum'), transfer_dir)

            sha1f = open(sha1filename)
            newsha1 = sha1f.read().split("\n")[0]

        if uri.is_remote_file() or copylocal:
            tmpfile = check_integrity and uri.filename() + ctx.const.temporary_suffix
            localfile = inary.util.join_path(transfer_dir, tmpfile or uri.filename())

            # TODO: code to use old .sha1sum file, is this a necessary optimization?
            # oldsha1fn = localfile + '.sha1sum'
            # if os.exists(oldsha1fn):
            # oldsha1 = file(oldsha1fn).readlines()[0]
            if sha1sum and os.path.exists(origfile):
                oldsha1 = inary.util.sha1_file(origfile)
                if newsha1 == oldsha1:
                    # early terminate, we already got it ;)
                    raise AlreadyHaveException(uri, origfile)

            if uri.is_remote_file():
                ctx.ui.info(_("Fetching {}").format(uri.get_uri()), verbose=True)
                inary.fetcher.fetch_url(uri, transfer_dir, ctx.ui.Progress, tmpfile,pkgname)
            else:
                # copy to transfer dir
                ctx.ui.info(_("Copying \"{}\" to transfer dir").format(uri.get_uri()), verbose=True)
                shutil.copy(uri.get_uri(), localfile)
        else:
            localfile = uri.get_uri()  # TODO: use a special function here?
            if not os.path.exists(localfile):
                raise IOError(_("File \"{}\" not found.").format(localfile))
            if not os.access(localfile, os.W_OK):
                oldfn = localfile
                localfile = inary.util.join_path(transfer_dir, os.path.basename(localfile))
                shutil.copy(oldfn, localfile)

        def clean_temporary():
            temp_files = []
            if sha1sum:
                temp_files.append(sha1filename)
            if check_integrity:
                temp_files.append(localfile)
            for filename in temp_files:
                try:
                    os.unlink(filename)
                except OSError:
                    pass

        if sha1sum:
            if inary.util.sha1_file(localfile) != newsha1:
                clean_temporary()
                raise Error(_("File integrity of \"{}\" compromised.\n localfile:{}\n newsha1: {}").format(uri,inary.util.sha1_file(localfile),newsha1))

        if check_integrity:
            shutil.move(localfile, origfile)
            localfile = origfile

        localfile = File.decompress(localfile, compress)

        return localfile

    def __init__(self, uri, mode, transfer_dir="/tmp",
                 sha1sum=False, compress=None, sign=None):
        """it is pointless to open a file without a URI and a mode"""

        self.transfer_dir = transfer_dir
        self.sha1sum = sha1sum
        self.compress = compress
        self.sign = sign

        uri = File.make_uri(uri)
        if mode == File.read or mode == File.write:
            self.mode = mode
        else:
            raise Error(_("File mode must be either File.read or File.write"))
        if uri.is_remote_file():
            if self.mode == File.read:
                localfile = File.download(uri, transfer_dir, sha1sum, compress, sign)
            else:
                raise Error(_("Remote write not implemented."))
        else:
            localfile = uri.get_uri()
            if self.mode == File.read:
                localfile = File.decompress(localfile, self.compress)

        if self.mode == File.read:
            access = 'r'
        else:
            access = 'w'
        self.__file__ = open(localfile, access)
        self.localfile = localfile

    def local_file(self):
        """returns the underlying file object"""
        return self.__file__

    def close(self, delete_transfer=False):  # FIXME: look this parameter
        """this method must be called at the end of operation"""
        self.__file__.close()
        if self.mode == File.write:
            compressed_files = []
            ctypes = self.compress or 0
            if ctypes & File.COMPRESSION_TYPE_XZ:
                compressed_file = self.localfile + ".xz"
                compressed_files.append(compressed_file)
                lzma_file = lzma.LZMAFile(compressed_file, "w")
                lzma_file.write(open(self.localfile, "rb").read())
                lzma_file.close()

            if ctypes & File.COMPRESSION_TYPE_BZ2:
                import bz2
                compressed_file = self.localfile + ".bz2"
                compressed_files.append(compressed_file)
                bz2.BZ2File(compressed_file, "w").write(open(self.localfile).read())

            if self.sha1sum:
                sha1 = inary.util.sha1_file(self.localfile)
                cs = open(self.localfile + '.sha1sum', 'w')
                cs.write(sha1)
                cs.close()
                for compressed_file in compressed_files:
                    sha1 = inary.util.sha1_file(compressed_file)
                    cs = open(compressed_file + '.sha1sum', 'w')
                    cs.write(sha1)
                    cs.close()

            if self.sign == File.detached:
                if inary.util.run_batch('gpg --detach-sig ' + self.localfile)[0]:
                    raise Error(_("ERROR: \'gpg --detach-sig {}\' failed.").format(self.localfile))
                for compressed_file in compressed_files:
                    if inary.util.run_batch('gpg --detach-sig ' + compressed_file)[0]:
                        raise Error(_("ERROR: \'gpg --detach-sig {}\' failed.").format(compressed_file))

    @staticmethod
    def check_signature(uri, transfer_dir, sign=detached):
        if sign == File.detached:
            try:
                sigfilename = File.download(inary.uri.URI(uri + '.sig'), transfer_dir)
            except KeyboardInterrupt:
                raise
            except IOError:  # FIXME: what exception could we catch here, replace with that.
                ctx.ui.error(NoSignatureFound(uri))
                return True

            ret, out, err = inary.util.run_batch('gpg --verify ' + sigfilename)
            if ctx.config.values.general.ssl_verify and ret:
                ctx.ui.info("Checking GPG Signature failed ('gpg --verify {}')".format(sigfilename), color='cyan')
                ctx.ui.info(err.decode("utf-8"), color='faintcyan')
                if not ctx.ui.confirm("Would you like to skip checking gpg signature?"):
                    raise InvalidSignature(uri)  # everything is all right here
        else:
            return True

    def flush(self):
        self.__file__.flush()

    def fileno(self):
        return self.__file__.fileno()

    def isatty(self):
        return self.__file__.isatty()

    def __next__(self):
        return next(self.__file__)

    def read(self, size=None):
        if size:
            return self.__file__.read(size)
        else:
            return self.__file__.read()

    def readline(self, size=None):
        if size:
            return self.__file__.readline(size)
        else:
            return self.__file__.readline()

    def readlines(self, size=None):
        if size:
            return self.__file__.readlines(size)
        else:
            return self.__file__.readlines()

    def xreadlines(self):
        return self.__file__

    def seek(self, offset, whence=0):
        self.__file__.seek(offset, whence)

    def tell(self):
        return self.__file__.tell()

    def truncate(self):
        self.__file__.truncate()

    def write(self, str):
        self.__file__.write(str)

    def writelines(self, sequence):
        self.__file__.writelines(sequence)


code:
# -*- coding: utf-8 -*-
#
# Copyright (C) 2018, Suleyman POYRAZ (Zaryob)
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option)
# any later version.
#
# Please read the COPYING file.
#