Add support for managing appstream catalogs as part of the index

pisi/api.py

@@ -21,6 +21,7 @@
 import pisi.db.historydb
 import pisi.db.componentdb
 import pisi.db.groupdb
+import pisi.db.appstreamdb
 import pisi.index
 import pisi.config
 import pisi.metadata
@@ -35,6 +36,7 @@
 import pisi.operations.check
 import pisi.operations.build
 import pisi.signalhandler as signalhandler
+import pisi.operations.appstream
 import pisi.errors
 
 
@@ -897,6 +899,7 @@ def __update_repo(repo, force=False):
     index = pisi.index.Index()
     if repodb.has_repo(repo):
         repouri = repodb.get_repo(repo).indexuri.get_uri()
+        pisi.operations.appstream.update_catalogs(repo, force)
         try:
             index.read_uri_of_repo(repouri, repo)
         except pisi.file.AlreadyHaveException as e:

pisi/appstream.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2005-2011 TUBITAK/UEKAE, 2013-2017 Ikey Doherty, Solus Project
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import pisi
+import pisi.pxml.xmlfile as xmlfile
+import pisi.pxml.autoxml as autoxml
+
+
+class Error(pisi.Error):
+    pass
+
+
+class Icons(xmlfile.XmlFile, metaclass=autoxml.autoxml):
+
+    a_size = [autoxml.String, autoxml.MANDATORY]
+    t_URI = [autoxml.String, autoxml.MANDATORY]
+
+
+class AppstreamCatalog(xmlfile.XmlFile, metaclass=autoxml.autoxml):
+    "representation for appstream declarations"
+
+    t_Origin = [autoxml.String, autoxml.MANDATORY]
+    t_URI = [autoxml.String, autoxml.MANDATORY]
+    t_Icons = [[Icons], autoxml.OPTIONAL, "Icons"]
+
+
+class AppstreamCatalogs(xmlfile.XmlFile, metaclass=autoxml.autoxml):
+    "representation for component declarations"
+
+    tag = "PISI"
+
+    t_Appstreams = [[AppstreamCatalog], autoxml.MANDATORY, "AppstreamCatalogs/AppstreamCatalog"]

pisi/db/appstreamdb.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: 2005-2011 TUBITAK/UEKAE, 2013-2017 Ikey Doherty, Solus Project
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import pisi
+import pisi.db
+import pisi.appstream
+import pisi.db.itembyrepo
+from pisi import translate as _
+from pisi.db import lazydb
+
+
+class AppstreamNotFound(Exception):
+    pass
+
+
+class AppstreamDB(lazydb.LazyDB):
+    def __init__(self):
+        lazydb.LazyDB.__init__(self, cacheable=False)
+
+    def init(self):
+        catalog_nodes = {}
+        catalog_components = {}
+
+        repodb = pisi.db.repodb.RepoDB()
+
+        for repo in repodb.list_repos():
+            doc = repodb.get_repo_doc(repo)
+            catalog_nodes[repo] = self.__generate_catalogs(doc)
+            catalog_components[repo] = self.__generate_catalog(doc)
+
+        self.adb = pisi.db.itembyrepo.ItemByRepo(catalog_nodes)
+        self.acdb = pisi.db.itembyrepo.ItemByRepo(catalog_components)
+
+    def __generate_catalog(self, doc):
+        catalogs = {}
+        for c in doc.tags("AppstreamCatalog"):
+            catalog = c.getTagData("Origin") or "unknown"
+            catalog_info = {
+                "uri": c.getTagData("URI"),
+                "icons_sizes": [x.getAttribute("size") for x in c.tags("Icons")],
+                "icon_urls": [x.getTagData("URI") for x in c.tags("Icons")],
+            }
+            catalogs.setdefault(catalog, []).append(catalog_info)
+        return catalogs
+
+    def __generate_catalogs(self, doc):
+        return dict([(x.getTagData("Origin"), x.toString()) for x in doc.tags("AppstreamCatalog")])
+
+    def has_catalog(self, name, repo=None):
+        return self.adb.has_item(name, repo)
+
+    def list_catalogs(self, repo=None):
+        return self.adb.get_item_keys(repo)
+
+    def get_catalog(self, name, repo=None):
+        if not self.has_catalog(name, repo):
+            raise AppstreamNotFound(_("Appstream catalog %s not found") % name)
+
+        catalog = pisi.appstream.AppstreamCatalog()
+        catalog.parse(self.adb.get_item(name, repo))
+
+        return catalog
+
+    def get_catalog_components(self, name, repo=None):
+        if not self.has_catalog(name, repo):
+            raise AppstreamNotFound(_("Appstream catalog %s not found") % name)
+
+        if self.acdb.has_item(name):
+            return self.acdb.get_item(name, repo)
+
+        return []

pisi/fetcher.py

@@ -106,14 +106,15 @@ class Fetcher:
     """Fetcher can fetch a file from various sources using various
     protocols."""
 
-    def __init__(self, url, destdir="/tmp", destfile=None):
+    def __init__(self, url, destdir="/tmp", destfile=None, headers_only=False):
         if not isinstance(url, pisi.uri.URI):
             url = pisi.uri.URI(url)
 
         self.url = url
         self.destdir = destdir
         self.destfile = destfile
         self.progress = None
+        self.headers_only = headers_only
 
         self.archive_file = os.path.join(destdir, destfile or url.filename())
         self.partial_file = (
@@ -169,7 +170,10 @@ def fetch(self):
                 with contextlib.closing(
                     urllib.request.urlopen(self.url.get_uri(), timeout=15)
                 ) as fp:
-                    headers = fp.info()
+                    self.headers = fp.info()
+
+                    if self.headers_only:
+                        return
 
                     if self.url.is_local_file():
                         return os.path.normpath(self.url.path())
@@ -184,8 +188,8 @@ def fetch(self):
                         size = -1
                         read = 0
                         blocknum = 0
-                        if "content-length" in headers:
-                            size = int(headers["Content-Length"])
+                        if "content-length" in self.headers:
+                            size = int(self.headers["Content-Length"])
                         fetch_handler.update(blocknum, bs, size)
                         while True:
                             block = fp.read(bs)
@@ -305,7 +309,9 @@ def _test_range_support(self):
 
 
 # helper function
-def fetch_url(url, destdir, progress=None, destfile=None):
-    fetch = Fetcher(url, destdir, destfile)
+def fetch_url(url, destdir, progress=None, destfile=None, headers_only=False):
+    fetch = Fetcher(url, destdir, destfile, headers_only)
     fetch.progress = progress
     fetch.fetch()
+    if headers_only:
+        return fetch

pisi/index.py

@@ -20,6 +20,7 @@
 import pisi.pxml.autoxml as autoxml
 import pisi.component as component
 import pisi.group as group
+import pisi.appstream as appstream
 import pisi.operations.build
 
 
@@ -36,6 +37,7 @@ class Index(xmlfile.XmlFile, metaclass=autoxml.autoxml):
     # t_Metadatas = [ [metadata.MetaData], autoxml.optional, "MetaData"]
     t_Components = [[component.Component], autoxml.OPTIONAL, "Component"]
     t_Groups = [[group.Group], autoxml.OPTIONAL, "Group"]
+    t_Appstreams = [[appstream.AppstreamCatalog], autoxml.OPTIONAL, "AppstreamCatalog"]
 
     def read_uri(self, uri, tmpdir, force=False):
         return self.read(
@@ -97,6 +99,8 @@ def index(self, repo_uri):
                 elif fn.endswith(ctx.const.package_suffix):
                     packages.append(os.path.join(root, fn))
 
+                if fn == "appstream.xml":
+                    self.appstreams.extend(add_appstreams(os.path.join(root, fn)))
                 if fn == "components.xml":
                     self.components.extend(add_components(os.path.join(root, fn)))
                 if fn == "distribution.xml":
@@ -240,6 +244,13 @@ def add_package(params):
         raise Exception
 
 
+def add_appstreams(path):
+    ctx.ui.info("Adding appstream.xml to index")
+    appstreams_xml = appstream.AppstreamCatalogs()
+    appstreams_xml.read(path)
+    return appstreams_xml.appstreams
+
+
 def add_groups(path):
     ctx.ui.info(_("Adding groups.xml to index"))
     groups_xml = group.Groups()