debian-cve-check: Improve error handling when dst.json download fails

If the Debian Security Tracker json (dst.json) download fails or file is
invalid, the CVE check cannot be performed.
In this case, output backtrace [1] because there is insufficient error checking.
So improve to error handling.

When cve-check cannot be performed, there are two ways of thinking depending on
the purpose of bitbake:
1. The purpose is build, want to continue to build
   e.g. `bitbake core-image-minimal`
2. The purpose is cve-check, want to immediately terminate with an error
   e.g. `bitbake bash -c cve_check`

Add "CVE_CHECK_ERROR_ON_FAILURE" variable to satisfy these wants.
- Set "0" (is default): skip the CVE check and continue with build of bitbake
  By disabling "CVE_CHECK_DB_FILE" variable, CVE check will be skipped in Poky's
  do_cve_check() function.
  This is the same behavior as if the NVD database download failed in Poky, skip
  the CVE check and continue with build.
- Set "1": bitbake return fatal error immediately
  Immediately exit with bb.fatal().

In summary, the following changes in this commit:
- Add exception handling to load_json()
  Delete file exist check as they are handled by exception handling.
- Add check timestamp of the dst.json file
  (Even if the dst.json download fails,) A successfully downloaded dst.json file
  may still exist, so if the timestamp within CVE_DB_UPDATE_INTERVAL (default
  24 hours), it is considered a valid file.
- Add error handling logic for "CVE_CHECK_ERROR_ON_FAILURE" variable
  Change the log output lebel to match behavior of this variable.
- Some code style fixes
  Add spaces after comma.

[1]
```
File: '<path-to>/meta-debian/classes/debian-cve-check.bbclass', lineno: 33, function: debian_cve_check
     0029:            _pkg_file_name = os.path.basename(_pkg_uri)
     0030:            pkgname = _pkg_file_name.split(";")[0].split("_")[0]
     0031:            break
     0032:
 *** 0033:    if pkgname not in dst_data.keys():
     0034:        bb.note("%s is not found in Debian Security Tracker." % pkgname)
     0035:        return
     0036:
     0037:    deb_patched, deb_unpatched = deb_check_cves(d, dst_data[pkgname])
Exception: AttributeError: 'NoneType' object has no attribute 'keys'
```

Signed-off-by: Takahiro Terada <[email protected]>

Author: teradat

classes/debian-cve-check.bbclass

@@ -10,18 +10,46 @@
 DEBIAN_CVE_CHECK_DB_DIR ?= "${CVE_CHECK_DB_DIR}/DEBIAN"
 DEBIAN_CODENAME ?= "${DISTRO_CODENAME}"
 DEBIAN_SECRUTY_TRACKER_JSON_URL ??= "https://deb.freexian.com/extended-lts/tracker/data/json"
+CVE_CHECK_ERROR_ON_FAILURE ??= "0"
+
+# CVE database update interval, in seconds. By default: once a day (24*60*60).
+# Use 0 to force the update
+# Use a negative value to skip the update
+CVE_DB_UPDATE_INTERVAL ??= "86400"
 
 python debian_cve_check () {
     """
     Check CVE in Debian source
     """
+    from datetime import datetime, date
+
     debian_src_uri = d.getVar("DEBIAN_SRC_URI", True)
     if debian_src_uri is None:
         bb.note("%s dosen't use debian source" % d.getVar("BPN"))
         return
 
-    json_path = os.path.join(d.getVar("DEBIAN_CVE_CHECK_DB_DIR", True),"dst.json")
-    dst_data = load_json(json_path)
+    json_path = os.path.join(d.getVar("DEBIAN_CVE_CHECK_DB_DIR", True), "dst.json")
+    cve_check_error = True
+    if os.path.isfile(json_path):
+        try:
+            update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL"))
+        except ValueError:
+            update_interval = 0
+        if update_interval == 0:
+            update_interval = 86400
+        if ((update_interval < 0) or
+          (time.time() - os.path.getmtime(json_path) < update_interval)):
+            dst_data = load_json(json_path)
+            if dst_data is not None:
+                cve_check_error = False
+
+    if cve_check_error:
+        if d.getVar("CVE_CHECK_ERROR_ON_FAILURE") == "0":
+            d.setVar("CVE_CHECK_DB_FILE", "")
+            bb.note("debian_cve_check: No DST database found, skipping CVE check")
+        else:
+            bb.fatal("debian_cve_check: No DST database found")
+        return
 
     # get package name from DEBIAN_SRC_URI
     for _pkg_uri in debian_src_uri.split():
@@ -52,15 +80,22 @@ python update_dst () {
     from datetime import datetime, date
 
     json_urls = d.getVar("DEBIAN_SECRUTY_TRACKER_JSON_URL", "").split(" ")
-    dist_path = os.path.join(d.getVar("DEBIAN_CVE_CHECK_DB_DIR", True),"dst.json")
+    dist_path = os.path.join(d.getVar("DEBIAN_CVE_CHECK_DB_DIR", True), "dst.json")
     dist_dir = os.path.dirname(dist_path)
 
     if not os.path.isdir(dist_dir):
         os.mkdir(dist_dir)
 
     if os.path.isfile(dist_path):
-        timestamp = datetime.fromtimestamp(os.path.getmtime(dist_path))
-        if timestamp.date() == date.today():
+        try:
+            update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL"))
+        except ValueError:
+            update_interval = 0
+        if update_interval < 0:
+            bb.note("DST database update skipped")
+            return
+        if time.time() - os.path.getmtime(dist_path) < update_interval:
+            bb.note("DST database recently updated, skipping")
             return
 
     for json_url in json_urls:
@@ -79,7 +114,7 @@ python update_dst () {
                 time.sleep(10 + (attempt * 10))
                 continue
     else:
-        bb.error("DST database received error")
+        bb.warn("DST database download failed")
         return
 }
 
@@ -90,11 +125,12 @@ def load_json(path):
     Load json file.
     """
     import json
-    if not os.path.isfile(path):
-        bb.error("%s dosen't exist" % path)
+    try:
+        with open(path, 'r') as f:
+            return json.load(f)
+    except Exception as e:
+        bb.debug(2, "%s json.load: %s" % (path, e))
         return
-    with open(path, 'r') as f:
-        return json.load(f)
 
 def deb_check_cves(d, pkg_data):
     """
@@ -132,8 +168,8 @@ def compare_versions(current_version, fixed_version):
     """
     import subprocess
 
-    ret = subprocess.run(["/usr/bin/dpkg","--compare-versions", current_version,
-                                        "ge",fixed_version]).returncode
+    ret = subprocess.run(["/usr/bin/dpkg", "--compare-versions", current_version,
+                                        "ge", fixed_version]).returncode
     if ret == 0:
         return True
     else:

doc/7.cve-check.md

@@ -22,3 +22,9 @@ Replace URL:
 DEBIAN_SECRUTY_TRACKER_JSON_URL = "http://localhost:8000/dst.json"
 ```
 
+- CVE\_CHECK\_ERROR\_ON\_FAILURE
+
+If the Debian security tracker json (dst.json) cannot be downloaded or file is invalid, the CVE check cannot be performed.
+In this case, set ```CVE_CHECK_ERROR_ON_FAILURE``` to "1" to make bitbake return fatal ERROR.
+The default for this variable is "0" to skip the CVE check and continue with bitbake.
+