5.4.0 (#64)

* Add path similarity logic. Update flag 2048. 

* Remove ignore file logic on scan and update ignored extensions.

* add lines coverage to snippet analysis.

* Update Makefile, add live ldb version check

* update help

* solve minor bug with hints and dependencies tiebreak.

* improve memory management for failed scans.

* Solve memory segfault processing sbom.

Author: mscasso-scanoss

Makefile

@@ -5,12 +5,9 @@ endif
 LDFLAGS+= -lldb -lm -lpthread -ldl
 
 LDB_CURRENT_VERSION := $(shell ldb -v | sed 's/ldb-//' | head -c 3)
-LDB_TARGET_VERSION := 3.2
+LDB_TARGET_VERSION := 4.1
 
 VERSION_IS_LESS := $(shell echo $(LDB_CURRENT_VERSION) \< $(LDB_TARGET_VERSION) | bc)
-ifeq ($(VERSION_IS_LESS),1)
-	LDFLAGS += -lcrypto -lz
-endif
 
 CCFLAGS ?= -O -lz -Wall -Wno-unused-result -Wno-deprecated-declarations -g -Iinc -Iexternal/inc -D_LARGEFILE64_SOURCE -D_GNU_SOURCE
 SOURCES=$(wildcard src/*.c) $(wildcard src/**/*.c)  $(wildcard external/*.c) $(wildcard external/**/*.c)
@@ -20,8 +17,10 @@ TARGET=scanoss
 
 # Regla de prueba
 $(TARGET): $(OBJECTS)
-	@echo "Current version: $(LDB_CURRENT_VERSION)"
-	@echo "LDFLAGS: $(LDFLAGS)"
+ifeq ($(VERSION_IS_LESS),1)
+	@echo "Current LDB version: $(LDB_CURRENT_VERSION) is too old, please update to the lastest version to continue."
+	exit 1
+endif
 
 	$(CC) -g -o $(TARGET) $^ $(LDFLAGS)
 

inc/component.h

@@ -41,6 +41,8 @@ typedef struct component_data_t
 	char * dependency_text; /* used in json output generation */
 	char * health_text; /* used in json output generation */
 	int hits; /*used in binary analysis*/
+	char * file_path_ref;
+	int path_rank;
 } component_data_t;
 
 component_data_t * component_init(void);

inc/scanoss.h

@@ -44,7 +44,7 @@
 #define WFP_REC_LN 18
 
 /* Log files */
-#define SCANOSS_VERSION "5.3.5"
+#define SCANOSS_VERSION "5.4.0"
 #define SCAN_LOG "/tmp/scanoss_scan.log"
 #define MAP_DUMP "/tmp/scanoss_map.dump"
 #define SLOW_QUERY_LOG "/tmp/scanoss_slow_query.log"
@@ -65,7 +65,7 @@
 #define DISABLE_BEST_MATCH 256
 #define DISABLE_REPORT_IDENTIFIED 512
 #define ENABLE_DOWNLOAD_URL 1024
-#define ENABLE_GITHUB_FULL_PATH 2048
+#define ENABLE_PATH_HINT 2048
 #define DISABLE_SERVER_INFO 4096
 #define DISABLE_HEALTH 8192
 #define ENABLE_HIGH_ACCURACY 16384

src/binary_scan.c

@@ -161,7 +161,11 @@ static bool get_all_file_ids(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8
 
 static void fhash_process(char * hash, component_list_t * comp_list)
 {
-	struct ldb_table oss_fhash = {.db = "oss", .table = "fhashes", .key_ln = 16, .rec_ln = 0, .ts_ln = 2, .tmp = false};
+	struct ldb_table oss_fhash = {.db = "oss", .table = "fhashes", .key_ln = 16, .rec_ln = 0, .ts_ln = 2, .tmp = false, .keys=2, .definitions = 0};
+	
+	if (!ldb_table_exists(oss_fhash.db, oss_fhash.table)) // skip if the table is not present
+		return;
+	
 	uint8_t fhash[16]; 
 	ldb_hex_to_bin(hash, 32, fhash);
 	/* Get all file IDs for given wfp */
@@ -304,7 +308,10 @@ int binary_scan(char * input)
 			break;
 		component_list_destroy(result.components);
 		free(result.file);
+		result.file = NULL;
 		free(result.md5);
+		result.md5 = NULL;
+		
 		sensibility++;
 	};
 

src/component.c

@@ -177,6 +177,32 @@ bool ignored_asset_match(uint8_t *url_record)
 	return found;
 }
 
+static char * look_for_version(char *in)
+{
+	if (!in)
+		return NULL;
+	bool is_ver = false;
+
+	char *v = strstr(in, "-v");
+	if (v && isdigit(*(v + 2)))
+		is_ver = true;
+	else
+	{
+		v = strchr(in, '.');
+		if (v && isdigit(*(v + 1)) && (*(v + 2) == '.' || isdigit(*(v + 2))))
+			is_ver = true;
+	}
+
+	if (is_ver)
+	{
+		char * p = strchr(v, '/');
+		if (p)
+			return (p+1);
+	}
+
+	return in;
+}
+
 /**
  * @brief Fill the match structure
  * @param url_key md5 of the match url
@@ -203,7 +229,7 @@ bool fill_component(component_data_t *component, uint8_t *url_key, char *file_pa
 		memcpy(component->url_md5, url_key, MD5_LEN);
 		if (file_path)
 		{
-			component->file = strdup(file_path);
+			component->file = strdup(look_for_version(file_path));
 			component->path_ln = strlen(file_path);
 			flip_slashes(component->file);
 		}

src/file.c

@@ -139,27 +139,32 @@ void get_file_md5(char *filepath, uint8_t *md5_result)
 
 	/* Read file contents into buffer */
 	FILE *in = fopen(filepath, "rb");
+
+	if (!in)
+	{
+		MD5(NULL, 0, md5_result);
+		return;
+	}
+
 	fseek(in, 0L, SEEK_END);
 	long filesize = ftell(in);
-
 	if (!filesize)
 	{
 		MD5(NULL, 0, md5_result);
 	}
-
 	else
 	{
 		/* Read file contents */
 		fseek(in, 0L, SEEK_SET);
 		uint8_t *buffer = malloc(filesize);
-		if (!fread(buffer, filesize, 1, in)) fprintf(stderr, "Warning: cannot open file %s\n", filepath);
+		if (!fread(buffer, filesize, 1, in))
+			fprintf(stderr, "Warning: cannot open file %s\n", filepath);
 
 		/* Calculate MD5sum */
 		MD5(buffer, filesize, md5_result);
-		free (buffer);
+		free(buffer);
+		fclose(in);
 	}
-
-	fclose(in);
 }
 
 /**

src/help.c

@@ -88,7 +88,7 @@ Alternatively, these value can be written in %s\n\
 |  256  | Disable best match only (default: enabled)            |\n\
 |  512  | Hide identified files (default: disabled)             |\n\
 | 1024  | Enable download_url (default: disabled)               |\n\
-| 2048  | Enable GitHub full path (default: disabled)           |\n\
+| 2048  | Enable \"use path hint\" logic (default: disabled)    |\n\
 | 4096  | Disable extended server stats (default: enabled)      |\n\
 | 8192  | Disable health layer (default: enabled)               |\n\
 | 16384 | Enable high accuracy, slower scan (default: disabled) |\n\

src/ignored_extensions.c

@@ -36,9 +36,9 @@ char *IGNORED_EXTENSIONS[] = {
 
 	/* File extensions */
 	".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am",
-	".asc", ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake",
+	".asc", ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".cmake",
 	".cnf", ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj",
-	".css", ".csv", ".cvsignore", ".dat", ".data", ".db", ".doc", ".ds_store",
+	".csv", ".cvsignore", ".dat", ".data", ".db", ".doc", ".ds_store",
 	".dtd", ".dts", ".dtsi", ".dump", ".eot", ".eps", ".geojson", ".gdoc", ".gif",
 	".gitignore", ".glif", ".gmo", ".gradle", ".guess", ".hex", ".htm", ".html",
 	".ico", ".in", ".inc", ".info", ".ini", ".ipynb", ".jpeg", ".jpg", ".json",
@@ -49,7 +49,7 @@ char *IGNORED_EXTENSIONS[] = {
 	".spec", ".sql", ".sub", ".svg", ".svn-base", ".tab", ".template", ".test",
 	".tex", ".tiff", ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl",
 	".woff", ".xht", ".xhtml", ".xls", ".xml", ".xpm", ".xsd", ".xul", ".yaml",
-	".yml", ".LAS",".adk",".asc",".cif",".cli",".cosmo",".deploy",
+	".yml", ".LAS",".adk",".asc",".cif",".cli",".cosmo",".deploy",".pom",
 	".dfm",".dmm",".fa",".fasta",".fcb",".flm",".fna",".gbr",".gen",".gro",
 	".hgtags",".hh",".ihex",".kp",".mpx",".pdb",".poly",".prn",".ps",".ref",
 	".resx",".smp",".stg",".tfa",".tsv",".vcf",".vhd",".xy",".xyz",

src/ignorelist.c

@@ -35,6 +35,7 @@
 
 #include "ignorelist.h"
 #include "ignored_extensions.h"
+#include "debug.h"
 
 /**
  * @brief Returns a pointer to the file extension of "path"
@@ -100,7 +101,11 @@ bool ignored_extension(char *name)
 {
 	int i=0;
 	while (IGNORED_EXTENSIONS[i])
-		if (ends_with(IGNORED_EXTENSIONS[i++], name)) return true;
+		if (ends_with(IGNORED_EXTENSIONS[i++], name)) 
+		{
+			scanlog("Component ignored by path extension: %s", name);
+			return true;
+		}
 
 	return false;
 }

src/main.c

@@ -65,91 +65,61 @@ component_item *declared_components;
 uint8_t trace_id[MD5_LEN];
 bool trace_on;
 
-
+#define LDB_VER_MIN "4.1.0"
 /* Initialize tables for the DB name indicated (defaults to oss) */
 void initialize_ldb_tables(char *name)
 {
+	
+	char * ldb_ver = NULL;
+	ldb_version(&ldb_ver);
+	scanlog("ldb version: %s\n", ldb_ver);
+	
+	if (!ldb_ver || strcmp(ldb_ver, LDB_VER_MIN) < 0)
+	{
+		fprintf(stderr, "The current ldb version %s is too old, please upgrade to %s to proceed\n", ldb_ver, LDB_VER_MIN);
+		exit(EXIT_FAILURE);
+	}
+	free(ldb_ver);
+	
 	char oss_db_name[MAX_ARGLN];
 
 	if (name) strcpy(oss_db_name, name);
 	else strcpy(oss_db_name, DEFAULT_OSS_DB_NAME);
 
-	strcpy(oss_url.db, oss_db_name);
-	strcpy(oss_url.table, "url");
-	oss_url.key_ln = 16;
-	oss_url.rec_ln = 0;
-	oss_url.ts_ln = 2;
-	oss_url.tmp = false;
-
-	strcpy(oss_file.db, oss_db_name);
-	strcpy(oss_file.table, "file");
-	oss_file.key_ln = 16;
-	oss_file.rec_ln = 0;
-	oss_file.ts_ln = 2;
-	oss_file.tmp = false;
-
-	strcpy(oss_wfp.db, oss_db_name);
-	strcpy(oss_wfp.table, "wfp");
-	oss_wfp.key_ln = 4;
-	oss_wfp.rec_ln = 18;
-	oss_wfp.ts_ln = 2;
-	oss_wfp.tmp = false;
-
-	strcpy(oss_purl.db, oss_db_name);
-	strcpy(oss_purl.table, "purl");
-	oss_purl.key_ln = 16;
-	oss_purl.rec_ln = 0;
-	oss_purl.ts_ln = 2;
-	oss_purl.tmp = false;
-
-	strcpy(oss_copyright.db, oss_db_name);
-	strcpy(oss_copyright.table, "copyright");
-	oss_copyright.key_ln = 16;
-	oss_copyright.rec_ln = 0;
-	oss_copyright.ts_ln = 2;
-	oss_copyright.tmp = false;
-
-	strcpy(oss_quality.db, oss_db_name);
-	strcpy(oss_quality.table, "quality");
-	oss_quality.key_ln = 16;
-	oss_quality.rec_ln = 0;
-	oss_quality.ts_ln = 2;
-	oss_quality.tmp = false;
-
-	strcpy(oss_vulnerability.db, oss_db_name);
-	strcpy(oss_vulnerability.table, "vulnerability");
-	oss_vulnerability.key_ln = 16;
-	oss_vulnerability.rec_ln = 0;
-	oss_vulnerability.ts_ln = 2;
-	oss_vulnerability.tmp = false;
-
-	strcpy(oss_dependency.db, oss_db_name);
-	strcpy(oss_dependency.table, "dependency");
-	oss_dependency.key_ln = 16;
-	oss_dependency.rec_ln = 0;
-	oss_dependency.ts_ln = 2;
-	oss_dependency.tmp = false;
-
-	strcpy(oss_license.db, oss_db_name);
-	strcpy(oss_license.table, "license");
-	oss_license.key_ln = 16;
-	oss_license.rec_ln = 0;
-	oss_license.ts_ln = 2;
-	oss_license.tmp = false;
-
-	strcpy(oss_attribution.db, oss_db_name);
-	strcpy(oss_attribution.table, "attribution");
-	oss_attribution.key_ln = 16;
-	oss_attribution.rec_ln = 0;
-	oss_attribution.ts_ln = 2;
-	oss_attribution.tmp = false;
-
-	strcpy(oss_cryptography.db, oss_db_name);
-	strcpy(oss_cryptography.table, "cryptography");
-	oss_cryptography.key_ln = 16;
-	oss_cryptography.rec_ln = 0;
-	oss_cryptography.ts_ln = 2;
-	oss_cryptography.tmp = false;
+	char dbtable[MAX_ARGLN * 2];
+	scanlog("Loading tables definitions\n");
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "url");
+	oss_url = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "file");
+	oss_file = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "wfp");
+	oss_wfp = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "purl");
+	oss_purl = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "copyright");
+	oss_copyright = ldb_read_cfg(dbtable);
+	
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "quality");
+	oss_quality = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "vulnerability");
+	oss_vulnerability = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "dependency");
+	oss_dependency = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "license");
+	oss_license = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "attribution");
+	oss_attribution = ldb_read_cfg(dbtable);
+
+	snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "cryptography");
+	oss_cryptography = ldb_read_cfg(dbtable);
 
 	kb_version_get();
 	osadl_load_file();
@@ -304,12 +274,10 @@ int main(int argc, char **argv)
 
 	microseconds_start = microseconds_now();
 
-	initialize_ldb_tables(NULL);
-
 	/* Parse arguments */
 	int option;
 	bool invalid_argument = false;
-
+	char * ldb_db_name = NULL;
 	while ((option = getopt(argc, argv, ":f:s:b:B:c:k:a:F:l:n:i:M:N:wtvhedqH")) != -1)
 	{
 		/* Check valid alpha is entered */
@@ -339,6 +307,7 @@ int main(int argc, char **argv)
 				break;
 
 			case 'k':
+				initialize_ldb_tables(ldb_db_name);
 				mz_file_contents(optarg, oss_file.db);
 				exit(EXIT_SUCCESS);
 				break;
@@ -359,7 +328,7 @@ int main(int argc, char **argv)
 				break;
 
 			case 'n':
-				initialize_ldb_tables(optarg);
+				ldb_db_name = strdup(optarg);
 				break;
 			case 'M':
 				scan_max_snippets = atol(optarg);
@@ -475,6 +444,9 @@ int main(int argc, char **argv)
 			exit(EXIT_FAILURE);
 		}
 
+		initialize_ldb_tables(ldb_db_name);
+		free(ldb_db_name);
+
 		/* Remove trailing backslashes from target (if any) */
 		strcpy (target, argv[argc-1]);
 		for (int i=strlen(target)-1; i>=0; i--) if (target[i]=='/') target[i]=0; else break;