sw-collector: Collects endpoint software events

conf/Makefile.am

@@ -24,7 +24,8 @@ options = \
 	options/scepclient.opt \
 	options/starter.opt \
 	options/swanctl.opt \
-	options/tnc.opt
+	options/tnc.opt \
+	options/sw-collector.opt
 
 plugins = \
 	plugins/addrblock.opt \

conf/options/sw-collector.opt

@@ -0,0 +1,17 @@
+sw-collector {}
+	Options for the sw-collector tool.
+
+	Options for the sw-collector tool.
+
+sw-collector.database =
+	Path to software collector database containing event timestamps, software
+	creation and deletion events and collected software identifiers.
+
+sw-collector.history =
+	Path pointing to apt history.log file.
+
+sw-collector.first_time = 0000-00-00T00:00:00Z
+	Time in UTC when the Linux OS was installed.
+
+sw-collector.load =
+	Plugins to load in sw-collector tool.

src/libimcv/plugins/imc_swima/.gitignore

@@ -1 +1,2 @@
 strongswan.org_*.swidtag
+sw-collector

src/libimcv/plugins/imc_swima/Makefile.am

@@ -30,7 +30,20 @@ imcv_LTLIBRARIES = imc-swima.la
 imc_swima_la_LIBADD = \
 	$(top_builddir)/src/libimcv/libimcv.la \
 	$(top_builddir)/src/libstrongswan/libstrongswan.la
-
 imc_swima_la_SOURCES = imc_swima.c imc_swima_state.h imc_swima_state.c
-
 imc_swima_la_LDFLAGS = -module -avoid-version -no-undefined
+
+ipsec_PROGRAMS = sw-collector
+sw_collector_SOURCES = \
+	sw_collector/sw-collector.c \
+	sw_collector/sw_collector_db.h sw_collector/sw_collector_db.c \
+	sw_collector/sw_collector_history.h sw_collector/sw_collector_history.c
+
+sw_collector_LDADD = \
+		$(top_builddir)/src/libstrongswan/libstrongswan.la \
+		$(top_builddir)/src/libimcv/libimcv.la
+sw-collector.o : $(top_builddir)/config.status
+
+templatesdir = $(pkgdatadir)/templates/database/sw-collector
+dist_templates_DATA = sw_collector/sw_collector_tables.sql
+

src/libimcv/plugins/imc_swima/sw_collector/sw-collector.c

@@ -0,0 +1,398 @@
+/*
+ * Copyright (C) 2017 Andreas Steffen
+ * HSR Hochschule fuer Technik Rapperswil
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <getopt.h>
+#include <unistd.h>
+#ifdef HAVE_SYSLOG
+# include <syslog.h>
+#endif
+
+#include "sw_collector_db.h"
+#include "sw_collector_history.h"
+
+#include <library.h>
+#include <utils/debug.h>
+#include <utils/lexparser.h>
+
+/**
+ * global debug output variables
+ */
+static int debug_level = 2;
+static bool stderr_quiet = FALSE;
+static int count = 0;
+
+typedef enum collector_op_t collector_op_t;
+
+enum collector_op_t {
+	COLLECTOR_OP_EXTRACT,
+	COLLECTOR_OP_LIST
+};
+
+/**
+ * sw_collector dbg function
+ */
+static void sw_collector_dbg(debug_t group, level_t level, char *fmt, ...)
+{
+	va_list args;
+
+	if (level <= debug_level)
+	{
+		if (!stderr_quiet)
+		{
+			va_start(args, fmt);
+			vfprintf(stderr, fmt, args);
+			fprintf(stderr, "\n");
+			va_end(args);
+		}
+
+#ifdef HAVE_SYSLOG
+		{
+			int priority = LOG_INFO;
+			char buffer[8192];
+			char *current = buffer, *next;
+
+			/* write in memory buffer first */
+			va_start(args, fmt);
+			vsnprintf(buffer, sizeof(buffer), fmt, args);
+			va_end(args);
+
+			/* do a syslog with every line */
+			while (current)
+			{
+				next = strchr(current, '\n');
+				if (next)
+				{
+					*(next++) = '\0';
+				}
+				syslog(priority, "%s\n", current);
+				current = next;
+			}
+		}
+#endif /* HAVE_SYSLOG */
+	}
+}
+
+/**
+ * atexit handler
+ */
+static void cleanup(void)
+{
+	library_deinit();
+#ifdef HAVE_SYSLOG
+	closelog();
+#endif
+}
+
+/**
+ * Display usage of sw-collector command
+ */
+static void usage(void)
+{
+	printf("\
+Usage:\n\
+  sw-collector --help\n\
+  sw-collector [--debug <level>] [--quiet] --list\n\
+  sw-collector [--debug <level>] [--quiet] [--count <event count>]\n");
+}
+
+/**
+ * Parse command line options
+ */
+static collector_op_t do_args(int argc, char *argv[])
+{
+	collector_op_t op = COLLECTOR_OP_EXTRACT;
+
+	/* reinit getopt state */
+	optind = 0;
+
+	while (TRUE)
+	{
+		int c;
+
+		struct option long_opts[] = {
+			{ "help", no_argument, NULL, 'h' },
+			{ "count", required_argument, NULL, 'c' },
+			{ "debug", required_argument, NULL, 'd' },
+			{ "list", no_argument, NULL, 'l' },
+			{ "quiet", no_argument, NULL, 'q' },
+			{ 0,0,0,0 }
+		};
+
+		c = getopt_long(argc, argv, "hc:d:lq", long_opts, NULL);
+		switch (c)
+		{
+			case EOF:
+				break;
+			case 'h':
+				usage();
+				exit(SUCCESS);
+				break;
+			case 'c':
+				count = atoi(optarg);
+				continue;
+			case 'd':
+				debug_level = atoi(optarg);
+				continue;
+			case 'l':
+				op = COLLECTOR_OP_LIST;
+				continue;
+			case 'q':
+				stderr_quiet = TRUE;
+				continue;
+			default:
+				usage();
+				exit(EXIT_FAILURE);
+		}
+		break;
+	}
+	return op;
+}
+
+/**
+ * Extract software events from apt history log files
+ */
+static int extract_history(sw_collector_db_t *db)
+{
+	sw_collector_history_t *history = NULL;
+	uint32_t epoch, last_eid, eid = 0;
+	char *history_path, *last_time = NULL, rfc_time[21];
+	chunk_t *h, history_chunk, line, cmd;
+	int status = EXIT_FAILURE;
+	bool skip = TRUE;
+
+	/* open history file for reading */
+	history_path= lib->settings->get_str(lib->settings, "sw-collector.history",
+										 NULL);
+	if (!history_path)
+	{
+		fprintf(stderr, "sw-collector.history path not set.\n");
+		return FALSE;
+	}
+	h = chunk_map(history_path, FALSE);
+	if (!h)
+	{
+		fprintf(stderr, "opening '%s' failed: %s", history, strerror(errno));
+		return FALSE;
+	}
+	history_chunk = *h;
+
+	/* Instantiate history extractor */
+	history = sw_collector_history_create(db, 1);
+	if (!history)
+	{
+		/* OS is not supported */
+		goto end;
+	}
+
+	/* retrieve last event in database */
+	if (!db->get_last_event(db, &last_eid, &epoch, &last_time) || !last_eid)
+	{
+		goto end;
+	}
+	DBG0(DBG_IMC, "Last-Event: %s, eid = %u, epoch = %u",
+				   last_time, last_eid, epoch);
+
+	/* parse history file */
+	while (fetchline(&history_chunk, &line))
+	{
+		if (line.len == 0)
+		{
+			continue;
+		}
+		if (!extract_token(&cmd, ':', &line))
+		{
+			fprintf(stderr, "terminator symbol ':' not found.\n");
+			goto end;
+		}
+		if (match("Start-Date", &cmd))
+		{
+			if (!history->extract_timestamp(history, line, rfc_time))
+			{
+				goto end;
+			}
+
+			/* have we reached new history entries? */
+			if (skip && strcmp(rfc_time, last_time) > 0)
+			{
+				skip = FALSE;
+			}
+			if (skip)
+			{
+				continue;
+			}
+
+			/* insert new event into database */
+			eid = db->add_event(db, rfc_time);
+			if (!eid)
+			{
+				goto end;
+			}
+			DBG1(DBG_IMC, "Start-Date: %s, eid = %u, epoch = %u",
+						   rfc_time, eid, epoch);
+		}
+		else if (skip)
+		{
+			/* skip old history entries which have already been processed */
+			continue;
+		}
+		else if (match("Install", &cmd))
+		{
+			DBG1(DBG_IMC, "  Install:");
+			if (!history->extract_packages(history, line, eid, SW_OP_INSTALL))
+			{
+				goto end;
+			}
+		}
+		else if (match("Upgrade", &cmd))
+		{
+			DBG1(DBG_IMC, "  Upgrade:");
+			if (!history->extract_packages(history, line, eid, SW_OP_UPGRADE))
+			{
+				goto end;
+			}
+		}
+		else if (match("Remove", &cmd))
+		{
+			DBG1(DBG_IMC, "  Remove:");
+			if (!history->extract_packages(history, line, eid, SW_OP_REMOVE))
+			{
+				goto end;
+			}
+		}
+		else if (match("Purge", &cmd))
+		{
+			DBG1(DBG_IMC, "  Purge:");
+			if (!history->extract_packages(history, line, eid, SW_OP_REMOVE))
+			{
+				goto end;
+			}
+		}
+		else if (match("End-Date", &cmd))
+		{
+			/* Process 'count' events at a time */
+			if (count > 0 && eid - last_eid == count)
+			{
+				fprintf(stderr, "added %d events\n", count);
+				goto end;
+			}
+		}
+	}
+
+	if (history->merge_installed_packages(history))
+	{
+		status = EXIT_SUCCESS;
+	}
+
+end:
+	free(last_time);
+	DESTROY_IF(history);
+	chunk_unmap(h);
+
+	return status;
+}
+
+/**
+ * List all software identifiers stored in the collector database
+ */
+static int list_identifiers(sw_collector_db_t *db)
+{
+	enumerator_t *e;
+	char *name, *package, *version;
+	uint32_t count = 0, installed_count = 0, installed;
+
+	e = db->create_sw_enumerator(db, FALSE);
+	if (!e)
+	{
+		return EXIT_FAILURE;
+	}
+	while (e->enumerate(e, &name, &package, &version, &installed))
+	{
+		printf("%s,%s,%s,%d\n", name, package, version, installed);
+		if (installed)
+		{
+			installed_count++;
+		}
+		count++;
+	}
+	e->destroy(e);
+	DBG1(DBG_IMC, "retrieved %u software identities with %u installed and %u "
+				  "deleted", count, installed_count, count - installed_count);
+
+	return EXIT_SUCCESS;
+}
+
+int main(int argc, char *argv[])
+{
+	sw_collector_db_t *db = NULL;
+	collector_op_t op;
+	char *uri;
+	int status;
+
+	op = do_args(argc, argv);
+
+	/* enable sw_collector debugging hook */
+	dbg = sw_collector_dbg;
+#ifdef HAVE_SYSLOG
+	openlog("sw-collector", 0, LOG_DEBUG);
+#endif
+
+	atexit(cleanup);
+
+	/* initialize library */
+	if (!library_init(NULL, "sw-collector"))
+	{
+		exit(SS_RC_LIBSTRONGSWAN_INTEGRITY);
+	}
+
+	/* load sw-collector plugins */
+	if (!lib->plugins->load(lib->plugins,
+			lib->settings->get_str(lib->settings, "sw-collector.load", PLUGINS)))
+	{
+		exit(SS_RC_INITIALIZATION_FAILED);
+	}
+
+	/* connect to sw-collector database */
+	uri = lib->settings->get_str(lib->settings, "sw-collector.database", NULL);
+	if (!uri)
+	{
+		fprintf(stderr, "sw-collector.database URI not set.\n");
+		exit(EXIT_FAILURE);
+	}
+	db = sw_collector_db_create(uri);
+	if (!db)
+	{
+		fprintf(stderr, "connection to sw-collector database failed.\n");
+		exit(EXIT_FAILURE);
+	}
+
+	switch (op)
+	{
+		case COLLECTOR_OP_EXTRACT:
+			status = extract_history(db);
+			break;
+		case COLLECTOR_OP_LIST:
+			status = list_identifiers(db);
+			break;
+		default:
+			break;
+	}
+	db->destroy(db);
+
+	exit(status);
+}

src/libimcv/plugins/imc_swima/sw_collector/sw_collector_db.c

@@ -0,0 +1,313 @@
+/*
+ * Copyright (C) 2017 Andreas Steffen
+ * HSR Hochschule fuer Technik Rapperswil
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+
+#include "sw_collector_db.h"
+
+#include "swima/swima_event.h"
+
+typedef struct private_sw_collector_db_t private_sw_collector_db_t;
+
+/**
+ * Private data of an sw_collector_db_t object.
+ */
+struct private_sw_collector_db_t {
+
+	/**
+	 * Public members of sw_collector_db_state_t
+	 */
+	sw_collector_db_t public;
+
+	/**
+	 * Epoch
+	 */
+	uint32_t epoch;
+
+	/**
+	 * Event ID of last event stored in database
+	 */
+	uint32_t last_eid;
+
+	/**
+	 * Software collector database
+	 */
+	database_t *db;
+
+};
+
+METHOD(sw_collector_db_t, add_event, uint32_t,
+	private_sw_collector_db_t *this, char *timestamp)
+{
+	uint32_t eid = 0;
+
+	if (this->db->execute(this->db, &eid,
+			"INSERT INTO events (epoch, timestamp) VALUES (?, ?)",
+			 DB_UINT, this->epoch, DB_TEXT, timestamp) != 1)
+	{
+		DBG1(DBG_IMC, "unable to insert event into database");
+		return 0;
+	}
+
+	return eid;
+}
+
+METHOD(sw_collector_db_t, get_last_event, bool,
+	private_sw_collector_db_t *this, uint32_t *eid, uint32_t *epoch,
+	char **last_time)
+{
+	char *timestamp;
+	enumerator_t *e;
+
+	e = this->db->query(this->db,
+			"SELECT id, epoch, timestamp FROM events ORDER BY timestamp DESC",
+			DB_UINT, DB_UINT, DB_TEXT);
+	if (!e)
+	{
+		DBG1(DBG_IMC, "database query for event failed");
+		return FALSE;
+	}
+	if (e->enumerate(e, eid, epoch, &timestamp))
+	{
+		if (last_time)
+		{
+			*last_time = strdup(timestamp);
+		}
+	}
+	else
+	{
+		*eid = 0;
+	}
+	e->destroy(e);
+
+	return TRUE;
+}
+
+METHOD(sw_collector_db_t, add_sw_event, bool,
+	private_sw_collector_db_t *this, uint32_t eid, uint32_t sw_id,
+	uint8_t action)
+{
+	if (this->db->execute(this->db, NULL,
+			"INSERT INTO sw_events (eid, sw_id, action) VALUES (?, ?, ?)",
+			 DB_UINT, eid, DB_UINT, sw_id, DB_UINT, action) != 1)
+	{
+		DBG1(DBG_IMC, "unable to insert sw_event into database");
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+METHOD(sw_collector_db_t, get_sw_id, uint32_t,
+	private_sw_collector_db_t *this, char *package, char *version, char *name,
+	uint8_t source, bool installed, bool check)
+{
+	uint32_t sw_id = 0, status;
+	enumerator_t *e;
+
+	/* Does software identifier already exist in database? */
+	e = this->db->query(this->db,
+			"SELECT id, installed FROM sw_identifiers WHERE name = ?",
+			DB_TEXT, name, DB_UINT, DB_UINT);
+	if (!e)
+	{
+		DBG1(DBG_IMC, "database query for sw_identifier failed");
+		return 0;
+	}
+	if (!e->enumerate(e, &sw_id, &status))
+	{
+		sw_id = 0;
+	}
+	e->destroy(e);
+
+	if (sw_id)
+	{
+		if (status == installed)
+		{
+			if (!check)
+			{
+				DBG1(DBG_IMC, "  Warning: sw_id %u is already %s", sw_id,
+					 status ? "installed" : "deleted");
+			}
+			return sw_id;
+		}
+		if (check)
+		{
+			DBG1(DBG_IMC, "  Warning: sw_id %u is %s", sw_id,
+				 status ? "installed" : "deleted");
+		}
+
+		/* Change installation status */
+		if (this->db->execute(this->db, NULL,
+				"UPDATE sw_identifiers SET installed = ? WHERE id = ?",
+				 DB_UINT, installed, DB_UINT, sw_id) != 1)
+		{
+			DBG1(DBG_IMC, "unable to update sw_id status in database");
+			return 0;
+		}
+	}
+	else
+	{
+		/* Create new software identifier */
+		if (this->db->execute(this->db, &sw_id,
+				"INSERT INTO sw_identifiers "
+				"(name, package, version, source, installed) VALUES "
+				"(?, ?, ?, ?, ?)",
+				 DB_TEXT, name, DB_TEXT, package, DB_TEXT, version,
+				 DB_UINT, source, DB_UINT, installed) != 1)
+		{
+			DBG1(DBG_IMC, "unable to insert sw_id into database");
+			return 0;
+		}
+
+		if (check || !installed)
+		{
+			add_sw_event(this, 1, sw_id, SWIMA_EVENT_ACTION_CREATION);
+		}
+	}
+
+	return sw_id;
+}
+
+METHOD(sw_collector_db_t, get_sw_id_count, uint32_t,
+	private_sw_collector_db_t *this, bool installed_only)
+{
+	uint32_t count;
+	enumerator_t *e;
+
+	if (installed_only)
+	{
+		e = this->db->query(this->db,
+			"SELECT COUNT(installed) FROM sw_identifiers WHERE installed = 1 ",
+			 DB_UINT);
+	}
+	else
+	{
+		e = this->db->query(this->db,
+			"SELECT COUNT(installed) FROM sw_identifiers", DB_UINT);
+	}
+
+	if (!e)
+	{
+		DBG1(DBG_IMC, "database query for sw_identifier count failed");
+		return 0;
+	}
+	if (!e->enumerate(e, &count))
+	{
+		count = 0;
+	}
+	e->destroy(e);
+
+	return count;
+}
+
+METHOD(sw_collector_db_t, create_sw_enumerator, enumerator_t*,
+	private_sw_collector_db_t *this, bool installed_only)
+{
+	enumerator_t *e;
+
+	if (installed_only)
+	{
+		e = this->db->query(this->db,
+				"SELECT name, package, version, installed FROM sw_identifiers "
+				"WHERE installed = 1 ORDER BY name ASC",
+				 DB_TEXT, DB_TEXT, DB_TEXT, DB_UINT);
+	}
+	else
+	{
+		e = this->db->query(this->db,
+				"SELECT name, package, version, installed FROM sw_identifiers "
+				"ORDER BY name ASC", DB_TEXT, DB_TEXT, DB_TEXT, DB_UINT);
+	}
+	if (!e)
+	{
+		DBG1(DBG_IMC, "database query for sw_identifier count failed");
+		return NULL;
+	}
+
+	return e;
+}
+
+METHOD(sw_collector_db_t, destroy, void,
+	private_sw_collector_db_t *this)
+{
+	this->db->destroy(this->db);
+	free(this);
+}
+
+/**
+ * Described in header.
+ */
+sw_collector_db_t *sw_collector_db_create(char *uri)
+{
+	private_sw_collector_db_t *this;
+	uint32_t first_eid, last_eid;
+	char *first_time;
+
+	INIT(this,
+		.public = {
+			.add_event = _add_event,
+			.get_last_event = _get_last_event,
+			.add_sw_event = _add_sw_event,
+			.get_sw_id = _get_sw_id,
+			.get_sw_id_count = _get_sw_id_count,
+			.create_sw_enumerator = _create_sw_enumerator,
+			.destroy = _destroy,
+		},
+		.db = lib->db->create(lib->db, uri),
+	);
+
+	if (!this->db)
+	{
+		DBG1(DBG_IMC, "opening database URI '%s' failed", uri);
+		return NULL;
+	}
+
+	/* Retrieve last event in database */
+	if (!get_last_event(this, &last_eid, &this->epoch, NULL))
+	{
+		destroy(this);
+		return NULL;
+	}
+
+	/* Create random epoch and first event if no events exist yet */
+	if (!last_eid)
+	{
+		rng_t *rng;
+
+		rng = lib->crypto->create_rng(lib->crypto, RNG_STRONG);
+		if (!rng ||
+			!rng->get_bytes(rng, sizeof(uint32_t), (uint8_t*)&this->epoch))
+		{
+			DESTROY_IF(rng);
+			destroy(this);
+			DBG1(DBG_IMC, "generating random epoch value failed");
+			return NULL;
+		}
+		rng->destroy(rng);
+
+		/* Create first event when the OS was installed */
+		first_time = lib->settings->get_str(lib->settings,
+						"sw-collector.first_time", "0000-00-00T00:00:00Z");
+		first_eid = add_event(this, first_time);
+		if (!first_eid)
+		{
+			destroy(this);
+			return NULL;
+		}
+		DBG0(DBG_IMC, "First-Date: %s, eid = %u, epoch = %u",
+					   first_time, first_eid, this->epoch);
+	}
+
+	return &this->public;
+}

src/libimcv/plugins/imc_swima/sw_collector/sw_collector_db.h

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2017 Andreas Steffen
+ * HSR Hochschule fuer Technik Rapperswil
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+
+/**
+ * @defgroup sw_collector_db_t sw_collector_db
+ * @{ @ingroup imc_swima
+ */
+
+#ifndef SW_COLLECTOR_DB_H_
+#define SW_COLLECTOR_DB_H_
+
+#include <library.h>
+
+typedef struct sw_collector_db_t sw_collector_db_t;
+
+/**
+ * Software collector database object
+ */
+struct sw_collector_db_t {
+
+	/**
+	 * Add event to database
+	 *
+	 * @param timestamp		Timestamp in 20 octet RFC 3339 format
+	 * @return				Primary key pointing to event ID or 0 if failed
+	 */
+	uint32_t (*add_event)(sw_collector_db_t *this, char *timestamp);
+
+	/**
+	 * Get last event, zero EID if none exists
+	 *
+	 * @param eid			Primary key pointing to last event
+	 * @param epoch			Epoch
+	 * @param last_time		Timestamp in 20 octet RFC 3339 format of last event
+	 * @return				
+	 */
+	bool (*get_last_event)(sw_collector_db_t *this, uint32_t *eid,
+							   uint32_t *epoch, char **last_time);
+
+	/**
+	 * Add software identifier event to database
+	 *
+	 * @param eid			Foreign key pointing to an event ID
+	 * @param sw_id			Foreign key pointing to a software identifier
+	 * @param action		1 for CREATION, 2 for deletion
+	 * @return				TRUE if successful
+	 */
+	bool (*add_sw_event)(sw_collector_db_t *this, uint32_t eid, uint32_t sw_id,
+						 uint8_t action);
+
+	/**
+	 * Get software_identifier, creating one if it doesn't exist yet
+	 *
+	 * @param package		Software package
+	 * @param version		Version of software package
+	 * @param name			Software identifier
+	 * @param source		Source ID of the software collector
+	 * @param installed		Installation status to be set, TRUE if installed
+	 * @param check			Check if SW ID is already installed
+	 * @return				Primary key pointing to SW ID or 0 if failed
+	 */
+	uint32_t (*get_sw_id)(sw_collector_db_t *this, char *package, char *version,
+						  char *name, uint8_t source, bool installed, bool check);
+
+	/**
+	 * Get number of installed or deleted software identifiers
+	 *
+	 * @param installed_only	Count installed SW IDs if TRUE
+	 * @return					Count
+	 */
+	uint32_t (*get_sw_id_count)(sw_collector_db_t *this, bool installed_only);
+
+	/**
+	 * Enumerate over all collected [installed] software identities
+	 *
+	 * @param installed_only	Return only installed software identities
+	 * @return					Enumerator
+	 */
+	enumerator_t* (*create_sw_enumerator)(sw_collector_db_t *this,
+										  bool installed_only);
+
+	/**
+	 * Destroy sw_collector_db_t object
+	 */
+	void (*destroy)(sw_collector_db_t *this);
+
+};
+
+/**
+ * Create an sw_collector_db_t instance
+ *
+ * @param uri				database URI
+ */
+sw_collector_db_t* sw_collector_db_create(char *uri);
+
+#endif /** SW_COLLECTOR_DB_H_ @}*/

src/libimcv/plugins/imc_swima/sw_collector/sw_collector_history.c

@@ -0,0 +1,458 @@
+/*
+ * Copyright (C) 2017 Andreas Steffen
+ * HSR Hochschule fuer Technik Rapperswil
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <time.h>
+
+#include "sw_collector_history.h"
+
+#include "imc/imc_os_info.h"
+#include "swima/swima_event.h"
+
+typedef struct private_sw_collector_history_t private_sw_collector_history_t;
+
+/**
+ * Private data of an sw_collector_history_t object.
+ */
+struct private_sw_collector_history_t {
+
+	/**
+	 * Public members of sw_collector_history_state_t
+	 */
+	sw_collector_history_t public;
+
+	/**
+	 * tagCreator
+	 */
+	char *tag_creator;
+
+	/**
+	 * OS string 'name_version-arch'
+	 */
+	char *os;
+
+	/**
+	 * OS info about endpoint
+	 */
+	imc_os_info_t *os_info;
+
+	/**
+	 * Software Event Source Number
+	 */
+	uint8_t source;
+
+	/**
+	 * Reference to collector database
+	 */
+	sw_collector_db_t *db;
+
+};
+
+/**
+ * Define auxiliary package_t list item object
+ */
+typedef struct package_t package_t;
+
+struct package_t {
+	char *package;
+	char *version;
+	char *old_version;
+	char *sw_id;
+	char *old_sw_id;
+};
+
+/**
+ * Replaces invalid character by a valid one
+ */
+static void sanitize_uri(char *uri, char a, char b)
+{
+	char *pos = uri;
+
+	while (TRUE)
+	{
+		pos = strchr(pos, a);
+		if (!pos)
+		{
+			break;
+		}
+		*pos = b;
+		pos++;
+	}
+}
+
+/**
+ * Create software identifier
+ */
+char* create_sw_id(char *tag_creator, char *os, char *package, char *version)
+{
+	char *pos, *sw_id;
+	size_t len;
+
+	/* Remove architecture from package name */
+	pos = strchr(package, ':');
+	len = pos ? (pos - package) : strlen(package);
+
+	/* Build software identifier */
+	if (asprintf(&sw_id, "%s__%s-%.*s%s%s", tag_creator, os, len, package,
+				 strlen(version) ? "-" : "", version) == -1)
+	{
+		return NULL;
+	}
+	sanitize_uri(sw_id, ':', '~');
+	sanitize_uri(sw_id, '+', '~');
+
+	return sw_id;
+}
+
+/**
+ * Create package_t list item object
+ */
+static package_t* create_package(char* tag_creator, char *os, chunk_t package,
+								 chunk_t version, chunk_t old_version)
+{
+	package_t *this;
+
+	INIT(this,
+		.package = strndup(package.ptr, package.len),
+		.version = strndup(version.ptr, version.len),
+		.old_version = strndup(old_version.ptr, old_version.len),
+	)
+
+	this->sw_id = create_sw_id(tag_creator, os, this->package, this->version);
+	if (old_version.len)
+	{
+		this->old_sw_id = create_sw_id(tag_creator, os, this->package,
+									   this->old_version);
+	}
+
+	return this;
+}
+
+/**
+ * Free package_t list item object
+ */
+static void free_package(package_t *this)
+{
+	if (this)
+	{
+		free(this->package);
+		free(this->version);
+		free(this->old_version);
+		free(this->sw_id);
+		free(this->old_sw_id);
+		free(this);
+	}
+}
+
+/**
+ * Extract and parse a single package item
+ */
+static package_t* extract_package(chunk_t item, char *tag_creator, char *os,
+								  sw_collector_history_op_t op)
+{
+	chunk_t package, version, old_version;
+	package_t *p;
+
+	/* extract package name */
+	if (!extract_token(&package, ' ', &item))
+	{
+		fprintf(stderr, "version not found.\n");
+		return NULL;
+	}
+	item = chunk_skip(item, 1);
+
+	/* extract versions */
+	version = old_version = chunk_empty;
+
+	if (item.len > 0)
+	{
+		if (extract_token(&version, ',', &item))
+		{
+			eat_whitespace(&item);
+			if (!match("automatic", &item))
+			{
+				old_version = version;
+				version = item;
+			}
+		}
+		else
+		{
+			version = item;
+		}
+	}
+	p = create_package(tag_creator, os, package, version, old_version);
+
+	/* generate log entry */
+	if (op == SW_OP_UPGRADE)
+	{
+		DBG2(DBG_IMC, "    %s (%s, %s)", p->package, p->old_version, p->version);
+		DBG2(DBG_IMC, "      +%s", p->sw_id);
+		DBG2(DBG_IMC, "      -%s", p->old_sw_id);
+	}
+	else
+	{
+		DBG2(DBG_IMC, "    %s (%s)", p->package, p->version);
+		DBG2(DBG_IMC, "      %s%s", (op == SW_OP_INSTALL) ? "+" : "-", p->sw_id);
+	}
+
+	return p;
+}
+
+METHOD(sw_collector_history_t, extract_timestamp, bool,
+	private_sw_collector_history_t *this, chunk_t args, char *buf)
+{
+	struct tm loc, utc;
+	chunk_t t1, t2;
+	time_t t;
+
+	/* Break down local time with format t1 = yyyy-mm-dd and t2 = hh:mm:ss */
+	if (!eat_whitespace(&args) || !extract_token(&t1, ' ', &args) ||
+		!eat_whitespace(&args) || t1.len != 10 || args.len != 8)
+	{
+		DBG1(DBG_IMC, "unable to parse start-date");
+		return FALSE;
+	}
+	t2 = args;
+
+	if (sscanf(t1.ptr, "%4d-%2d-%2d",
+						&loc.tm_year, &loc.tm_mon, &loc.tm_mday) != 3)
+	{
+		DBG1(DBG_IMC, "unable to parse date format yyyy-mm-dd");
+		return FALSE;
+	}
+	loc.tm_year -= 1900;
+	loc.tm_mon  -= 1;
+	loc.tm_isdst = -1;
+
+	if (sscanf(t2.ptr, "%2d:%2d:%2d",
+						&loc.tm_hour, &loc.tm_min, &loc.tm_sec) != 3)
+	{
+		DBG1(DBG_IMC, "unable to parse time format hh:mm:ss");
+		return FALSE;
+	}
+
+	/* Convert from local time to UTC */
+	t = mktime(&loc);
+	gmtime_r(&t, &utc);
+	utc.tm_year += 1900;
+	utc.tm_mon  += 1;
+
+	/* Form timestamp according to RFC 3339 (20 characters) */
+	snprintf(buf, 21, "%4d-%02d-%02dT%02d:%02d:%02dZ",
+			 utc.tm_year, utc.tm_mon, utc.tm_mday,
+			 utc.tm_hour, utc.tm_min, utc.tm_sec);
+
+	return TRUE;
+}
+
+METHOD(sw_collector_history_t, extract_packages, bool,
+	private_sw_collector_history_t *this, chunk_t args, uint32_t eid,
+	sw_collector_history_op_t op)
+{
+	package_t *p = NULL;
+	uint32_t sw_id;
+	chunk_t item;
+	bool success = FALSE;
+
+	eat_whitespace(&args);
+
+	while (extract_token(&item, ')', &args))
+	{
+		p = extract_package(item, this->tag_creator, this->os, op);
+		if (!p)
+		{
+			goto end;
+		}
+
+		sw_id = this->db->get_sw_id(this->db, p->package, p->version, p->sw_id,
+									this->source, op != SW_OP_REMOVE, FALSE);
+		if (!sw_id)
+		{
+			goto end;
+		}
+		if (!this->db->add_sw_event(this->db, eid, sw_id, op != SW_OP_REMOVE ?
+					SWIMA_EVENT_ACTION_CREATION : SWIMA_EVENT_ACTION_DELETION))
+		{
+			goto end;
+		}
+
+		if (op == SW_OP_UPGRADE)
+		{
+			sw_id = this->db->get_sw_id(this->db, p->package, p->old_version,
+										p->old_sw_id, this->source, FALSE, FALSE);
+			if (!sw_id)
+			{
+				goto end;
+			}
+			if (!this->db->add_sw_event(this->db, eid, sw_id,
+										SWIMA_EVENT_ACTION_DELETION))
+			{
+				goto end;
+			}
+		}
+		free_package(p);
+
+		if (args.len < 2)
+		{
+			break;
+		}
+		args = chunk_skip(args, 2);
+	}
+	p = NULL;
+	success = TRUE;
+
+end:
+	free_package(p);
+
+	return success;
+}
+
+METHOD(sw_collector_history_t, merge_installed_packages, bool,
+	private_sw_collector_history_t *this)
+{
+	FILE *file;
+	uint32_t sw_id, count = 0;
+	char line[BUF_LEN], *pos, *package, *version, *state, *name;
+	bool success = FALSE;
+	char cmd[] = "dpkg-query -W -f=\'${Package}\t${Version}\t${Status}\n\'";
+
+	DBG1(DBG_IMC, "Merging:");
+
+	file = popen(cmd, "r");
+	if (!file)
+	{
+		DBG1(DBG_IMC, "failed to run dpgk-query command");
+		return FALSE;
+	}
+
+	while (TRUE)
+	{
+		if (!fgets(line, sizeof(line), file))
+		{
+			break;
+		}
+
+		package = line;
+		pos = strchr(line, '\t');
+		if (!pos)
+		{
+			goto end;
+		}
+		*pos = '\0';
+
+		version = ++pos;
+		pos = strchr(pos, '\t');
+		if (!pos)
+		{
+			goto end;
+		}
+		*pos = '\0';
+
+		state = ++pos;
+		pos = strchr(pos, '\n');
+		if (!pos)
+		{
+			goto end;
+		}
+		*pos = '\0';
+
+		if (!streq(state, "install ok installed"))
+		{
+			continue;
+		}
+		name = create_sw_id(this->tag_creator, this->os, package, version);
+		DBG3(DBG_IMC, "  %s merged", name);
+
+		sw_id = this->db->get_sw_id(this->db, package, version, name,
+									this->source, TRUE, TRUE);
+		free(name);
+		if (!sw_id)
+		{
+			goto end;
+		}
+		count++;
+	}
+	success = TRUE;
+	DBG1(DBG_IMC, "  merged %u installed packages, %u registed in database",
+		 count, this->db->get_sw_id_count(this->db, TRUE));
+
+end:
+	pclose(file);
+	return success;
+}
+
+METHOD(sw_collector_history_t, destroy, void,
+	private_sw_collector_history_t *this)
+{
+	this->os_info->destroy(this->os_info);
+	free(this->os);
+	free(this);
+}
+
+/**
+ * Described in header.
+ */
+sw_collector_history_t *sw_collector_history_create(sw_collector_db_t *db,
+													uint8_t source)
+{
+	private_sw_collector_history_t *this;
+	chunk_t os_name, os_version, os_arch;
+	os_type_t os_type;
+
+	INIT(this,
+		.public = {
+			.extract_timestamp = _extract_timestamp,
+			.extract_packages = _extract_packages,
+			.merge_installed_packages = _merge_installed_packages,
+			.destroy = _destroy,
+		},
+		.db = db,
+		.source = source,
+		.os_info = imc_os_info_create(),
+		.tag_creator = lib->settings->get_str(lib->settings,
+				"sw-collector.tag_creator", "strongswan.org"),
+	);
+
+	os_type = this->os_info->get_type(this->os_info);
+	os_name = this->os_info->get_name(this->os_info);
+	os_arch = this->os_info->get_version(this->os_info);
+
+	/* check if OS is supported */
+	if (os_type != 	OS_TYPE_DEBIAN && os_type != OS_TYPE_UBUNTU)
+	{
+		DBG1(DBG_IMC, "%.*s OS not supported", os_name.len, os_name.ptr);
+		destroy(this);
+		return NULL;
+	}
+
+	/* get_version() returns version followed by arch */ 
+	if (!extract_token(&os_version, ' ', &os_arch))
+	{
+		DBG1(DBG_IMC, "separation of OS version from arch failed");
+		destroy(this);
+		return NULL;
+	}
+	if (asprintf(&this->os, "%.*s_%.*s-%.*s", os_name.len, os_name.ptr,
+											  os_version.len, os_version.ptr,
+		 									  os_arch.len, os_arch.ptr) == -1)
+	{
+		DBG1(DBG_IMC, "constructon of OS string failed");
+		destroy(this);
+		return NULL;
+	}
+
+	return &this->public;
+}

src/libimcv/plugins/imc_swima/sw_collector/sw_collector_history.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2017 Andreas Steffen
+ * HSR Hochschule fuer Technik Rapperswil
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+
+/**
+ * @defgroup sw_collector_history_t sw_collector_history
+ * @{ @ingroup imc_swima
+ */
+
+#ifndef SW_COLLECTOR_HISTORY_H_
+#define SW_COLLECTOR_HISTORY_H_
+
+#include "sw_collector_db.h"
+
+#include <library.h>
+#include <utils/debug.h>
+#include <utils/lexparser.h>
+
+typedef struct sw_collector_history_t sw_collector_history_t;
+typedef enum sw_collector_history_op_t sw_collector_history_op_t;
+
+/**
+ * Define major history event operations
+ */
+enum sw_collector_history_op_t {
+	SW_OP_INSTALL,
+	SW_OP_UPGRADE,
+	SW_OP_REMOVE
+};
+
+/**
+ * Software collector history object
+ */
+struct sw_collector_history_t {
+
+	/**
+	 * Extract timestamp from event in installation history
+	 *
+	 * @param args			Arguments to be processed
+	 * @param buf			timestamp buffer for 21 byte RFC 3339 string
+	 * @return				TRUE if extraction succeeded
+	 */
+	bool (*extract_timestamp)(sw_collector_history_t *this, chunk_t args,
+							  char *buf);
+
+	/**
+	 * Extract packages from event in installation history 
+	 *
+	 * @param args			Arguments to be processed
+	 * @param eid			Primary key pointing to current event
+	 * @param op			Extraction operation 
+	 * @return				TRUE if extraction succeeded
+	 */
+	bool (*extract_packages)(sw_collector_history_t *this, chunk_t args,
+							 uint32_t eid, sw_collector_history_op_t op);
+
+	/**
+	 * Merge packages from initial installation
+	 *
+	 * @return				TRUE if merge succeeded
+	 */
+	bool (*merge_installed_packages)(sw_collector_history_t *this);
+
+	/**
+	 * Destroy sw_collector_history_t object
+	 */
+	void (*destroy)(sw_collector_history_t *this);
+
+};
+
+/**
+ * Create an sw_collector_history_t instance
+ *
+ * @param db				Internal reference to collector database
+ * @param source			Software event source number
+ */
+sw_collector_history_t* sw_collector_history_create(sw_collector_db_t *db,
+													uint8_t source);
+
+#endif /** SW_COLLECTOR_HISTORY_H_ @}*/

src/libimcv/plugins/imc_swima/sw_collector/sw_collector_tables.sql

@@ -0,0 +1,31 @@
+/* SQLit database for an Endpoint Collector */
+
+DROP TABLE IF EXISTS "events";
+CREATE TABLE "events" (
+  "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+  "epoch" INTEGER NOT NULL,
+  "timestamp" CHAR(20) NOT NULL
+);
+
+DROP TABLE IF EXISTS "sw_identifiers";
+CREATE TABLE "sw_identifiers" (
+  "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+  "name" VARCHAR(255) NOT NULL,
+  "package" VARCHAR(255) NOT NULL,
+  "version" VARCHAR(255) NOT NULL,
+  "source" INTEGER DEFAULT 0,
+  "installed" INTEGER DEFAULT 1,
+  "tag" TEXT
+  );
+DROP INDEX IF EXISTS "sw_identifiers_name";
+CREATE INDEX "sw_identifiers_name" ON "sw_identifiers" (
+  "name"
+);
+
+DROP TABLE IF EXISTS "sw_events";
+CREATE TABLE "sw_events" (
+  "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+  "eid" INTEGER REFERENCES "events" ("id"),
+  "sw_id" INTEGER NOT NULL REFERENCES "sw_identifiers" ("id"),
+  "action" INTEGER NOT NULL
+);