sharpetronics/mailcow-dockerized
1
0
Fork 0
mirror of https://github.com/mailcow/mailcow-dockerized.git synced 2025-11-27 00:04:39 -05:00
Code Issues Projects Releases Wiki Activity

[DB][Web] optimize qhandler by keeping SHA2 in new column qhash (#6556)

Browse Source 
* [DB][Web] optimize qhandler by keeping SHA2(id+qid) in new column quarantine.qhash, for feature #6555, might also help with #6361

* rspamd: only add qhash to new entries while passing rspamd not all existing

* compose: bump dovecot image + push to registry

---------

Co-authored-by: DerLinkman <niklas.meyer@servercow.de>
This commit is contained in:
Christoph Lechleitner 2025-08-06 16:02:25 +02:00 committed by GitHub
parent 8cdb0b869e
commit 2b2da1679e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/Dockerfiles/dovecot/quarantine_notify.py
View File

@ -76,7 +76,7 @@ try:
def notify_rcpt(rcpt, msg_count, quarantine_acl, category): def notify_rcpt(rcpt, msg_count, quarantine_acl, category):
if category == "add_header": category = "add header" if category == "add_header": category = "add header"
meta_query = query_mysql('SELECT SHA2(CONCAT(id, qid), 256) AS qhash, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = "%s" AND score < %f AND (action = "%s" OR "all" = "%s")' % (rcpt, max_score, category, category)) meta_query = query_mysql('SELECT `qhash`, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = "%s" AND score < %f AND (action = "%s" OR "all" = "%s")' % (rcpt, max_score, category, category))
print("%s: %d of %d messages qualify for notification" % (rcpt, len(meta_query), msg_count)) print("%s: %d of %d messages qualify for notification" % (rcpt, len(meta_query), msg_count))
if len(meta_query) == 0: if len(meta_query) == 0:
return return

3
data/conf/rspamd/meta_exporter/pipe.php
View File

@ -236,6 +236,9 @@ foreach ($rcpt_final_mailboxes as $rcpt_final) {
':action' => $action, ':action' => $action,
':fuzzy_hashes' => $fuzzy ':fuzzy_hashes' => $fuzzy
)); ));
$lastId = $pdo->lastInsertId();
$stmt_update = $pdo->prepare("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE `id` = :id");
$stmt_update->execute(array(':id' => $lastId));
$stmt = $pdo->prepare('DELETE FROM `quarantine` WHERE `rcpt` = :rcpt AND `id` NOT IN ( $stmt = $pdo->prepare('DELETE FROM `quarantine` WHERE `rcpt` = :rcpt AND `id` NOT IN (
SELECT `id` SELECT `id`
FROM ( FROM (

6
data/web/inc/functions.quarantine.inc.php
View File

@ -22,7 +22,7 @@ function quarantine($_action, $_data = null) {
return false; return false;
} }
$stmt = $pdo->prepare('SELECT `id` FROM `quarantine` LEFT OUTER JOIN `user_acl` ON `user_acl`.`username` = `rcpt` $stmt = $pdo->prepare('SELECT `id` FROM `quarantine` LEFT OUTER JOIN `user_acl` ON `user_acl`.`username` = `rcpt`
WHERE SHA2(CONCAT(`id`, `qid`), 256) = :hash WHERE `qhash` = :hash
AND user_acl.quarantine = 1 AND user_acl.quarantine = 1
AND rcpt IN (SELECT username FROM mailbox)'); AND rcpt IN (SELECT username FROM mailbox)');
$stmt->execute(array(':hash' => $hash)); $stmt->execute(array(':hash' => $hash));
@ -65,7 +65,7 @@ function quarantine($_action, $_data = null) {
return false; return false;
} }
$stmt = $pdo->prepare('SELECT `id` FROM `quarantine` LEFT OUTER JOIN `user_acl` ON `user_acl`.`username` = `rcpt` $stmt = $pdo->prepare('SELECT `id` FROM `quarantine` LEFT OUTER JOIN `user_acl` ON `user_acl`.`username` = `rcpt`
WHERE SHA2(CONCAT(`id`, `qid`), 256) = :hash WHERE `qhash` = :hash
AND `user_acl`.`quarantine` = 1 AND `user_acl`.`quarantine` = 1
AND `username` IN (SELECT `username` FROM `mailbox`)'); AND `username` IN (SELECT `username` FROM `mailbox`)');
$stmt->execute(array(':hash' => $hash)); $stmt->execute(array(':hash' => $hash));
@ -833,7 +833,7 @@ function quarantine($_action, $_data = null) {
))); )));
return false; return false;
} }
$stmt = $pdo->prepare('SELECT * FROM `quarantine` WHERE SHA2(CONCAT(`id`, `qid`), 256) = :hash'); $stmt = $pdo->prepare('SELECT * FROM `quarantine` WHERE `qhash` = :hash');
$stmt->execute(array(':hash' => $hash)); $stmt->execute(array(':hash' => $hash));
return $stmt->fetch(PDO::FETCH_ASSOC); return $stmt->fetch(PDO::FETCH_ASSOC);
break; break;

10
data/web/inc/init_db.inc.php
View File

@ -4,7 +4,7 @@ function init_db_schema()
try { try {
global $pdo; global $pdo;
$db_version = "27012025_1555"; $db_version = "21052025_2215";
$stmt = $pdo->query("SHOW TABLES LIKE 'versions'"); $stmt = $pdo->query("SHOW TABLES LIKE 'versions'");
$num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@ -345,10 +345,14 @@ function init_db_schema()
"notified" => "TINYINT(1) NOT NULL DEFAULT '0'", "notified" => "TINYINT(1) NOT NULL DEFAULT '0'",
"created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
"user" => "VARCHAR(255) NOT NULL DEFAULT 'unknown'", "user" => "VARCHAR(255) NOT NULL DEFAULT 'unknown'",
"qhash" => "VARCHAR(255)",
), ),
"keys" => array( "keys" => array(
"primary" => array( "primary" => array(
"" => array("id") "" => array("id")
),
"key" => array(
"qhash" => array("qhash")
) )
), ),
"attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
@ -1482,6 +1486,10 @@ function init_db_schema()
'msg' => 'db_init_complete' 'msg' => 'db_init_complete'
); );
} }
// fill quarantine.qhash
$pdo->query("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE ISNULL(`qhash`)");
} catch (PDOException $e) { } catch (PDOException $e) {
if (php_sapi_name() == "cli") { if (php_sapi_name() == "cli") {
echo "DB initialization failed: " . print_r($e, true) . PHP_EOL; echo "DB initialization failed: " . print_r($e, true) . PHP_EOL;