Manage targets and certificates based on their availability

raviks789 · raviks789 · commit 66f6335885e8 · 2022-10-21T12:47:28.000+02:00
The following two parameters are introduced to `icingacli x509 scan` command to manage targets and certificates:
1) `since-last-scan` - used to check when the target was last scanned (stored in column `last_scan` of table `x509_target`)
2) `since-last-seen` - used to check when the target or certificate was last seen (stored in column `last_seen of tables
`x509_target` and `x509_certificate`).
diff --git a/application/clicommands/CheckCommand.php b/application/clicommands/CheckCommand.php
@@ -4,6 +4,7 @@
 
 namespace Icinga\Module\X509\Clicommands;
 
+use DateTime;
 use Icinga\Application\Logger;
 use Icinga\Module\X509\Command;
 use Icinga\Module\X509\DbTool;
@@ -119,9 +120,9 @@ public function hostAction()
                 $state = 2;
             }
 
-            $now = new \DateTime();
-            $validFrom = (new \DateTime())->setTimestamp($target['valid_from']);
-            $validTo = (new \DateTime())->setTimestamp($target['valid_to']);
+            $now = new DateTime();
+            $validFrom = (new DateTime())->setTimestamp($target->valid_from);
+            $validTo = (new DateTime())->setTimestamp($target->valid_to);
             $criticalAfter = $this->thresholdToDateTime($validFrom, $validTo, $criticalThreshold, $criticalUnit);
             $warningAfter = $this->thresholdToDateTime($validFrom, $validTo, $warningThreshold, $warningUnit);
 
diff --git a/application/clicommands/ImportCommand.php b/application/clicommands/ImportCommand.php
@@ -4,6 +4,7 @@
 
 namespace Icinga\Module\X509\Clicommands;
 
+use DateTime;
 use Icinga\Application\Logger;
 use Icinga\Module\X509\CertificateUtils;
 use Icinga\Module\X509\Command;
@@ -45,7 +46,7 @@ public function indexAction()
 
                 $db->update(
                     'x509_certificate',
-                    ['trusted' => 'y'],
+                    ['trusted' => 'y', 'last_seen' => (new DateTime())->getTimestamp()],
                     ['id = ?' => $id]
                 );
 
diff --git a/application/clicommands/ScanCommand.php b/application/clicommands/ScanCommand.php
@@ -4,28 +4,36 @@
 
 namespace Icinga\Module\X509\Clicommands;
 
+use DateTime;
 use Icinga\Application\Logger;
 use Icinga\Module\X509\CertificateUtils;
 use Icinga\Module\X509\Command;
 use Icinga\Module\X509\Hook\SniHook;
 use Icinga\Module\X509\Job;
+use InvalidArgumentException;
+use ipl\Sql\Connection;
+use ipl\Sql\Expression;
 
 class ScanCommand extends Command
 {
     /**
      * Scans IP and port ranges to find X.509 certificates.
      *
      * This command starts scanning the IP and port ranges which belong to the job that was specified with the
-     * --job parameter.
+     * --job parameter and optional --since-last-scan and --since-last-seen parameters.
      *
      * USAGE
      *
-     * icingacli x509 scan --job <name>
+     * icingacli x509 scan --job <name> --since-last-scan <duration> --since-last-seen <duration>
      */
     public function indexAction()
     {
         $name = $this->params->shiftRequired('job');
 
+        $sinceLastSeen = $this->processThresholdToDateTiime($this->params->get('since-last-seen'));
+
+        $sinceLastScan = $this->processThresholdToDateTiime($this->params->get('since-last-scan'));
+
         $parallel = (int) $this->Config()->get('scan', 'parallel', 256);
 
         if ($parallel <= 0) {
@@ -58,9 +66,66 @@ public function indexAction()
                 $name
             );
 
+            if ($sinceLastScan !== null) {
+                $this->cleanupLastScanTargets($this->getDb(), $sinceLastScan);
+            }
+
             $verified = CertificateUtils::verifyCertificates($this->getDb());
 
+            if ($sinceLastSeen !== null) {
+                $this->cleanupLastSeenTargets($this->getDb(), $sinceLastSeen);
+                CertificateUtils::cleanupCertificates($this->getDb(), $sinceLastSeen);
+            }
+
+            CertificateUtils::cleanupCertificateChains($this->getDb());
             Logger::info("Checked %d certificate chain%s.", $verified, $verified !== 1 ? 's' : '');
         }
     }
+
+    protected function cleanupLastScanTargets(Connection $db, DateTime $sincLastScan)
+    {
+        $db->delete(
+            'x509_target',
+            ['last_scan < ?' => $sincLastScan->getTimestamp()]
+        );
+    }
+
+    protected function cleanupLastSeenTargets(Connection $db, DateTime $sincLastSeen)
+    {
+        $db->delete(
+            'x509_target',
+            [new Expression(sprintf(
+                "last_seen < last_scan - %d",
+                (new DateTime())->getTimestamp() - $sincLastSeen->getTimestamp()
+            ))]
+        );
+    }
+
+    /**
+     * @param string|null $threshold
+     * @return DateTime|null
+     */
+    protected function processThresholdToDateTiime(?string $threshold)
+    {
+        $since = $threshold;
+        if ($threshold !== null) {
+            if ($since[0] !== '-') {
+                // When the user specified "2 days" as a threshold strtotime() will compute the
+                // timestamp NOW() + 2 days, but it has to be NOW() + (-2 days)
+                $since = "-$since";
+            }
+
+            $since = strtotime($since);
+            if ($since === false) {
+                throw new InvalidArgumentException(sprintf(
+                    "The specified last scan time is in an unknown format: '%s'",
+                    $threshold
+                ));
+            }
+
+            $since = (new DateTime())->setTimestamp($since);
+        }
+
+        return $since;
+    }
 }
diff --git a/etc/schema/mysql.schema.sql b/etc/schema/mysql.schema.sql
@@ -18,6 +18,7 @@ CREATE TABLE x509_certificate (
   fingerprint binary(32) NOT NULL COMMENT 'sha256 hash',
   `serial` blob NOT NULL,
   certificate blob NOT NULL COMMENT 'DER encoded certificate',
+  last_seen bigint(20) NOT NULL,
   ctime timestamp NULL DEFAULT NULL,
   mtime timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (id),
@@ -86,6 +87,8 @@ CREATE TABLE x509_target (
   hostname varchar(255) NULL DEFAULT NULL,
   latest_certificate_chain_id int(10) unsigned NULL DEFAULT NULL,
   ctime timestamp NULL DEFAULT NULL,
+  last_scan bigint(20) NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW())),
+  last_seen bigint(20) NOT NULL,
   mtime timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (id),
   INDEX x509_idx_target_ip_port (ip, port)
diff --git a/etc/schema/postgresql.schema.sql b/etc/schema/postgresql.schema.sql
@@ -35,6 +35,7 @@ CREATE TABLE x509_certificate (
   valid_to bigint NOT NULL,
   fingerprint bytea NOT NULL,
   serial bytea NOT NULL,
+  last_seen bigint NOT NULL,
   certificate bytea NOT NULL,
   ctime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
   mtime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -98,6 +99,8 @@ CREATE TABLE x509_target (
   port uint2 NOT NULL,
   hostname varchar(255) NULL DEFAULT NULL,
   latest_certificate_chain_id int NULL DEFAULT NULL,
+  last_seen bigint NOT NULL,
+  last_scan bigint NOT NULL,
   ctime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
   mtime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
diff --git a/library/X509/CertificateUtils.php b/library/X509/CertificateUtils.php
@@ -4,6 +4,7 @@
 
 namespace Icinga\Module\X509;
 
+use DateTime;
 use Exception;
 use Icinga\Application\Logger;
 use Icinga\File\Storage\TemporaryLocalFileStorage;
@@ -244,7 +245,8 @@ public static function findOrInsertCert(Connection $db, $cert)
                 'valid_to'            => $certInfo['validTo_time_t'],
                 'fingerprint'         => $dbTool->marshalBinary($fingerprint),
                 'serial'              => $dbTool->marshalBinary(gmp_export($certInfo['serialNumber'])),
-                'certificate'         => $dbTool->marshalBinary($der)
+                'certificate'         => $dbTool->marshalBinary($der),
+                'last_seen'           => (new DateTime())->getTimestamp()
             ]
         );
 
@@ -353,6 +355,27 @@ private static function findOrInsertDn($db, $certInfo, $type)
         return $hash;
     }
 
+    public static function cleanupCertificates(Connection $db, DateTime $lastSeen)
+    {
+        $db->delete(
+            'x509_certificate',
+            [
+                'last_seen < ?' => $lastSeen->getTimestamp(),
+                'trusted = ?' => 'no'
+            ]
+        );
+    }
+
+    public static function cleanupCertificateChains(Connection $db)
+    {
+        $db->delete(
+            'x509_certificate_chain',
+            ['target_id NOT IN (?)' => (new Select())
+                ->from('x509_target')
+                ->columns(['id'])]
+        );
+    }
+
     /**
      * Verify certificates
      *
@@ -414,6 +437,12 @@ public static function verifyCertificates(Connection $db)
                 $collection = [];
 
                 foreach ($certs as $cert) {
+                    $db->update(
+                        'x509_certificate',
+                        ['last_seen' => (new DateTime())->getTimestamp()],
+                        ['id = ?' => $cert->id]
+                    );
+
                     $collection[] = CertificateUtils::der2pem(DbTool::unmarshalBinary($cert->certificate));
                 }
 
diff --git a/library/X509/Job.php b/library/X509/Job.php
@@ -4,6 +4,7 @@
 
 namespace Icinga\Module\X509;
 
+use DateTime;
 use Icinga\Application\Config;
 use Icinga\Application\Logger;
 use Icinga\Data\ConfigObject;
@@ -41,6 +42,7 @@ class Job
     private $snimap;
     private $parallel;
     private $name;
+    private $currentDateTime;
 
     public function __construct($name, ConfigObject $jobDescription, array $snimap, $parallel)
     {
@@ -50,6 +52,7 @@ public function __construct($name, ConfigObject $jobDescription, array $snimap,
         $this->snimap = $snimap;
         $this->parallel = $parallel;
         $this->name = $name;
+        $this->currentDateTime = (new DateTime())->getTimestamp();
     }
 
     private function getConnector($peerName)
@@ -203,6 +206,16 @@ function (ConnectionInterface $conn) use ($target, $streamCapture) {
                 // Close connection in order to capture stream context options
                 $conn->close();
 
+                $this->db->update(
+                    'x509_target',
+                    ['last_seen' => $this->currentDateTime],
+                    [
+                        'ip = ?'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
+                        'port = ?'     => $target->port,
+                        'hostname = ?' => $target->hostname
+                    ]
+                );
+
                 $capturedStreamOptions = $streamCapture->getCapturedStreamOptions();
 
                 $this->processChain($target, $capturedStreamOptions['ssl']['peer_certificate_chain']);
@@ -240,6 +253,16 @@ function (\Exception $exception) use ($target, $streamCapture) {
         )->otherwise(function (\Exception $e) {
             echo $e->getMessage() . PHP_EOL;
             echo $e->getTraceAsString() . PHP_EOL;
+        })->always(function () use ($target) {
+            $this->db->update(
+                'x509_target',
+                ['last_scan' => $this->currentDateTime],
+                [
+                    'ip = ?'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
+                    'port = ?'     => $target->port,
+                    'hostname = ?' => $target->hostname
+                ]
+            );
         });
     }
 
@@ -309,7 +332,9 @@ protected function processChain($target, $chain)
                     [
                         'ip'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
                         'port'     => $target->port,
-                        'hostname' => $target->hostname
+                        'hostname' => $target->hostname,
+                        'last_scan' => $this->currentDateTime,
+                        'last_seen' => $this->currentDateTime
                     ]
                 );
                 $targetId = $this->db->lastInsertId();
@@ -369,6 +394,12 @@ protected function processChain($target, $chain)
 
                     $certId = CertificateUtils::findOrInsertCert($this->db, $cert, $certInfo);
 
+                    $this->db->update(
+                        'x509_certificate',
+                        ['last_seen' => $this->currentDateTime],
+                        ['id = ?' => $certId]
+                    );
+
                     $this->db->insert(
                         'x509_certificate_chain_link',
                         [
