WIP

raviks789 · raviks789 · commit 0b9a937286e1 · 2022-10-20T15:59:00.000+02:00
diff --git a/application/clicommands/ImportCommand.php b/application/clicommands/ImportCommand.php
@@ -4,10 +4,12 @@
 
 namespace Icinga\Module\X509\Clicommands;
 
+use DateTime;
 use Icinga\Application\Logger;
 use Icinga\Module\X509\CertificateUtils;
 use Icinga\Module\X509\Command;
 use ipl\Sql\Connection;
+use ipl\Sql\Expression;
 
 class ImportCommand extends Command
 {
@@ -45,7 +47,7 @@ public function indexAction()
 
                 $db->update(
                     'x509_certificate',
-                    ['trusted' => 'yes'],
+                    ['trusted' => 'yes', 'last_seen' => (new DateTime())->getTimestamp()],
                     ['id = ?' => $id]
                 );
 
diff --git a/application/clicommands/ScanCommand.php b/application/clicommands/ScanCommand.php
@@ -4,11 +4,13 @@
 
 namespace Icinga\Module\X509\Clicommands;
 
+use DateTime;
 use Icinga\Application\Logger;
 use Icinga\Module\X509\CertificateUtils;
 use Icinga\Module\X509\Command;
 use Icinga\Module\X509\Hook\SniHook;
 use Icinga\Module\X509\Job;
+use InvalidArgumentException;
 use ipl\Sql\Connection;
 use ipl\Sql\Expression;
 
@@ -18,16 +20,20 @@ class ScanCommand extends Command
      * Scans IP and port ranges to find X.509 certificates.
      *
      * This command starts scanning the IP and port ranges which belong to the job that was specified with the
-     * --job parameter.
+     * --job parameter and optional --last-scan and --last-seen parameter.
      *
      * USAGE
      *
-     * icingacli x509 scan --job <name>
+     * icingacli x509 scan --job <name> --since-last-scan <days> --since-last-seen <days>
      */
     public function indexAction()
     {
         $name = $this->params->shiftRequired('job');
 
+        $sinceLastSeen = $this->processThresholdToDateTiime($this->params->get('since-last-seen'));
+
+        $sinceLastScan = $this->processThresholdToDateTiime($this->params->get('since-last-scan'));
+
         $parallel = (int) $this->Config()->get('scan', 'parallel', 256);
 
         if ($parallel <= 0) {
@@ -60,19 +66,66 @@ public function indexAction()
                 $name
             );
 
-//            $this->cleanupTargets($this->getDb());
+            if ($sinceLastScan !== null) {
+                $this->cleanupLastScanTargets($this->getDb(), $sinceLastScan);
+            }
 
             $verified = CertificateUtils::verifyCertificates($this->getDb());
 
+            if ($sinceLastSeen !== null) {
+                $this->cleanupLastSeenTargets($this->getDb(), $sinceLastSeen);
+                CertificateUtils::cleanupCertificates($this->getDb(), $sinceLastSeen);
+            }
+
+            CertificateUtils::cleanupCertificateChains($this->getDb());
             Logger::info("Checked %d certificate chain%s.", $verified, $verified !== 1 ? 's' : '');
         }
     }
 
-    protected function cleanupTargets(Connection $db)
+    protected function cleanupLastScanTargets(Connection $db, DateTime $sincLastScan)
+    {
+        $db->delete(
+            'x509_target',
+            ['last_scan < ?' => $sincLastScan->getTimestamp()]
+        );
+    }
+
+    protected function cleanupLastSeenTargets(Connection $db, DateTime $sincLastSeen)
     {
         $db->delete(
             'x509_target',
-            ['last_seen = ?' => new Expression('NOW() - INTERVAL 1 DAY')]
+            [new Expression(sprintf(
+                "last_seen < last_scan - %d",
+                (new DateTime())->getTimestamp() - $sincLastSeen->getTimestamp()
+            ))]
         );
     }
+
+    /**
+     * @param string|null $threshold
+     * @return DateTime|null
+     */
+    protected function processThresholdToDateTiime(?string $threshold)
+    {
+        $since = $threshold;
+        if ($threshold !== null) {
+            if ($since[0] !== '-') {
+                // When the user specified "2 days" as a threshold strtotime() will compute the
+                // timestamp NOW() + 2 days, but it has to be NOW() + (-2 days)
+                $since = "-$since";
+            }
+
+            $since = strtotime($since);
+            if ($since === false) {
+                throw new InvalidArgumentException(sprintf(
+                    "The specified last scan time is in an unknown format: '%s'",
+                    $threshold
+                ));
+            }
+
+            $since = (new DateTime())->setTimestamp($since);
+        }
+
+        return $since;
+    }
 }
diff --git a/etc/schema/mysql.schema.sql b/etc/schema/mysql.schema.sql
@@ -18,6 +18,7 @@ CREATE TABLE x509_certificate (
   fingerprint binary(32) NOT NULL COMMENT 'sha256 hash',
   `serial` blob NOT NULL,
   certificate blob NOT NULL COMMENT 'DER encoded certificate',
+  last_seen bigint(20) NOT NULL,
   ctime timestamp NULL DEFAULT NULL,
   mtime timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (id),
@@ -86,6 +87,8 @@ CREATE TABLE x509_target (
   hostname varchar(255) NULL DEFAULT NULL,
   latest_certificate_chain_id int(10) unsigned NULL DEFAULT NULL,
   ctime timestamp NULL DEFAULT NULL,
+  last_scan bigint(20) NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW())),
+  last_seen bigint(20) NOT NULL,
   mtime timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (id),
   INDEX x509_idx_target_ip_port (ip, port)
diff --git a/etc/schema/postgresql.schema.sql b/etc/schema/postgresql.schema.sql
@@ -35,6 +35,7 @@ CREATE TABLE x509_certificate (
   valid_to bigint NOT NULL,
   fingerprint bytea NOT NULL,
   serial bytea NOT NULL,
+  last_seen bigint NOT NULL,
   certificate bytea NOT NULL,
   ctime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
   mtime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -98,6 +99,8 @@ CREATE TABLE x509_target (
   port uint2 NOT NULL,
   hostname varchar(255) NULL DEFAULT NULL,
   latest_certificate_chain_id int NULL DEFAULT NULL,
+  last_seen bigint NOT NULL,
+  last_scan bigint NOT NULL,
   ctime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
   mtime timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
diff --git a/library/X509/CertificateUtils.php b/library/X509/CertificateUtils.php
@@ -4,6 +4,7 @@
 
 namespace Icinga\Module\X509;
 
+use DateTime;
 use Exception;
 use Icinga\Application\Logger;
 use Icinga\File\Storage\TemporaryLocalFileStorage;
@@ -246,7 +247,7 @@ public static function findOrInsertCert(Connection $db, $cert)
                 'fingerprint'         => $dbTool->marshalBinary($fingerprint),
                 'serial'              => $dbTool->marshalBinary(gmp_export($certInfo['serialNumber'])),
                 'certificate'         => $dbTool->marshalBinary($der),
-                'last_seen'           => new Expression('NOW()')
+                'last_seen'           => (new DateTime())->getTimestamp()
             ]
         );
 
@@ -355,24 +356,25 @@ private static function findOrInsertDn($db, $certInfo, $type)
         return $hash;
     }
 
-    public static function cleanupCertificates(Connection $db)
+    public static function cleanupCertificates(Connection $db, DateTime $lastSeen)
     {
         $db->delete(
             'x509_certificate',
-            ['last_seen = ?' => new Expression('NOW() - INTERVAL 7 DAY')]
+            [
+                'last_seen < ?' => $lastSeen->getTimestamp(),
+                'trusted = ?' => 'no'
+            ]
         );
+    }
 
+    public static function cleanupCertificateChains(Connection $db)
+    {
         $db->delete(
-            'x509_dn',
-            ['last_seen' => new Expression('NOW() - INTERVAL 7 DAY')]
+            'x509_certificate_chain',
+            ['target_id NOT IN (?)' => (new Select())
+                ->from('x509_target')
+                ->columns(['id'])]
         );
-
-//        DELETE
-//FROM x509_dn
-//WHERE type='subject' AND HASH <> ALL (
-//        SELECT DISTINCT subject_hash
-//FROM x509_certificate
-//);
     }
 
     /**
@@ -438,9 +440,10 @@ public static function verifyCertificates(Connection $db)
                 foreach ($certs as $cert) {
                     $db->update(
                         'x509_certificate',
-                        ['last_seen' => new Expression('NOW()')],
-                        ['id = ?' => $chain->id]
+                        ['last_seen' => (new DateTime())->getTimestamp()],
+                        ['id = ?' => $cert->id]
                     );
+
                     $collection[] = CertificateUtils::der2pem(DbTool::unmarshalBinary($cert->certificate));
                 }
 
diff --git a/library/X509/Job.php b/library/X509/Job.php
@@ -4,6 +4,7 @@
 
 namespace Icinga\Module\X509;
 
+use DateTime;
 use Icinga\Application\Config;
 use Icinga\Application\Logger;
 use Icinga\Data\ConfigObject;
@@ -41,6 +42,7 @@ class Job
     private $snimap;
     private $parallel;
     private $name;
+    private $currentDateTime;
 
     public function __construct($name, ConfigObject $jobDescription, array $snimap, $parallel)
     {
@@ -50,6 +52,7 @@ public function __construct($name, ConfigObject $jobDescription, array $snimap,
         $this->snimap = $snimap;
         $this->parallel = $parallel;
         $this->name = $name;
+        $this->currentDateTime = (new DateTime())->getTimestamp();
     }
 
     private function getConnector($peerName)
@@ -85,7 +88,7 @@ private static function numberToAddr($num, $ipv6 = true)
         }
     }
 
-    private function generateTargets(ConfigObject $jobDescription, array $hostnamesConfig)
+    private static function generateTargets(ConfigObject $jobDescription, array $hostnamesConfig)
     {
         foreach (StringHelper::trimSplit($jobDescription->get('cidrs')) as $cidr) {
             $pieces = explode('/', $cidr);
@@ -124,17 +127,6 @@ private function generateTargets(ConfigObject $jobDescription, array $hostnamesC
                     foreach (range($start_port, $end_port) as $port) {
                         $hostnames = isset($hostnamesConfig[$ip]) ? $hostnamesConfig[$ip] : [];
 
-                        if (! empty($hostnames)) {
-                            $this->db->delete(
-                                'x509_target',
-                                [
-                                    'ip = ?'       => $this->dbTool->marshalBinary(static::binary($ip)),
-                                    'port = ?'     => $port,
-                                    'hostname NOT IN (?)' => array_values($hostnames)
-                                ]
-                            );
-                        }
-
                         if (empty($hostnames)) {
                             $hostnames[] = null;
                         }
@@ -144,8 +136,6 @@ private function generateTargets(ConfigObject $jobDescription, array $hostnamesC
                             $target->ip = $ip;
                             $target->port = $port;
                             $target->hostname = $hostname;
-
-                            $this->updateLastScan($target);
                             yield $target;
                         }
                     }
@@ -216,6 +206,16 @@ function (ConnectionInterface $conn) use ($target, $streamCapture) {
                 // Close connection in order to capture stream context options
                 $conn->close();
 
+                $this->db->update(
+                    'x509_target',
+                    ['last_seen' => $this->currentDateTime],
+                    [
+                        'ip = ?'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
+                        'port = ?'     => $target->port,
+                        'hostname = ?' => $target->hostname
+                    ]
+                );
+
                 $capturedStreamOptions = $streamCapture->getCapturedStreamOptions();
 
                 $this->processChain($target, $capturedStreamOptions['ssl']['peer_certificate_chain']);
@@ -253,25 +253,19 @@ function (\Exception $exception) use ($target, $streamCapture) {
         )->otherwise(function (\Exception $e) {
             echo $e->getMessage() . PHP_EOL;
             echo $e->getTraceAsString() . PHP_EOL;
+        })->always(function () use ($target) {
+            $this->db->update(
+                'x509_target',
+                ['last_scan' => $this->currentDateTime],
+                [
+                    'ip = ?'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
+                    'port = ?'     => $target->port,
+                    'hostname = ?' => $target->hostname
+                ]
+            );
         });
     }
 
-    protected function updateLastScan($target)
-    {
-        if (isset($target->id)) {
-            $filter = ['id = ?' => $target->id];
-        } else {
-            $filter = [
-                'ip = ?'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
-                'port = ?'     => $target->port,
-                'hostname = ?' => $target->hostname
-            ];
-        }
-
-        $now = new Expression('NOW()');
-        $this->db->update('x509_target', ['last_scan' => $now], $filter);
-    }
-
     public function getJobId()
     {
         return $this->jobId;
@@ -281,13 +275,13 @@ public function run()
     {
         $this->loop = Factory::create();
 
-        $this->totalTargets = iterator_count($this->generateTargets($this->jobDescription, $this->snimap));
+        $this->totalTargets = iterator_count(static::generateTargets($this->jobDescription, $this->snimap));
 
         if ($this->totalTargets == 0) {
             return null;
         }
 
-        $this->targets = $this->generateTargets($this->jobDescription, $this->snimap);
+        $this->targets = static::generateTargets($this->jobDescription, $this->snimap);
 
         $this->db->insert(
             'x509_job_run',
@@ -339,7 +333,8 @@ protected function processChain($target, $chain)
                         'ip'       => $this->dbTool->marshalBinary(static::binary($target->ip)),
                         'port'     => $target->port,
                         'hostname' => $target->hostname,
-                        'last_scan' => new Expression('NOW()')
+                        'last_scan' => $this->currentDateTime,
+                        'last_seen' => $this->currentDateTime
                     ]
                 );
                 $targetId = $this->db->lastInsertId();
@@ -401,7 +396,7 @@ protected function processChain($target, $chain)
 
                     $this->db->update(
                         'x509_certificate',
-                        ['last_seen' => new Expression('NOW()')],
+                        ['last_seen' => $this->currentDateTime],
                         ['id = ?' => $certId]
                     );
 
@@ -419,9 +414,7 @@ protected function processChain($target, $chain)
             $this->db->update(
                 'x509_target',
                 [
-                    'latest_certificate_chain_id' => $chainId,
-                    'last_seen' => new Expression('NOW()')
-                ],
+                    'latest_certificate_chain_id' => $chainId                ],
                 ['id = ?' => $targetId]
             );
         });
