Updating fork

.github/workflows/lint-and-test.yml

@@ -28,6 +28,10 @@ jobs:
           pip install flake8
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           flake8 . --count --max-line-length=120 --show-source --statistics
+      - name: Check isort
+        run: |
+          pip install isort
+          isort --check --profile=django .
       - name: Set secret key
         run: ./sbin/patchman-set-secret-key
       - name: Test with django

arch/admin.py

@@ -16,7 +16,8 @@
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
 from django.contrib import admin
-from arch.models import PackageArchitecture, MachineArchitecture
+
+from arch.models import MachineArchitecture, PackageArchitecture
 
 admin.site.register(PackageArchitecture)
 admin.site.register(MachineArchitecture)

arch/serializers.py

@@ -16,7 +16,7 @@
 
 from rest_framework import serializers
 
-from arch.models import PackageArchitecture, MachineArchitecture
+from arch.models import MachineArchitecture, PackageArchitecture
 
 
 class PackageArchitectureSerializer(serializers.HyperlinkedModelSerializer):

arch/utils.py

@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
-from arch.models import PackageArchitecture, MachineArchitecture
-from patchman.signals import info_message
+from arch.models import MachineArchitecture, PackageArchitecture
+from util.logging import info_message
 
 
 def clean_package_architectures():
@@ -24,9 +24,9 @@ def clean_package_architectures():
     parches = PackageArchitecture.objects.filter(package__isnull=True)
     plen = parches.count()
     if plen == 0:
-        info_message.send(sender=None, text='No orphaned PackageArchitectures found.')
+        info_message(text='No orphaned PackageArchitectures found.')
     else:
-        info_message.send(sender=None, text=f'Removing {plen} orphaned PackageArchitectures')
+        info_message(text=f'Removing {plen} orphaned PackageArchitectures')
         parches.delete()
 
 
@@ -39,9 +39,9 @@ def clean_machine_architectures():
     )
     mlen = marches.count()
     if mlen == 0:
-        info_message.send(sender=None, text='No orphaned MachineArchitectures found.')
+        info_message(text='No orphaned MachineArchitectures found.')
     else:
-        info_message.send(sender=None, text=f'Removing {mlen} orphaned MachineArchitectures')
+        info_message(text=f'Removing {mlen} orphaned MachineArchitectures')
         marches.delete()
 
 

arch/views.py

@@ -16,9 +16,10 @@
 
 from rest_framework import viewsets
 
-from arch.models import PackageArchitecture, MachineArchitecture
-from arch.serializers import PackageArchitectureSerializer, \
-    MachineArchitectureSerializer
+from arch.models import MachineArchitecture, PackageArchitecture
+from arch.serializers import (
+    MachineArchitectureSerializer, PackageArchitectureSerializer,
+)
 
 
 class PackageArchitectureViewSet(viewsets.ModelViewSet):

debian/control

@@ -20,7 +20,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2),
  python3-requests, python3-colorama, python3-magic, python3-humanize,
  python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3,
  celery, python3-celery, python3-django-celery-beat, redis-server,
- python3-redis, python3-git, python3-django-taggit
+ python3-redis, python3-git, python3-django-taggit, python3-zstandard
 Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached
 Description: Django-based patch status monitoring tool for linux systems.
  .

debian/python3-patchman.install

@@ -1,4 +1,5 @@
 #!/usr/bin/dh-exec
 etc/patchman/apache.conf.example => etc/apache2/conf-available/patchman.conf
 etc/patchman/local_settings.py etc/patchman
-etc/systemd/system/patchman-celery.service => lib/systemd/system/patchman-celery.service
+etc/systemd/system/patchman-celery-worker.service => lib/systemd/system/patchman-celery-worker@.service
+etc/systemd/system/patchman-celery-beat.service => lib/systemd/system/patchman-celery-beat.service

debian/python3-patchman.postinst

@@ -22,10 +22,39 @@ if [ "$1" = "configure" ] ; then
     patchman-manage migrate --run-syncdb --fake-initial
     sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;'
 
-    chown -R www-data:www-data /var/lib/patchman
-    adduser --system --group patchman-celery
-    usermod -a -G www-data patchman-celery
+    adduser --quiet --system --group patchman
+    adduser --quiet www-data patchman
+    chown root:patchman /etc/patchman/celery.conf
+    chmod 640 /etc/patchman/celery.conf
     chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db
+    chown -R patchman:patchman /var/lib/patchman
+
+    WORKER_COUNT=1
+    if [ -f /etc/patchman/celery.conf ]; then
+        . /etc/patchman/celery.conf
+        WORKER_COUNT=${CELERY_WORKER_COUNT:-1}
+    fi
+
+    if [ -d /run/systemd/system ]; then
+        systemctl daemon-reload >/dev/null || true
+        for i in $(seq 1 "${WORKER_COUNT}"); do
+            deb-systemd-helper enable "patchman-celery-worker@$i.service" >/dev/null || true
+            deb-systemd-invoke start "patchman-celery-worker@$i.service" >/dev/null || true
+        done
+
+        active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}')
+
+        for service in $active_instances; do
+            inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1)
+            if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then
+                deb-systemd-invoke stop "$service" >/dev/null || true
+                deb-systemd-helper disable "$service" >/dev/null || true
+            fi
+        done
+
+        deb-systemd-helper enable "patchman-celery-beat.service" >/dev/null || true
+        deb-systemd-invoke start "patchman-celery-beat.service" >/dev/null || true
+    fi
 
     echo
     echo "Remember to run 'patchman-manage createsuperuser' to create a user."

docker/docker-entrypoint.sh

@@ -92,17 +92,16 @@ if "${USE_CACHE}"; then
         redisPort="6379"
     fi
 
-    # Comment DummyCache Block
-    sed -i '47,51 {/^#/ ! s/\(.*\)/#\1/}' "$conf"
-
-    # Uncomment RedisCache Block
-    sed -i '55,61 {s/^# //}' "$conf"
-
-    sed -i "58 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" 
+    # Change RedisCache LOCATION
+    sed -i "62 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf"
 
     if [ -n "${CACHE_TIMEOUT}" ]; then
-        sed -i "59 {s/30/${CACHE_TIMEOUT}/}" "$conf" 
+        sed -i "67 {s/0/${CACHE_TIMEOUT}/}" "$conf"
     fi
+else
+    # Change RedisCache to DummyCache to avoid ConnectionError and comment LOCATION
+    sed -i '61 {s/redis.RedisCache/dummy.DummyCache/}' "$conf"
+    sed -i '62 {/^#/ ! s/\(.*\)/#\1/}' "$conf"
 fi
 
 # Set sendmail destination

domains/admin.py

@@ -16,6 +16,7 @@
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
 from django.contrib import admin
+
 from domains.models import Domain
 
 admin.site.register(Domain)

errata/admin.py

@@ -15,6 +15,7 @@
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
 from django.contrib import admin
+
 from errata.models import Erratum
 
 

errata/models.py

@@ -16,17 +16,16 @@
 
 import json
 
-from django.db import models
+from django.db import IntegrityError, models
 from django.urls import reverse
-from django.db import IntegrityError
 
+from errata.managers import ErratumManager
 from packages.models import Package, PackageUpdate
 from packages.utils import find_evr, get_matching_packages
-from errata.managers import ErratumManager
 from security.models import CVE, Reference
 from security.utils import get_or_create_cve, get_or_create_reference
-from patchman.signals import error_message
 from util import get_url
+from util.logging import error_message
 
 
 class Erratum(models.Model):
@@ -70,7 +69,7 @@ def scan_for_security_updates(self):
                     try:
                         affected_update.save()
                     except IntegrityError as e:
-                        error_message.send(sender=None, text=e)
+                        error_message(text=e)
                         # a version of this update already exists that is
                         # marked as a security update, so delete this one
                         affected_update.delete()
@@ -84,7 +83,7 @@ def scan_for_security_updates(self):
                     try:
                         affected_update.save()
                     except IntegrityError as e:
-                        error_message.send(sender=None, text=e)
+                        error_message(text=e)
                         # a version of this update already exists that is
                         # marked as a security update, so delete this one
                         affected_update.delete()
@@ -93,7 +92,7 @@ def fetch_osv_dev_data(self):
         osv_dev_url = f'https://api.osv.dev/v1/vulns/{self.name}'
         res = get_url(osv_dev_url)
         if res.status_code == 404:
-            error_message.send(sender=None, text=f'404 - Skipping {self.name} - {osv_dev_url}')
+            error_message(text=f'404 - Skipping {self.name} - {osv_dev_url}')
             return
         data = res.content
         osv_dev_json = json.loads(data)
@@ -102,7 +101,7 @@ def fetch_osv_dev_data(self):
     def parse_osv_dev_data(self, osv_dev_json):
         name = osv_dev_json.get('id')
         if name != self.name:
-            error_message.send(sender=None, text=f'Erratum name mismatch - {self.name} != {name}')
+            error_message(text=f'Erratum name mismatch - {self.name} != {name}')
             return
         related = osv_dev_json.get('related')
         if related:
@@ -155,7 +154,7 @@ def add_cve(self, cve_id):
         """ Add a CVE to an Erratum object
         """
         if not cve_id.startswith('CVE') or not cve_id.split('-')[1].isdigit():
-            error_message.send(sender=None, text=f'Not a CVE ID: {cve_id}')
+            error_message(text=f'Not a CVE ID: {cve_id}')
             return
         self.cves.add(get_or_create_cve(cve_id))
 

errata/sources/distros/alma.py

@@ -22,8 +22,8 @@
 from operatingsystems.utils import get_or_create_osrelease
 from packages.models import Package
 from packages.utils import get_or_create_package, parse_package_string
-from util import get_url, fetch_content, get_setting_of_type
 from patchman.signals import pbar_start, pbar_update
+from util import fetch_content, get_setting_of_type, get_url
 
 
 def update_alma_errata(concurrent_processing=True):

errata/sources/distros/arch.py

@@ -20,10 +20,13 @@
 from django.db import connections
 
 from operatingsystems.utils import get_or_create_osrelease
-from patchman.signals import error_message, pbar_start, pbar_update
 from packages.models import Package
-from packages.utils import find_evr, get_matching_packages, get_or_create_package
-from util import get_url, fetch_content
+from packages.utils import (
+    find_evr, get_matching_packages, get_or_create_package,
+)
+from patchman.signals import pbar_start, pbar_update
+from util import fetch_content, get_url
+from util.logging import error_message
 
 
 def update_arch_errata(concurrent_processing=False):
@@ -99,7 +102,7 @@ def process_arch_erratum(advisory, osrelease):
         add_arch_erratum_references(e, advisory)
         add_arch_erratum_packages(e, advisory)
     except Exception as exc:
-        error_message.send(sender=None, text=exc)
+        error_message(text=exc)
 
 
 def add_arch_linux_osrelease():

errata/sources/distros/centos.py

@@ -15,13 +15,15 @@
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
 import re
+
 from defusedxml import ElementTree
 
 from operatingsystems.utils import get_or_create_osrelease
 from packages.models import Package
-from packages.utils import parse_package_string, get_or_create_package
-from patchman.signals import error_message, pbar_start, pbar_update
-from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type
+from packages.utils import get_or_create_package, parse_package_string
+from patchman.signals import pbar_start, pbar_update
+from util import bunzip2, fetch_content, get_setting_of_type, get_sha1, get_url
+from util.logging import error_message
 
 
 def update_centos_errata():
@@ -34,7 +36,7 @@ def update_centos_errata():
     if actual_checksum != expected_checksum:
         e = 'CEFS checksum mismatch, skipping CentOS errata parsing\n'
         e += f'{actual_checksum} (actual) != {expected_checksum} (expected)'
-        error_message.send(sender=None, text=e)
+        error_message(text=e)
     else:
         if data:
             parse_centos_errata(bunzip2(data))

errata/sources/distros/debian.py

@@ -18,17 +18,18 @@
 import csv
 import re
 from datetime import datetime
-from debian.deb822 import Dsc
 from io import StringIO
 
+from debian.deb822 import Dsc
 from django.db import connections
 
 from operatingsystems.models import OSRelease
 from operatingsystems.utils import get_or_create_osrelease
 from packages.models import Package
-from packages.utils import get_or_create_package, find_evr
-from patchman.signals import error_message, pbar_start, pbar_update, warning_message
-from util import get_url, fetch_content, get_setting_of_type, extract
+from packages.utils import find_evr, get_or_create_package
+from patchman.signals import pbar_start, pbar_update
+from util import extract, fetch_content, get_setting_of_type, get_url
+from util.logging import error_message, warning_message
 
 DSCs = {}
 
@@ -217,7 +218,7 @@ def process_debian_erratum(erratum, accepted_codenames):
             for package in packages:
                 process_debian_erratum_fixed_packages(e, package)
     except Exception as exc:
-        error_message.send(sender=None, text=exc)
+        error_message(text=exc)
 
 
 def parse_debian_erratum_package(line, accepted_codenames):
@@ -249,7 +250,7 @@ def fetch_debian_dsc_package_list(package, version):
     """ Fetch the package list from a DSC file for a given source package/version
     """
     if not DSCs.get(package) or not DSCs[package].get(version):
-        warning_message.send(sender=None, text=f'No DSC found for {package} {version}')
+        warning_message(text=f'No DSC found for {package} {version}')
         return
     source_url = DSCs[package][version]['url']
     res = get_url(source_url)
@@ -263,7 +264,7 @@ def get_accepted_debian_codenames():
     """ Get acceptable Debian OS codenames
         Can be overridden by specifying DEBIAN_CODENAMES in settings
     """
-    default_codenames = ['bookworm', 'bullseye']
+    default_codenames = ['bookworm', 'trixie']
     accepted_codenames = get_setting_of_type(
         setting_name='DEBIAN_CODENAMES',
         setting_type=list,