diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 20b221aa..436698be 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -28,6 +28,10 @@ jobs: pip install flake8 flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --max-line-length=120 --show-source --statistics + - name: Check isort + run: | + pip install isort + isort --check --profile=django . - name: Set secret key run: ./sbin/patchman-set-secret-key - name: Test with django diff --git a/arch/admin.py b/arch/admin.py index 624a3720..5224711c 100644 --- a/arch/admin.py +++ b/arch/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from arch.models import PackageArchitecture, MachineArchitecture + +from arch.models import MachineArchitecture, PackageArchitecture admin.site.register(PackageArchitecture) admin.site.register(MachineArchitecture) diff --git a/arch/serializers.py b/arch/serializers.py index 5319e796..a5765128 100644 --- a/arch/serializers.py +++ b/arch/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture class PackageArchitectureSerializer(serializers.HyperlinkedModelSerializer): diff --git a/arch/utils.py b/arch/utils.py index 1498fdec..3db6ac70 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -14,8 +14,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from arch.models import PackageArchitecture, MachineArchitecture -from patchman.signals import info_message +from arch.models import MachineArchitecture, PackageArchitecture +from util.logging import info_message def clean_package_architectures(): @@ -24,9 +24,9 @@ def clean_package_architectures(): parches = PackageArchitecture.objects.filter(package__isnull=True) plen = parches.count() if plen == 0: - info_message.send(sender=None, text='No orphaned PackageArchitectures found.') + info_message(text='No orphaned PackageArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned PackageArchitectures') + info_message(text=f'Removing {plen} orphaned PackageArchitectures') parches.delete() @@ -39,9 +39,9 @@ def clean_machine_architectures(): ) mlen = marches.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned MachineArchitectures found.') + info_message(text='No orphaned MachineArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {mlen} orphaned MachineArchitectures') + info_message(text=f'Removing {mlen} orphaned MachineArchitectures') marches.delete() diff --git a/arch/views.py b/arch/views.py index 56a2a150..21f6b7c7 100644 --- a/arch/views.py +++ b/arch/views.py @@ -16,9 +16,10 @@ from rest_framework import viewsets -from arch.models import PackageArchitecture, MachineArchitecture -from arch.serializers import PackageArchitectureSerializer, \ - MachineArchitectureSerializer +from arch.models import MachineArchitecture, PackageArchitecture +from arch.serializers import ( + MachineArchitectureSerializer, PackageArchitectureSerializer, +) class PackageArchitectureViewSet(viewsets.ModelViewSet): diff --git a/debian/control b/debian/control index 67026269..7bfe320e 100644 --- a/debian/control +++ b/debian/control @@ -20,7 +20,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, - python3-redis, python3-git, python3-django-taggit + python3-redis, python3-git, python3-django-taggit, python3-zstandard Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. . diff --git a/debian/python3-patchman.cron.daily b/debian/python3-patchman.cron.daily deleted file mode 100644 index d4752f75..00000000 --- a/debian/python3-patchman.cron.daily +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -/usr/bin/patchman -a -q diff --git a/debian/python3-patchman.install b/debian/python3-patchman.install index e13b11ca..71f47b3a 100755 --- a/debian/python3-patchman.install +++ b/debian/python3-patchman.install @@ -1,4 +1,5 @@ #!/usr/bin/dh-exec etc/patchman/apache.conf.example => etc/apache2/conf-available/patchman.conf etc/patchman/local_settings.py etc/patchman -etc/systemd/system/patchman-celery.service => lib/systemd/system/patchman-celery.service +etc/systemd/system/patchman-celery-worker.service => lib/systemd/system/patchman-celery-worker@.service +etc/systemd/system/patchman-celery-beat.service => lib/systemd/system/patchman-celery-beat.service diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index b64cb816..b36a5422 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -22,10 +22,39 @@ if [ "$1" = "configure" ] ; then patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' - chown -R www-data:www-data /var/lib/patchman - adduser --system --group patchman-celery - usermod -a -G www-data patchman-celery + adduser --quiet --system --group patchman + adduser --quiet www-data patchman + chown root:patchman /etc/patchman/celery.conf + chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db + chown -R patchman:patchman /var/lib/patchman + + WORKER_COUNT=1 + if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} + fi + + if [ -d /run/systemd/system ]; then + systemctl daemon-reload >/dev/null || true + for i in $(seq 1 "${WORKER_COUNT}"); do + deb-systemd-helper enable "patchman-celery-worker@$i.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-worker@$i.service" >/dev/null || true + done + + active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') + + for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + deb-systemd-invoke stop "$service" >/dev/null || true + deb-systemd-helper disable "$service" >/dev/null || true + fi + done + + deb-systemd-helper enable "patchman-celery-beat.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-beat.service" >/dev/null || true + fi echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index aa9b6618..257c459b 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -92,17 +92,16 @@ if "${USE_CACHE}"; then redisPort="6379" fi - # Comment DummyCache Block - sed -i '47,51 {/^#/ ! s/\(.*\)/#\1/}' "$conf" - - # Uncomment RedisCache Block - sed -i '55,61 {s/^# //}' "$conf" - - sed -i "58 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" + # Change RedisCache LOCATION + sed -i "62 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" if [ -n "${CACHE_TIMEOUT}" ]; then - sed -i "59 {s/30/${CACHE_TIMEOUT}/}" "$conf" + sed -i "67 {s/0/${CACHE_TIMEOUT}/}" "$conf" fi +else + # Change RedisCache to DummyCache to avoid ConnectionError and comment LOCATION + sed -i '61 {s/redis.RedisCache/dummy.DummyCache/}' "$conf" + sed -i '62 {/^#/ ! s/\(.*\)/#\1/}' "$conf" fi # Set sendmail destination diff --git a/domains/admin.py b/domains/admin.py index 2ef883e3..5cb0fee3 100644 --- a/domains/admin.py +++ b/domains/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from domains.models import Domain admin.site.register(Domain) diff --git a/errata/admin.py b/errata/admin.py index 88190ff6..ac4b8a50 100644 --- a/errata/admin.py +++ b/errata/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from errata.models import Erratum diff --git a/errata/models.py b/errata/models.py index b10daf4d..8c21bcfa 100644 --- a/errata/models.py +++ b/errata/models.py @@ -16,17 +16,16 @@ import json -from django.db import models +from django.db import IntegrityError, models from django.urls import reverse -from django.db import IntegrityError +from errata.managers import ErratumManager from packages.models import Package, PackageUpdate from packages.utils import find_evr, get_matching_packages -from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from patchman.signals import error_message from util import get_url +from util.logging import error_message class Erratum(models.Model): @@ -70,7 +69,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -84,7 +83,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -93,7 +92,7 @@ def fetch_osv_dev_data(self): osv_dev_url = f'https://api.osv.dev/v1/vulns/{self.name}' res = get_url(osv_dev_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.name} - {osv_dev_url}') + error_message(text=f'404 - Skipping {self.name} - {osv_dev_url}') return data = res.content osv_dev_json = json.loads(data) @@ -102,7 +101,7 @@ def fetch_osv_dev_data(self): def parse_osv_dev_data(self, osv_dev_json): name = osv_dev_json.get('id') if name != self.name: - error_message.send(sender=None, text=f'Erratum name mismatch - {self.name} != {name}') + error_message(text=f'Erratum name mismatch - {self.name} != {name}') return related = osv_dev_json.get('related') if related: @@ -155,7 +154,7 @@ def add_cve(self, cve_id): """ Add a CVE to an Erratum object """ if not cve_id.startswith('CVE') or not cve_id.split('-')[1].isdigit(): - error_message.send(sender=None, text=f'Not a CVE ID: {cve_id}') + error_message(text=f'Not a CVE ID: {cve_id}') return self.cves.add(get_or_create_cve(cve_id)) diff --git a/errata/sources/distros/alma.py b/errata/sources/distros/alma.py index e0f2d4ae..0091b8bf 100644 --- a/errata/sources/distros/alma.py +++ b/errata/sources/distros/alma.py @@ -22,8 +22,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, parse_package_string -from util import get_url, fetch_content, get_setting_of_type from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_setting_of_type, get_url def update_alma_errata(concurrent_processing=True): diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 40d0dada..e22de403 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,10 +20,13 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from patchman.signals import error_message, pbar_start, pbar_update from packages.models import Package -from packages.utils import find_evr, get_matching_packages, get_or_create_package -from util import get_url, fetch_content +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, +) +from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_url +from util.logging import error_message def update_arch_errata(concurrent_processing=False): @@ -99,7 +102,7 @@ def process_arch_erratum(advisory, osrelease): add_arch_erratum_references(e, advisory) add_arch_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_arch_linux_osrelease(): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index eefb2b88..8f4aa4a1 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -15,13 +15,15 @@ # along with Patchman. If not, see import re + from defusedxml import ElementTree from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package -from patchman.signals import error_message, pbar_start, pbar_update -from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type +from packages.utils import get_or_create_package, parse_package_string +from patchman.signals import pbar_start, pbar_update +from util import bunzip2, fetch_content, get_setting_of_type, get_sha1, get_url +from util.logging import error_message def update_centos_errata(): @@ -34,7 +36,7 @@ def update_centos_errata(): if actual_checksum != expected_checksum: e = 'CEFS checksum mismatch, skipping CentOS errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) else: if data: parse_centos_errata(bunzip2(data)) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 93ae2bd5..8025b1bf 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -18,17 +18,18 @@ import csv import re from datetime import datetime -from debian.deb822 import Dsc from io import StringIO +from debian.deb822 import Dsc from django.db import connections from operatingsystems.models import OSRelease from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, find_evr -from patchman.signals import error_message, pbar_start, pbar_update, warning_message -from util import get_url, fetch_content, get_setting_of_type, extract +from packages.utils import find_evr, get_or_create_package +from patchman.signals import pbar_start, pbar_update +from util import extract, fetch_content, get_setting_of_type, get_url +from util.logging import error_message, warning_message DSCs = {} @@ -217,7 +218,7 @@ def process_debian_erratum(erratum, accepted_codenames): for package in packages: process_debian_erratum_fixed_packages(e, package) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def parse_debian_erratum_package(line, accepted_codenames): @@ -249,7 +250,7 @@ def fetch_debian_dsc_package_list(package, version): """ Fetch the package list from a DSC file for a given source package/version """ if not DSCs.get(package) or not DSCs[package].get(version): - warning_message.send(sender=None, text=f'No DSC found for {package} {version}') + warning_message(text=f'No DSC found for {package} {version}') return source_url = DSCs[package][version]['url'] res = get_url(source_url) @@ -263,7 +264,7 @@ def get_accepted_debian_codenames(): """ Get acceptable Debian OS codenames Can be overridden by specifying DEBIAN_CODENAMES in settings """ - default_codenames = ['bookworm', 'bullseye'] + default_codenames = ['bookworm', 'trixie'] accepted_codenames = get_setting_of_type( setting_name='DEBIAN_CODENAMES', setting_type=list, diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 693d7b0c..2d805985 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -14,18 +14,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import json import concurrent.futures -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential +import json from django.db import connections from django.db.utils import OperationalError +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, info_message, error_message +from util import fetch_content, get_url +from util.logging import error_message, info_message def update_rocky_errata(concurrent_processing=True): @@ -50,16 +53,16 @@ def check_rocky_errata_endpoint_health(rocky_errata_api_host): health = json.loads(data) if health.get('status') == 'ok': s = f'Rocky Errata API healthcheck OK: {rocky_errata_healthcheck_url}' - info_message.send(sender=None, text=s) + info_message(text=s) return True else: s = f'Rocky Errata API healthcheck FAILED: {rocky_errata_healthcheck_url}' - error_message.send(sender=None, text=s) + error_message(text=s) return False except Exception as e: s = f'Rocky Errata API healthcheck exception occured: {rocky_errata_healthcheck_url}\n' s += str(e) - error_message.send(sender=None, text=s) + error_message(text=s) return False @@ -194,7 +197,7 @@ def process_rocky_erratum(advisory): add_rocky_erratum_oses(e, advisory) add_rocky_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_rocky_erratum_references(e, advisory): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 7f50962c..5616331f 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -16,8 +16,8 @@ import concurrent.futures import csv -import os import json +import os from io import StringIO from urllib.parse import urlparse @@ -26,9 +26,15 @@ from operatingsystems.models import OSRelease, OSVariant from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages -from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from patchman.signals import error_message, pbar_start, pbar_update +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, + parse_package_string, +) +from patchman.signals import pbar_start, pbar_update +from util import ( + bunzip2, fetch_content, get_setting_of_type, get_sha256, get_url, +) +from util.logging import error_message def update_ubuntu_errata(concurrent_processing=False): @@ -45,7 +51,7 @@ def update_ubuntu_errata(concurrent_processing=False): else: e = 'Ubuntu USN DB checksum mismatch, skipping Ubuntu errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) def fetch_ubuntu_usn_db(): @@ -126,7 +132,7 @@ def process_usn(usn_id, advisory, accepted_releases): add_ubuntu_erratum_references(e, usn_id, advisory) add_ubuntu_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_ubuntu_erratum_osreleases(e, affected_releases, accepted_releases): @@ -202,7 +208,7 @@ def get_accepted_ubuntu_codenames(): """ Get acceptable Ubuntu OS codenames Can be overridden by specifying UBUNTU_CODENAMES in settings """ - default_codenames = ['focal', 'jammy', 'noble'] + default_codenames = ['jammy', 'noble'] accepted_codenames = get_setting_of_type( setting_name='UBUNTU_CODENAMES', setting_type=list, diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index dfeed879..8b6732c4 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -16,16 +16,17 @@ import concurrent.futures from io import BytesIO -from defusedxml import ElementTree +from defusedxml import ElementTree from django.db import connections from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from patchman.signals import pbar_start, pbar_update, error_message +from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url +from util.logging import error_message def extract_updateinfo(data, url, concurrent_processing=True): @@ -38,7 +39,7 @@ def extract_updateinfo(data, url, concurrent_processing=True): elen = root.__len__() updates = root.findall('update') except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing updateinfo file from {url} : {e}') + error_message(text=f'Error parsing updateinfo file from {url} : {e}') if concurrent_processing: extract_updateinfo_concurrently(updates, elen) else: diff --git a/errata/tasks.py b/errata/tasks.py index fe53b415..9d1f3ed9 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -15,20 +15,21 @@ # along with Patchman. If not, see from celery import shared_task +from django.core.cache import cache -from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata -from errata.sources.distros.debian import update_debian_errata +from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.centos import update_centos_errata +from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from patchman.signals import error_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type +from util.logging import error_message, warning_message -@shared_task +@shared_task(priority=1) def update_yum_repo_errata(repo_id=None, force=False): """ Update all yum repos errata """ @@ -40,41 +41,51 @@ def update_yum_repo_errata(repo_id=None, force=False): repo.refresh_errata(force) -@shared_task +@shared_task(priority=1) def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ - errata_os_updates = [] - erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] - erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] - if erratum_type: - if erratum_type not in erratum_types: - error_message.send(sender=None, text=f'Erratum type `{erratum_type}` not in {erratum_types}') - else: - errata_os_updates = erratum_type + lock_key = 'update_errata_lock' + # lock will expire after 48 hours + lock_expire = 60 * 60 * 48 + + if cache.add(lock_key, 'true', lock_expire): + try: + errata_os_updates = [] + erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] + erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + if erratum_type: + if erratum_type not in erratum_types: + error_message(text=f'Erratum type `{erratum_type}` not in {erratum_types}') + else: + errata_os_updates = erratum_type + else: + errata_os_updates = get_setting_of_type( + setting_name='ERRATA_OS_UPDATES', + setting_type=list, + default=erratum_type_defaults, + ) + if 'yum' in errata_os_updates: + update_yum_repo_errata(repo_id=repo, force=force) + if 'arch' in errata_os_updates: + update_arch_errata() + if 'alma' in errata_os_updates: + update_alma_errata() + if 'rocky' in errata_os_updates: + update_rocky_errata() + if 'debian' in errata_os_updates: + update_debian_errata() + if 'ubuntu' in errata_os_updates: + update_ubuntu_errata() + if 'centos' in errata_os_updates: + update_centos_errata() + finally: + cache.delete(lock_key) else: - errata_os_updates = get_setting_of_type( - setting_name='ERRATA_OS_UPDATES', - setting_type=list, - default=erratum_type_defaults, - ) - if 'yum' in errata_os_updates: - update_yum_repo_errata(repo_id=repo, force=force) - if 'arch' in errata_os_updates: - update_arch_errata() - if 'alma' in errata_os_updates: - update_alma_errata() - if 'rocky' in errata_os_updates: - update_rocky_errata() - if 'debian' in errata_os_updates: - update_debian_errata() - if 'ubuntu' in errata_os_updates: - update_ubuntu_errata() - if 'centos' in errata_os_updates: - update_centos_errata() + warning_message('Already updating Errata, skipping task.') -@shared_task +@shared_task(priority=2) def update_errata_and_cves(): """ Task to update all errata """ diff --git a/errata/utils.py b/errata/utils.py index d8099db4..e0a5e01b 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -18,10 +18,11 @@ from django.db import connections -from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from patchman.signals import pbar_start, pbar_update, warning_message +from patchman.signals import pbar_start, pbar_update +from util import tz_aware_datetime +from util.logging import warning_message def get_or_create_erratum(name, e_type, issue_date, synopsis): @@ -36,16 +37,16 @@ def get_or_create_erratum(name, e_type, issue_date, synopsis): days_delta = abs(e.issue_date.date() - issue_date_tz.date()).days updated = False if e.e_type != e_type: - warning_message.send(sender=None, text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') + warning_message(text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') e.e_type = e_type updated = True if days_delta > 1: text = f'Updating {name} issue date `{e.issue_date.date()}` -> `{issue_date_tz.date()}`' - warning_message.send(sender=None, text=text) + warning_message(text=text) e.issue_date = issue_date_tz updated = True if e.synopsis != synopsis: - warning_message.send(sender=None, text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') + warning_message(text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') e.synopsis = synopsis updated = True if updated: diff --git a/errata/views.py b/errata/views.py index 42d12f71..8e1c0b2f 100644 --- a/errata/views.py +++ b/errata/views.py @@ -14,16 +14,15 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from operatingsystems.models import OSRelease from errata.models import Erratum from errata.serializers import ErratumSerializer +from operatingsystems.models import OSRelease from util.filterspecs import Filter, FilterBar diff --git a/etc/patchman/apache.conf.example b/etc/patchman/apache.conf.example index 5434cf97..c055eba0 100644 --- a/etc/patchman/apache.conf.example +++ b/etc/patchman/apache.conf.example @@ -1,5 +1,5 @@ Define patchman_pythonpath /srv/patchman/ -WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py +WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py application-group=%{GLOBAL} WSGIPythonPath ${patchman_pythonpath} diff --git a/etc/patchman/celery.conf b/etc/patchman/celery.conf index 7afc96ee..2e6f9855 100644 --- a/etc/patchman/celery.conf +++ b/etc/patchman/celery.conf @@ -1,2 +1,5 @@ REDIS_HOST=127.0.0.1 REDIS_PORT=6379 +CELERY_POOL_TYPE=solo +CELERY_WORKER_COUNT=1 +CELERY_CONCURRENCY=1 diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d..cb19e268 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -41,27 +41,35 @@ # Number of days to wait before raising that a host has not reported DAYS_WITHOUT_REPORT = 14 +# list of errata sources to update, remove unwanted ones to improve performance +ERRATA_OS_UPDATES = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + +# list of Alma Linux releases to update +ALMA_RELEASES = [8, 9, 10] + +# list of Debian Linux releases to update +DEBIAN_CODENAMES = ['bookworm', 'trixie'] + +# list of Ubuntu Linux releases to update +UBUNTU_CODENAMES = ['jammy', 'noble'] + # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379', } } - -# Uncomment to enable redis caching for e.g. 30 seconds +# Set the default timeout to e.g. 30 seconds to enable UI caching # Note that the UI results may be out of date for this amount of time -# CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.redis.RedisCache', -# 'LOCATION': 'redis://127.0.0.1:6379', -# 'TIMEOUT': 30, -# } -# } - -from datetime import timedelta # noqa +CACHE_MIDDLEWARE_SECONDS = 0 + +from datetime import timedelta # noqa + from celery.schedules import crontab # noqa + CELERY_BEAT_SCHEDULE = { 'process_all_unprocessed_reports': { 'task': 'reports.tasks.process_reports', @@ -88,3 +96,20 @@ 'schedule': timedelta(hours=24), }, } + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'root': { + 'handlers': ['console'], + }, + 'loggers': { + 'urllib3': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + 'git': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + } +} diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service new file mode 100644 index 00000000..c9bce722 --- /dev/null +++ b/etc/systemd/system/patchman-celery-beat.service @@ -0,0 +1,21 @@ +[Unit] +Description=Patchman Celery Beat Scheduler Service +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman +Group=patchman +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ + --app patchman \ + beat \ + --loglevel info \ + --scheduler django_celery_beat.schedulers:DatabaseScheduler + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service new file mode 100644 index 00000000..b2d6f6b7 --- /dev/null +++ b/etc/systemd/system/patchman-celery-worker.service @@ -0,0 +1,25 @@ +[Unit] +Description=Patchman Celery Worker Service %i +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman +Group=patchman +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" +Environment="CELERY_POOL_TYPE=solo" +Environment="CELERY_CONCURRENCY=1" +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ + --app patchman \ + worker \ + --task-events \ + --pool ${CELERY_POOL_TYPE} \ + --concurrency ${CELERY_CONCURRENCY} \ + --hostname patchman-celery-worker%i@%%h + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery.service b/etc/systemd/system/patchman-celery.service deleted file mode 100644 index 6408d818..00000000 --- a/etc/systemd/system/patchman-celery.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Patchman Celery Service -Requires=network-online.target -After=network-onlne.target - -[Service] -Type=simple -User=patchman-celery -Group=patchman-celery -EnvironmentFile=/etc/patchman/celery.conf -ExecStart=/usr/bin/celery --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 --app patchman worker --loglevel info --beat --scheduler django_celery_beat.schedulers:DatabaseScheduler --task-events --pool threads - -[Install] -WantedBy=multi-user.target diff --git a/hooks/yum/patchman.py b/hooks/yum/patchman.py index 343144eb..52f9cc8b 100644 --- a/hooks/yum/patchman.py +++ b/hooks/yum/patchman.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see import os + from yum.plugins import TYPE_CORE requires_api_version = '2.1' diff --git a/hooks/zypper/patchman.py b/hooks/zypper/patchman.py index 14781565..d9d478f3 100755 --- a/hooks/zypper/patchman.py +++ b/hooks/zypper/patchman.py @@ -18,8 +18,9 @@ # # zypp system plugin for patchman -import os import logging +import os + from zypp_plugin import Plugin diff --git a/hosts/admin.py b/hosts/admin.py index 8a42e8cc..43bf31da 100644 --- a/hosts/admin.py +++ b/hosts/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from hosts.models import Host, HostRepo diff --git a/hosts/migrations/0001_initial.py b/hosts/migrations/0001_initial.py index 43366684..0037e094 100644 --- a/hosts/migrations/0001_initial.py +++ b/hosts/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.db import migrations, models + try: import tagging.fields has_tagging = True diff --git a/hosts/migrations/0002_initial.py b/hosts/migrations/0002_initial.py index cc59a70e..6c453c49 100644 --- a/hosts/migrations/0002_initial.py +++ b/hosts/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/hosts/migrations/0004_remove_host_tags_host_tags.py b/hosts/migrations/0004_remove_host_tags_host_tags.py index 84e7affe..bf03a84e 100644 --- a/hosts/migrations/0004_remove_host_tags_host_tags.py +++ b/hosts/migrations/0004_remove_host_tags_host_tags.py @@ -1,8 +1,9 @@ # Generated by Django 4.2.18 on 2025-02-04 23:37 +import taggit.managers from django.apps import apps from django.db import migrations -import taggit.managers + try: import tagging # noqa except ImportError: diff --git a/hosts/migrations/0006_migrate_to_tz_aware.py b/hosts/migrations/0006_migrate_to_tz_aware.py index e36bbf1f..c14ea50b 100644 --- a/hosts/migrations/0006_migrate_to_tz_aware.py +++ b/hosts/migrations/0006_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Host = apps.get_model('hosts', 'Host') for host in Host.objects.all(): diff --git a/hosts/migrations/0007_alter_host_tags.py b/hosts/migrations/0007_alter_host_tags.py index 3858b847..3910a06f 100644 --- a/hosts/migrations/0007_alter_host_tags.py +++ b/hosts/migrations/0007_alter_host_tags.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.19 on 2025-02-28 19:53 -from django.db import migrations import taggit.managers +from django.db import migrations class Migration(migrations.Migration): diff --git a/hosts/models.py b/hosts/models.py index 5b7b3979..4689ccc5 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -24,6 +24,7 @@ from version_utils.rpm import labelCompare except ImportError: from rpm import labelCompare + from taggit.managers import TaggableManager from arch.models import MachineArchitecture @@ -34,9 +35,9 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from patchman.signals import info_message from repos.models import Repository from repos.utils import find_best_repo +from util.logging import info_message class Host(models.Model): @@ -85,12 +86,12 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {" ".join(self.tags.slugs())}\n' + text += f'Tags : {" ".join(self.tags.names())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def get_absolute_url(self): return reverse('hosts:host_detail', args=[self.hostname]) @@ -114,13 +115,13 @@ def check_rdns(self): if self.check_dns: update_rdns(self) if self.hostname.lower() == self.reversedns.lower(): - info_message.send(sender=None, text='Reverse DNS matches') + info_message(text='Reverse DNS matches') else: text = 'Reverse DNS mismatch found: ' text += f'{self.hostname} != {self.reversedns}' - info_message.send(sender=None, text=text) + info_message(text=text) else: - info_message.send(sender=None, text='Reverse DNS check disabled') + info_message(text='Reverse DNS check disabled') def clean_reports(self): """ Remove all but the last 3 reports for a host @@ -131,7 +132,7 @@ def clean_reports(self): for report in Report.objects.filter(host=self).order_by('-created')[3:]: report.delete() if rlen > 0: - info_message.send(sender=None, text=f'{self.hostname}: removed {rlen} old reports') + info_message(text=f'{self.hostname}: removed {rlen} old reports') def get_host_repo_packages(self): if self.host_repos_only: @@ -163,7 +164,7 @@ def process_update(self, package, highest_package): security = True update = get_or_create_package_update(oldpackage=package, newpackage=highest_package, security=security) self.updates.add(update) - info_message.send(sender=None, text=f'{update}') + info_message(text=f'{update}') return update.id def find_updates(self): diff --git a/hosts/tasks.py b/hosts/tasks.py index 2fdce96f..f186760f 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -15,15 +15,14 @@ # along with Patchman. If not, see from celery import shared_task - from django.db.models import Count from hosts.models import Host from util import get_datetime_now -from patchman.signals import info_message +from util.logging import info_message -@shared_task +@shared_task(priority=0) def find_host_updates(host_id): """ Task to find updates for a host """ @@ -31,7 +30,7 @@ def find_host_updates(host_id): host.find_updates() -@shared_task +@shared_task(priority=1) def find_all_host_updates(): """ Task to find updates for all hosts """ @@ -39,7 +38,7 @@ def find_all_host_updates(): find_host_updates.delay(host.id) -@shared_task +@shared_task(priority=1) def find_all_host_updates_homogenous(): """ Task to find updates for all hosts where hosts are expected to be homogenous """ @@ -78,4 +77,4 @@ def find_all_host_updates_homogenous(): phost.updated_at = ts phost.save() updated_hosts.append(phost) - info_message.send(sender=None, text=f'Added the same updates to {phost}') + info_message(text=f'Added the same updates to {phost}') diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index a28c5058..48d8f966 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -17,9 +17,9 @@ from datetime import timedelta from django.template import Library -from django.utils.html import format_html from django.templatetags.static import static from django.utils import timezone +from django.utils.html import format_html from util import get_setting_of_type diff --git a/hosts/utils.py b/hosts/utils.py index f07d5d1e..73db8613 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -15,12 +15,12 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from socket import gethostbyaddr, gaierror, herror +from socket import gaierror, gethostbyaddr, herror -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from taggit.models import Tag -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message def update_rdns(host): @@ -70,7 +70,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.reboot_required = False host.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) if host: host.check_rdns() return host @@ -84,7 +84,13 @@ def clean_tags(): ) tlen = tags.count() if tlen == 0: +<<<<<<< HEAD info_message.send(sender=None, text='No orphaned Tags found.') else: info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') +======= + info_message(text='No orphaned Tags found.') + else: + info_message(text=f'{tlen} orphaned Tags found.') +>>>>>>> docker tags.delete() diff --git a/hosts/views.py b/hosts/views.py index 0fc83ffa..8f20ab19 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -15,24 +15,23 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages - -from taggit.models import Tag +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets +from taggit.models import Tag -from util.filterspecs import Filter, FilterBar -from hosts.models import Host, HostRepo -from domains.models import Domain from arch.models import MachineArchitecture -from operatingsystems.models import OSVariant, OSRelease -from reports.models import Report +from domains.models import Domain from hosts.forms import EditHostForm -from hosts.serializers import HostSerializer, HostRepoSerializer +from hosts.models import Host, HostRepo +from hosts.serializers import HostRepoSerializer, HostSerializer +from operatingsystems.models import OSRelease, OSVariant +from reports.models import Report +from util.filterspecs import Filter, FilterBar @login_required diff --git a/modules/admin.py b/modules/admin.py index 33b94d20..9cf21e6c 100644 --- a/modules/admin.py +++ b/modules/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from modules.models import Module admin.site.register(Module) diff --git a/modules/migrations/0001_initial.py b/modules/migrations/0001_initial.py index 12a8e278..9c27d425 100644 --- a/modules/migrations/0001_initial.py +++ b/modules/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:17 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/modules/migrations/0005_alter_module_unique_together.py b/modules/migrations/0005_alter_module_unique_together.py new file mode 100644 index 00000000..046a0c54 --- /dev/null +++ b/modules/migrations/0005_alter_module_unique_together.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.25 on 2025-11-25 16:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('arch', '0001_initial'), + ('repos', '0006_mirror_errata_checksum_mirror_modules_checksum'), + ('modules', '0004_alter_module_options'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='module', + unique_together={('name', 'stream', 'version', 'context', 'arch', 'repo')}, + ), + ] diff --git a/modules/models.py b/modules/models.py index 931a41c3..e1f8071b 100644 --- a/modules/models.py +++ b/modules/models.py @@ -35,7 +35,7 @@ class Module(models.Model): class Meta: verbose_name = 'Module' verbose_name_plural = 'Modules' - unique_together = ['name', 'stream', 'version', 'context', 'arch'] + unique_together = ['name', 'stream', 'version', 'context', 'arch', 'repo'] ordering = ['name', 'stream'] def __str__(self): diff --git a/modules/utils.py b/modules/utils.py index f56a0f62..248b8b45 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,20 +15,19 @@ # along with Patchman. If not, see from django.db import IntegrityError -from patchman.signals import error_message, info_message -from modules.models import Module from arch.models import PackageArchitecture +from modules.models import Module +from util.logging import error_message, info_message def get_or_create_module(name, stream, version, context, arch, repo): """ Get or create a module object Returns the module """ - created = False - m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) + m_arch, _ = PackageArchitecture.objects.get_or_create(name=arch) try: - module, created = Module.objects.get_or_create( + module, _ = Module.objects.get_or_create( name=name, stream=stream, version=version, @@ -37,7 +36,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): repo=repo, ) except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) module = Module.objects.get( name=name, stream=stream, @@ -73,7 +72,7 @@ def clean_modules(): ) mlen = modules.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned Modules found.') + info_message(text='No orphaned Modules found.') else: - info_message.send(sender=None, text=f'{mlen} orphaned Modules found.') + info_message(text=f'{mlen} orphaned Modules found.') modules.delete() diff --git a/modules/views.py b/modules/views.py index b897a709..2d017220 100644 --- a/modules/views.py +++ b/modules/views.py @@ -14,12 +14,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - -from rest_framework import viewsets, permissions +from django.shortcuts import get_object_or_404, render +from rest_framework import permissions, viewsets from modules.models import Module from modules.serializers import ModuleSerializer diff --git a/operatingsystems/admin.py b/operatingsystems/admin.py index 15f5e200..4884b5eb 100644 --- a/operatingsystems/admin.py +++ b/operatingsystems/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from operatingsystems.models import OSVariant, OSRelease + +from operatingsystems.models import OSRelease, OSVariant class OSReleaseAdmin(admin.ModelAdmin): diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py index 548a7d88..fa319182 100644 --- a/operatingsystems/forms.py +++ b/operatingsystems/forms.py @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ModelForm, ModelMultipleChoiceField -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant from repos.models import Repository diff --git a/operatingsystems/managers.py b/operatingsystems/managers.py index 630484a1..b7b9f24f 100644 --- a/operatingsystems/managers.py +++ b/operatingsystems/managers.py @@ -18,5 +18,5 @@ class OSReleaseManager(models.Manager): - def get_by_natural_key(self, name, codename): - return self.get(name=name, codename=codename) + def get_by_natural_key(self, name, codename, cpe_name): + return self.get(name=name, codename=codename, cpe_name=cpe_name) diff --git a/operatingsystems/migrations/0002_initial.py b/operatingsystems/migrations/0002_initial.py index 517a3f9a..04cbb411 100644 --- a/operatingsystems/migrations/0002_initial.py +++ b/operatingsystems/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/migrations/0003_os_arch.py b/operatingsystems/migrations/0003_os_arch.py index 2778ca3f..4d0e0f93 100644 --- a/operatingsystems/migrations/0003_os_arch.py +++ b/operatingsystems/migrations/0003_os_arch.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:02 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/serializers.py b/operatingsystems/serializers.py index 8418c720..be178909 100644 --- a/operatingsystems/serializers.py +++ b/operatingsystems/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant class OSVariantSerializer(serializers.HyperlinkedModelSerializer): diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 2b696f92..6009f119 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -15,19 +15,22 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse - from rest_framework import viewsets from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from operatingsystems.forms import AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm -from operatingsystems.serializers import OSVariantSerializer, OSReleaseSerializer +from operatingsystems.forms import ( + AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm, +) +from operatingsystems.models import OSRelease, OSVariant +from operatingsystems.serializers import ( + OSReleaseSerializer, OSVariantSerializer, +) @login_required diff --git a/packages/admin.py b/packages/admin.py index 979ba779..bc4b1aaa 100644 --- a/packages/admin.py +++ b/packages/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from packages.models import Package, PackageName, PackageUpdate diff --git a/packages/migrations/0001_initial.py b/packages/migrations/0001_initial.py index 07e7bcb0..bfea3e1a 100644 --- a/packages/migrations/0001_initial.py +++ b/packages/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/migrations/0002_auto_20250207_1319.py b/packages/migrations/0002_auto_20250207_1319.py index 1563d139..4c744203 100644 --- a/packages/migrations/0002_auto_20250207_1319.py +++ b/packages/migrations/0002_auto_20250207_1319.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:19 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/serializers.py b/packages/serializers.py index 902cb3e0..b6e3fb83 100644 --- a/packages/serializers.py +++ b/packages/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from packages.models import PackageName, Package, PackageUpdate +from packages.models import Package, PackageName, PackageUpdate class PackageNameSerializer(serializers.HyperlinkedModelSerializer): diff --git a/packages/utils.py b/packages/utils.py index 9b098225..87395ff6 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -21,8 +21,10 @@ from django.db import IntegrityError, transaction from arch.models import PackageArchitecture -from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString -from patchman.signals import error_message, info_message, warning_message +from packages.models import ( + Package, PackageCategory, PackageName, PackageString, PackageUpdate, +) +from util.logging import error_message, info_message, warning_message def convert_package_to_packagestring(package): @@ -141,7 +143,7 @@ def parse_redhat_package_string(pkg_str): name, epoch, ver, rel, dist, arch = m.groups() else: e = f'Error parsing package string: "{pkg_str}"' - error_message.send(sender=None, text=e) + error_message(text=e) return if dist: rel = f'{rel}.{dist}' @@ -195,7 +197,7 @@ def get_or_create_package(name, epoch, version, release, arch, p_type): package = packages.first() # TODO this should handle gentoo package categories too, otherwise we may be deleting packages # that should be kept - warning_message.send(sender=None, text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') + warning_message(text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') packages.exclude(id=package.id).delete() return package @@ -218,10 +220,10 @@ def get_or_create_package_update(oldpackage, newpackage, security): except MultipleObjectsReturned: e = 'Error: MultipleObjectsReturned when attempting to add package \n' e += f'update with oldpackage={oldpackage} | newpackage={newpackage}:' - error_message.send(sender=None, text=e) + error_message(text=e) updates = PackageUpdate.objects.filter(oldpackage=oldpackage, newpackage=newpackage) for update in updates: - error_message.send(sender=None, text=str(update)) + error_message(text=str(update)) return try: if update: @@ -281,13 +283,13 @@ def clean_packageupdates(): for update in package_updates: if update.host_set.count() == 0: text = f'Removing unused PackageUpdate {update}' - info_message.send(sender=None, text=text) + info_message(text=text) update.delete() for duplicate in package_updates: if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \ update.security == duplicate.security and update.id != duplicate.id: text = f'Removing duplicate PackageUpdate: {update}' - info_message.send(sender=None, text=text) + info_message(text=text) for host in duplicate.host_set.all(): host.updates.remove(duplicate) host.updates.add(update) @@ -307,12 +309,12 @@ def clean_packages(remove_duplicates=False): ) plen = packages.count() if plen == 0: - info_message.send(sender=None, text='No orphaned Packages found.') + info_message(text='No orphaned Packages found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned Packages') + info_message(text=f'Removing {plen} orphaned Packages') packages.delete() if remove_duplicates: - info_message.send(sender=None, text='Checking for duplicate Packages...') + info_message(text='Checking for duplicate Packages...') for package in Package.objects.all(): potential_duplicates = Package.objects.filter( name=package.name, @@ -326,7 +328,7 @@ def clean_packages(remove_duplicates=False): if potential_duplicates.count() > 1: for dupe in potential_duplicates: if dupe.id != package.id: - info_message.send(sender=None, text=f'Removing duplicate Package {dupe}') + info_message(text=f'Removing duplicate Package {dupe}') dupe.delete() @@ -336,7 +338,7 @@ def clean_packagenames(): names = PackageName.objects.filter(package__isnull=True) nlen = names.count() if nlen == 0: - info_message.send(sender=None, text='No orphaned PackageNames found.') + info_message(text='No orphaned PackageNames found.') else: - info_message.send(sender=None, text=f'Removing {nlen} orphaned PackageNames') + info_message(text=f'Removing {nlen} orphaned PackageNames') names.delete() diff --git a/packages/views.py b/packages/views.py index cd53fa6e..413faee0 100644 --- a/packages/views.py +++ b/packages/views.py @@ -15,17 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar -from packages.models import PackageName, Package, PackageUpdate from arch.models import PackageArchitecture -from packages.serializers import PackageNameSerializer, PackageSerializer, PackageUpdateSerializer +from packages.models import Package, PackageName, PackageUpdate +from packages.serializers import ( + PackageNameSerializer, PackageSerializer, PackageUpdateSerializer, +) +from util.filterspecs import Filter, FilterBar @login_required @@ -62,9 +63,16 @@ def package_list(request): if 'affected_by_errata' in request.GET: affected_by_errata = request.GET['affected_by_errata'] == 'true' if affected_by_errata: - packages = packages.filter(erratum__isnull=False) + packages = packages.filter(affected_by_erratum__isnull=False) + else: + packages = packages.filter(affected_by_erratum__isnull=True) + + if 'provides_fix_in_erratum' in request.GET: + provides_fix_in_erratum = request.GET['provides_fix_in_erratum'] == 'true' + if provides_fix_in_erratum: + packages = packages.filter(provides_fix_in_erratum__isnull=False) else: - packages = packages.filter(erratum__isnull=True) + packages = packages.filter(provides_fix_in_erratum__isnull=True) if 'installed_on_hosts' in request.GET: installed_on_hosts = request.GET['installed_on_hosts'] == 'true' @@ -102,6 +110,8 @@ def package_list(request): filter_list = [] filter_list.append(Filter(request, 'Affected by Errata', 'affected_by_errata', {'true': 'Yes', 'false': 'No'})) + filter_list.append(Filter(request, 'Provides Fix in Errata', 'provides_fix_in_erratum', + {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Installed on Hosts', 'installed_on_hosts', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Available in Repos', 'available_in_repos', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Package Type', 'packagetype', Package.PACKAGE_TYPES)) diff --git a/patchman-client.spec b/patchman-client.spec index 68736038..f2f8279a 100644 --- a/patchman-client.spec +++ b/patchman-client.spec @@ -10,7 +10,7 @@ Source: %{expand:%%(pwd)} BuildArch: noarch Requires: curl which coreutils util-linux gawk -%define binary_payload w9.gzdio +%define _binary_payload w9.gzdio %description patchman-client provides a client that uploads reports to a patchman server @@ -20,14 +20,15 @@ find . -mindepth 1 -delete cp -af %{SOURCEURL0}/. . %install -mkdir -p %{buildroot}/usr/sbin -mkdir -p %{buildroot}/etc/patchman -cp ./client/%{name} %{buildroot}/usr/sbin -cp ./client/%{name}.conf %{buildroot}/etc/patchman +mkdir -p %{buildroot}%{_sbindir} +mkdir -p %{buildroot}%{_sysconfdir}/patchman +install -m 755 client/%{name} %{buildroot}%{_sbindir}/%{name} +install -m 644 client/%{name}.conf %{buildroot}%{_sysconfdir}/patchman/%{name}.conf %files -%defattr(755,root,root) -/usr/sbin/patchman-client -%config(noreplace) /etc/patchman/patchman-client.conf +%defattr(-,root,root) +%{_sbindir}/patchman-client +%dir %{_sysconfdir}/patchman +%config(noreplace) %{_sysconfdir}/patchman/patchman-client.conf %changelog diff --git a/patchman/__init__.py b/patchman/__init__.py index af122cc6..321dd7e5 100644 --- a/patchman/__init__.py +++ b/patchman/__init__.py @@ -14,10 +14,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from .receivers import * # noqa - # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app +from .receivers import * # noqa __all__ = ('celery_app',) diff --git a/patchman/celery.py b/patchman/celery.py index 3c58edc5..c47f994d 100644 --- a/patchman/celery.py +++ b/patchman/celery.py @@ -15,10 +15,11 @@ # along with Patchman. If not, see import os + from celery import Celery os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa +from django.conf import settings # noqa app = Celery('patchman') app.config_from_object('django.conf:settings', namespace='CELERY') diff --git a/patchman/receivers.py b/patchman/receivers.py index 5ec32cdd..9393d891 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -15,15 +15,16 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from colorama import init, Fore, Style -from tqdm import tqdm - +from colorama import Fore, Style, init +from django.conf import settings from django.dispatch import receiver +from tqdm.contrib.logging import logging_redirect_tqdm -from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, info_message, warning_message, error_message, debug_message - -from django.conf import settings +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, + warning_message_s, +) +from util.logging import create_pbar, get_quiet_mode, logger, update_pbar init(autoreset=True) @@ -47,37 +48,42 @@ def pbar_update_receiver(**kwargs): update_pbar(index) -@receiver(info_message) -def print_info_message(sender=None, **kwargs): - """ Receiver to print an info message, no color +@receiver(info_message_s) +def print_info_message(**kwargs): + """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.RESET_ALL + Fore.RESET + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(loggers=[logger]): + for line in text.splitlines(): + logger.info(Style.RESET_ALL + Fore.RESET + line) -@receiver(warning_message) +@receiver(warning_message_s) def print_warning_message(**kwargs): - """ Receiver to print a warning message in yellow text + """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.BRIGHT + Fore.YELLOW + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(): + logger.warning(Style.BRIGHT + Fore.YELLOW + text) -@receiver(error_message) +@receiver(error_message_s) def print_error_message(**kwargs): - """ Receiver to print an error message in red text + """ Receiver to handle an error message, red text """ text = str(kwargs.get('text')) if text: - tqdm.write(Style.BRIGHT + Fore.RED + text) + with logging_redirect_tqdm(): + logger.error(Style.BRIGHT + Fore.RED + text) -@receiver(debug_message) +@receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to print a debug message in blue, if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if DEBUG is set """ text = str(kwargs.get('text')) - if get_verbosity() and settings.DEBUG and text: - tqdm.write(Style.BRIGHT + Fore.BLUE + text) + if settings.DEBUG and text: + with logging_redirect_tqdm(loggers=[logger]): + logger.debug(Style.BRIGHT + Fore.BLUE + text) diff --git a/patchman/settings.py b/patchman/settings.py index 557e8c68..c3089caa 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -109,6 +109,10 @@ TAGGIT_CASE_INSENSITIVE = True CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' +CELERY_BROKER_TRANSPORT_OPTIONS = { + 'queue_order_strategy': 'priority', +} +CELERY_WORKER_PREFETCH_MULTIPLIER = 1 LOGIN_REDIRECT_URL = '/patchman/' LOGOUT_REDIRECT_URL = '/patchman/login/' diff --git a/patchman/signals.py b/patchman/signals.py index 917a48e4..799b9c98 100644 --- a/patchman/signals.py +++ b/patchman/signals.py @@ -19,7 +19,7 @@ pbar_start = Signal() pbar_update = Signal() -info_message = Signal() -warning_message = Signal() -error_message = Signal() -debug_message = Signal() +info_message_s = Signal() +warning_message_s = Signal() +error_message_s = Signal() +debug_message_s = Signal() diff --git a/patchman/urls.py b/patchman/urls.py index ee786566..2ae64f56 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -15,12 +15,11 @@ # You should have received a copy of the GNU General Public License # along with If not, see -from django.conf.urls import include, handler404, handler500 # noqa from django.conf import settings +from django.conf.urls import handler404, handler500, include # noqa from django.contrib import admin from django.urls import path from django.views import static - from rest_framework import routers from arch import views as arch_views diff --git a/patchman/wsgi.py b/patchman/wsgi.py index 9a9b4b7f..16f02d5a 100644 --- a/patchman/wsgi.py +++ b/patchman/wsgi.py @@ -19,7 +19,6 @@ from django.core.wsgi import get_wsgi_application os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa - +from django.conf import settings # noqa application = get_wsgi_application() diff --git a/reports/admin.py b/reports/admin.py index a37ec4d0..66e0bf5b 100644 --- a/reports/admin.py +++ b/reports/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from reports.models import Report diff --git a/reports/migrations/0004_migrate_to_tz_aware.py b/reports/migrations/0004_migrate_to_tz_aware.py index 98176510..20510dcd 100644 --- a/reports/migrations/0004_migrate_to_tz_aware.py +++ b/reports/migrations/0004_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Report = apps.get_model('reports', 'Report') for report in Report.objects.all(): diff --git a/reports/models.py b/reports/models.py index 6818ea23..f1e0f6f3 100644 --- a/reports/models.py +++ b/reports/models.py @@ -19,7 +19,7 @@ from django.urls import reverse from hosts.utils import get_or_create_host -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message class Report(models.Model): @@ -97,23 +97,25 @@ def process(self, find_updates=True, verbose=False): """ Process a report and extract os, arch, domain, packages, repos etc """ if not self.os or not self.kernel or not self.arch: - error_message.send(sender=None, text=f'Error: OS, kernel or arch not sent with report {self.id}') + error_message(text=f'Error: OS, kernel or arch not sent with report {self.id}') return if self.processed: - info_message.send(sender=None, text=f'Report {self.id} has already been processed') + info_message(text=f'Report {self.id} has already been processed') return - from reports.utils import get_arch, get_os, get_domain + from reports.utils import get_arch, get_domain, get_os arch = get_arch(self.arch) osvariant = get_os(self.os, arch) domain = get_domain(self.domain) host = get_or_create_host(self, arch, osvariant, domain) if verbose: - info_message.send(sender=None, text=f'Processing report {self.id} - {self.host}') + info_message(text=f'Processing report {self.id} - {self.host}') - from reports.utils import process_packages, process_repos, process_updates, process_modules + from reports.utils import ( + process_modules, process_packages, process_repos, process_updates, + ) process_repos(report=self, host=host) process_modules(report=self, host=host) process_packages(report=self, host=host) @@ -124,5 +126,5 @@ def process(self, find_updates=True, verbose=False): if find_updates: if verbose: - info_message.send(sender=None, text=f'Finding updates for report {self.id} - {self.host}') + info_message(text=f'Finding updates for report {self.id} - {self.host}') host.find_updates() diff --git a/reports/tasks.py b/reports/tasks.py index db9e4103..6bf16e7c 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -16,23 +16,58 @@ # along with Patchman. If not, see from celery import shared_task - +from django.core.cache import cache from django.db.utils import OperationalError from hosts.models import Host from reports.models import Report -from util import info_message +from util.logging import info_message, warning_message -@shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) +@shared_task( + bind=True, + priority=0, + autoretry_for=(OperationalError,), + retry_backoff=True, + retry_kwargs={'max_retries': 5} +) def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - report.process() + report_id_lock_key = f'process_report_id_lock_{report_id}' + if report.host: + report_host_lock_key = f'process_report_host_lock_{report.host}' + else: + report_host_lock_key = f'process_report_host_lock_{report.report_ip}' + # locks will expire after 2 hours + lock_expire = 60 * 60 * 2 + + if cache.add(report_id_lock_key, 'true', lock_expire): + try: + processing_report_id = cache.get(report_host_lock_key) + if processing_report_id: + if processing_report_id > report.id: + warning_message(f'Currently processing a newer report for {report.host} or {report.report_ip}, \ + marking report {report.id} as processed.') + report.processed = True + report.save() + else: + warning_message(f'Currently processing an older report for {report.host} or {report.report_ip}, \ + will skip processing this report.') + else: + try: + cache.set(report_host_lock_key, report.id, lock_expire) + report.process() + finally: + cache.delete(report_host_lock_key) + finally: + cache.delete(report_id_lock_key) + else: + warning_message(f'Already processing report {report_id}, skipping task.') -@shared_task +@shared_task(priority=1) def process_reports(): """ Task to process all unprocessed reports """ @@ -41,12 +76,12 @@ def process_reports(): process_report.delay(report.id) -@shared_task -def clean_reports_with_no_hosts(): - """ Task to clean processed reports where the host no longer exists +@shared_task(priority=2) +def remove_reports_with_no_hosts(): + """ Task to remove processed reports where the host no longer exists """ for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): text = f'Deleting report {report.id} for Host `{report.host}` as the host no longer exists' - info_message.send(sender=None, text=text) + info_message(text=text) report.delete() diff --git a/reports/utils.py b/reports/utils.py index 641f90df..1a08cf3c 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -23,12 +23,18 @@ from domains.models import Domain from hosts.models import HostRepo from modules.utils import get_or_create_module -from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant +from operatingsystems.utils import ( + get_or_create_osrelease, get_or_create_osvariant, +) from packages.models import Package, PackageCategory -from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from patchman.signals import pbar_start, pbar_update, info_message -from repos.models import Repository, Mirror, MirrorPackage +from packages.utils import ( + find_evr, get_or_create_package, get_or_create_package_update, + parse_package_string, +) +from patchman.signals import pbar_start, pbar_update +from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo +from util.logging import debug_message, info_message def process_repos(report, host): @@ -41,6 +47,7 @@ def process_repos(report, host): pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos)) for i, repo_str in enumerate(repos): + debug_message(f'Processing report {report.id} repo: {repo_str}') repo, priority = process_repo(repo_str, report.arch) if repo: repo_ids.append(repo.id) @@ -87,13 +94,14 @@ def process_packages(report, host): packages = parse_packages(report.packages) pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages)) for i, pkg_str in enumerate(packages): + debug_message(f'Processing report {report.id} package: {pkg_str}') package = process_package(pkg_str, report.protocol) if package: package_ids.append(package.id) host.packages.add(package) else: if pkg_str[0].lower() != 'gpg-pubkey': - info_message.send(sender=None, text=f'No package returned for {pkg_str}') + info_message(text=f'No package returned for {pkg_str}') pbar_update.send(sender=None, index=i + 1) for package in host.packages.all(): diff --git a/reports/views.py b/reports/views.py index ccef1bb2..f247deb2 100644 --- a/reports/views.py +++ b/reports/views.py @@ -15,20 +15,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential - -from django.http import HttpResponse, Http404 -from django.views.decorators.csrf import csrf_exempt -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages from django.db.utils import OperationalError +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) -from util.filterspecs import Filter, FilterBar from reports.models import Report +from util.filterspecs import Filter, FilterBar @retry( diff --git a/repos/admin.py b/repos/admin.py index bea87567..a516ff8a 100644 --- a/repos/admin.py +++ b/repos/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from repos.models import Repository, Mirror, MirrorPackage + +from repos.models import Mirror, MirrorPackage, Repository class MirrorAdmin(admin.ModelAdmin): diff --git a/repos/forms.py b/repos/forms.py index 0800a5c3..9cb66897 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -15,10 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField, TextInput, Form, ModelChoiceField, ValidationError from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ( + Form, ModelChoiceField, ModelForm, ModelMultipleChoiceField, TextInput, + ValidationError, +) -from repos.models import Repository, Mirror +from repos.models import Mirror, Repository class EditRepoForm(ModelForm): diff --git a/repos/migrations/0001_initial.py b/repos/migrations/0001_initial.py index a99f6878..1ae96a98 100644 --- a/repos/migrations/0001_initial.py +++ b/repos/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/repos/migrations/0003_migrate_to_tz_aware.py b/repos/migrations/0003_migrate_to_tz_aware.py index dddd78ba..38e30488 100644 --- a/repos/migrations/0003_migrate_to_tz_aware.py +++ b/repos/migrations/0003_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Mirror = apps.get_model('repos', 'Mirror') for mirror in Mirror.objects.all(): diff --git a/repos/models.py b/repos/models.py index 181a103d..9b9082af 100644 --- a/repos/models.py +++ b/repos/models.py @@ -20,13 +20,12 @@ from arch.models import MachineArchitecture from packages.models import Package -from util import get_setting_of_type - -from repos.repo_types.deb import refresh_deb_repo -from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo +from repos.repo_types.deb import refresh_deb_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from patchman.signals import info_message, warning_message, error_message +from repos.repo_types.rpm import refresh_repo_errata, refresh_rpm_repo +from util import get_setting_of_type +from util.logging import error_message, info_message, warning_message class Repository(models.Model): @@ -72,7 +71,7 @@ def show(self): text += f'arch: {self.arch}\n' text += 'Mirrors:' - info_message.send(sender=None, text=text) + info_message(text=text) for mirror in self.mirror_set.all(): mirror.show() @@ -99,10 +98,10 @@ def refresh(self, force=False): refresh_gentoo_repo(self) else: text = f'Error: unknown repo type for repo {self.id}: {self.repotype}' - error_message.send(sender=None, text=text) + error_message(text=text) else: text = 'Repo requires authentication, not updating' - warning_message.send(sender=None, text=text) + warning_message(text=text) def refresh_errata(self, force=False): """ Refresh errata metadata for all of a repos mirrors @@ -168,7 +167,7 @@ def show(self): text = f' {self.id} : {self.url}\n' text += ' last updated: ' text += f'{self.timestamp} checksum: {self.packages_checksum}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def fail(self): """ Records that the mirror has failed @@ -178,10 +177,10 @@ def fail(self): """ if self.repo.auth_required: text = f'Mirror requires authentication, not updating - {self.url}' - warning_message.send(sender=None, text=text) + warning_message(text=text) return text = f'No usable mirror found at {self.url}' - error_message.send(sender=None, text=text) + error_message(text=text) default_max_mirror_failures = 28 max_mirror_failures = get_setting_of_type( setting_name='MAX_MIRROR_FAILURES', @@ -191,11 +190,11 @@ def fail(self): self.fail_count = self.fail_count + 1 if max_mirror_failures == -1: text = f'Mirror has failed {self.fail_count} times, but MAX_MIRROR_FAILURES=-1, not disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) elif self.fail_count > max_mirror_failures: self.refresh = False text = f'Mirror has failed {self.fail_count} times (max={max_mirror_failures}), disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) self.last_access_ok = False self.save() diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 6e85b153..b339ee6a 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,9 +18,13 @@ from io import BytesIO from packages.models import PackageString -from patchman.signals import info_message, warning_message, pbar_start, pbar_update -from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages -from util import get_datetime_now, get_checksum, Checksum +from patchman.signals import pbar_start, pbar_update +from repos.utils import ( + fetch_mirror_data, find_mirror_url, get_max_mirrors, + update_mirror_packages, +) +from util import Checksum, get_checksum, get_datetime_now +from util.logging import info_message, warning_message def refresh_arch_repo(repo): @@ -34,7 +38,7 @@ def refresh_arch_repo(repo): for i, mirror in enumerate(enabled_mirrors): if i >= max_mirrors: text = f'{max_mirrors} Mirrors already refreshed (max={max_mirrors}), skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) break res = find_mirror_url(mirror.url, [fname]) @@ -42,19 +46,19 @@ def refresh_arch_repo(repo): continue mirror_url = res.url text = f'Found Arch Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Arch Repo data') if not package_data: continue computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum @@ -111,5 +115,5 @@ def extract_arch_packages(data): packagetype='A') packages.add(package) else: - info_message.send(sender=None, text='No Packages found in Repo') + info_message(text='No Packages found in Repo') return packages diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 25d8eba7..eea6593f 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -15,13 +15,17 @@ # along with Patchman. If not, see import re + from debian.deb822 import Packages from debian.debian_support import Version from packages.models import PackageString -from patchman.signals import error_message, pbar_start, pbar_update, info_message, warning_message -from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url -from util import get_datetime_now, get_checksum, Checksum, extract +from patchman.signals import pbar_start, pbar_update +from repos.utils import ( + fetch_mirror_data, find_mirror_url, update_mirror_packages, +) +from util import Checksum, extract, get_checksum, get_datetime_now +from util.logging import error_message, info_message, warning_message def extract_deb_packages(data, url): @@ -30,7 +34,7 @@ def extract_deb_packages(data, url): try: extracted = extract(data, url).decode('utf-8') except UnicodeDecodeError as e: - error_message.send(sender=None, text=f'Skipping {url} : {e}') + error_message(text=f'Skipping {url} : {e}') return package_re = re.compile('^Package: ', re.M) plen = len(package_re.findall(extracted)) @@ -61,7 +65,7 @@ def extract_deb_packages(data, url): packagetype='D') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages @@ -71,7 +75,12 @@ def refresh_deb_repo(repo): are and then fetches and extracts packages from those files. """ - formats = ['Packages.xz', 'Packages.bz2', 'Packages.gz', 'Packages'] + formats = [ + 'Packages.xz', + 'Packages.bz2', + 'Packages.gz', + 'Packages', + ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(refresh=True, enabled=True) @@ -81,19 +90,19 @@ def refresh_deb_repo(repo): continue mirror_url = res.url text = f'Found deb Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Debian Repo data') if not package_data: continue computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 8e4198d9..61966f7a 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -14,21 +14,28 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import git import os import shutil import tarfile import tempfile -from defusedxml import ElementTree from fnmatch import fnmatch from io import BytesIO from pathlib import Path +import git +from defusedxml import ElementTree + from packages.models import PackageString from packages.utils import find_evr -from patchman.signals import info_message, warning_message, error_message, pbar_start, pbar_update -from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages -from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid +from patchman.signals import pbar_start, pbar_update +from repos.utils import ( + add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages, +) +from util import ( + Checksum, extract, fetch_content, get_checksum, get_datetime_now, get_url, + response_is_valid, +) +from util.logging import error_message, info_message, warning_message def refresh_gentoo_main_repo(repo): @@ -44,7 +51,7 @@ def refresh_gentoo_main_repo(repo): continue res = get_url(mirror.url + '.md5sum') - data = fetch_content(res, 'Fetching Repo checksum') + data = fetch_content(res, 'Fetching Gentoo Repo checksum') if data is None: mirror.fail() continue @@ -56,7 +63,7 @@ def refresh_gentoo_main_repo(repo): if mirror.packages_checksum == checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue res = get_url(mirror.url) @@ -65,12 +72,12 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Gentoo Repo data') if data is None: mirror.fail() continue extracted = extract(data, mirror.url) - info_message.send(sender=None, text=f'Found Gentoo Repo - {mirror.url}') + info_message(text=f'Found Gentoo Repo - {mirror.url}') computed_checksum = get_checksum(data, Checksum.md5) if not mirror_checksum_is_valid(computed_checksum, checksum, mirror, 'package'): @@ -165,7 +172,7 @@ def get_gentoo_overlay_mirrors(repo_name): if element.text.startswith('http'): mirrors.append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_overlays_url}: {e}') + error_message(text=f'Error parsing {gentoo_overlays_url}: {e}') return mirrors @@ -199,7 +206,7 @@ def get_gentoo_mirror_urls(): if element.get('protocol') == 'http': mirrors[name]['urls'].append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_distfiles_url}: {e}') + error_message(text=f'Error parsing {gentoo_distfiles_url}: {e}') mirror_urls = [] # for now, ignore region data and choose MAX_MIRRORS mirrors at random for _, v in mirrors.items(): @@ -274,7 +281,7 @@ def extract_gentoo_packages_from_ebuilds(extracted_ebuilds): ) packages.add(package) plen = len(packages) - info_message.send(sender=None, text=f'Extracted {plen} Packages', plen=plen) + info_message(text=f'Extracted {plen} Packages', plen=plen) return packages @@ -282,7 +289,7 @@ def extract_gentoo_overlay_packages(mirror): """ Extract packages from gentoo overlay repo """ t = tempfile.mkdtemp() - info_message.send(sender=None, text=f'Extracting Gentoo packages from {mirror.url}') + info_message(text=f'Extracting Gentoo packages from {mirror.url}') git.Repo.clone_from(mirror.url, t, depth=1) packages = set() extracted_ebuilds = extract_gentoo_overlay_ebuilds(t) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index d9501cde..d1482272 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,11 +16,14 @@ from django.db.models import Q -from patchman.signals import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo -from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data +from repos.utils import ( + check_for_metalinks, check_for_mirrorlists, fetch_mirror_data, + find_mirror_url, get_max_mirrors, +) from util import get_datetime_now +from util.logging import info_message, warning_message def refresh_repo_errata(repo): @@ -47,7 +50,7 @@ def max_mirrors_refreshed(repo, checksum, ts): have_checksum_and_ts = repo.mirror_set.filter(mirrors_q).count() if have_checksum_and_ts >= max_mirrors: text = f'{max_mirrors} Mirrors already have this checksum and timestamp, skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) return True return False @@ -57,10 +60,12 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): which type of repo it is, then refreshes the mirrors """ formats = [ + 'repodata/repomd.xml.zst', 'repodata/repomd.xml.xz', 'repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', + 'suse/repodata/repomd.xml.zst', 'suse/repodata/repomd.xml.xz', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', @@ -79,17 +84,17 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): repo_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching rpm Repo data') if not repo_data: continue if mirror_url.endswith('content'): text = f'Found yast rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yast_repo(mirror, repo_data) else: text = f'Found yum rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yum_repo(mirror, repo_data, mirror_url, errata_only) if mirror.last_access_ok: mirror.timestamp = ts diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index 0ef54358..e37b9934 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,9 +17,10 @@ import re from packages.models import PackageString -from patchman.signals import pbar_start, pbar_update, info_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract +from util.logging import info_message def refresh_yast_repo(mirror, data): @@ -65,5 +66,5 @@ def extract_yast_packages(data): packagetype='R') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index 7ac85816..1e96db39 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -15,16 +15,18 @@ # along with Patchman. If not, see from celery import shared_task +from django.core.cache import cache from repos.models import Repository +from util.logging import warning_message -@shared_task +@shared_task(priority=0) def refresh_repo(repo_id, force=False): """ Refresh metadata for a single repo """ - repo = Repository.objects.get(id=repo_id) - repo.refresh(force) + repo_id_lock_key = f'refresh_repos_{repo_id}_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + if cache.add(repo_id_lock_key, 'true', lock_expire): + try: + repo = Repository.objects.get(id=repo_id) + repo.refresh(force) + finally: + cache.delete(repo_id_lock_key) + else: + warning_message(f'Already refreshing repo {repo_id}, skipping task.') -@shared_task + +@shared_task(priority=1) def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ repos = Repository.objects.filter(enabled=True) - for repo in repos: - refresh_repo.delay(repo.id, force) + lock_key = 'refresh_repos_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(lock_key, 'true', lock_expire): + try: + for repo in repos: + refresh_repo.delay(repo.id, force) + finally: + cache.delete(lock_key) + else: + warning_message('Already refreshing repos, skipping task.') diff --git a/repos/utils.py b/repos/utils.py index 49b5d07f..13cee149 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -17,16 +17,24 @@ import re from io import BytesIO -from defusedxml import ElementTree -from tenacity import RetryError +from defusedxml import ElementTree from django.db import IntegrityError from django.db.models import Q +from tenacity import RetryError from packages.models import Package -from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package -from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from patchman.signals import info_message, warning_message, error_message, debug_message, pbar_start, pbar_update +from packages.utils import ( + convert_package_to_packagestring, convert_packagestring_to_package, +) +from patchman.signals import pbar_start, pbar_update +from util import ( + Checksum, extract, fetch_content, get_checksum, get_setting_of_type, + get_url, response_is_valid, +) +from util.logging import ( + debug_message, error_message, info_message, warning_message, +) def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -77,7 +85,7 @@ def update_mirror_packages(mirror, packages): package = convert_packagestring_to_package(strpackage) mirror_package, c = MirrorPackage.objects.get_or_create(mirror=mirror, package=package) except Package.MultipleObjectsReturned: - error_message.send(sender=None, text=f'Duplicate Package found in {mirror}: {strpackage}') + error_message(text=f'Duplicate Package found in {mirror}: {strpackage}') def find_mirror_url(stored_mirror_url, formats): @@ -89,7 +97,7 @@ def find_mirror_url(stored_mirror_url, formats): if mirror_url.endswith(f): mirror_url = mirror_url[:-len(f)] mirror_url = f"{mirror_url.rstrip('/')}/{fmt}" - debug_message.send(sender=None, text=f'Checking for Mirror at {mirror_url}') + debug_message(text=f'Checking for Mirror at {mirror_url}') try: res = get_url(mirror_url) except RetryError: @@ -133,7 +141,7 @@ def get_metalink_urls(url): if greatgreatgrandchild.attrib.get('protocol') in ['https', 'http']: metalink_urls.append(greatgreatgrandchild.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing metalink {url}: {e}') + error_message(text=f'Error parsing metalink {url}: {e}') return metalink_urls @@ -147,17 +155,17 @@ def get_mirrorlist_urls(url): return if response_is_valid(res): try: - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Repo data to check for mirrorlist') if data is None: return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) if mirror_urls: - debug_message.send(sender=None, text=f'Found mirrorlist: {url}') + debug_message(text=f'Found mirrorlist: {url}') return mirror_urls else: - debug_message.send(sender=None, text=f'Not a mirrorlist: {url}') + debug_message(text=f'Not a mirrorlist: {url}') except Exception as e: - error_message.send(sender=None, text=f'Error attempting to parse a mirrorlist: {e} {url}') + error_message(text=f'Error attempting to parse a mirrorlist: {e} {url}') def add_mirrors_from_urls(repo, mirror_urls): @@ -172,15 +180,16 @@ def add_mirrors_from_urls(repo, mirror_urls): existing = repo.mirror_set.filter(q).count() if existing >= max_mirrors: text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more' - warning_message.send(sender=None, text=text) + warning_message(text=text) break from repos.models import Mirror + # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? # that is what metalink urls return now m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) if c: text = f'Added Mirror - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) def check_for_mirrorlists(repo): @@ -193,7 +202,7 @@ def check_for_mirrorlists(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found mirrorlist - {mirror.url}') + info_message(text=f'Found mirrorlist - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -210,7 +219,7 @@ def check_for_metalinks(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found metalink - {mirror.url}') + info_message(text=f'Found metalink - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -249,9 +258,9 @@ def mirror_checksum_is_valid(computed, provided, mirror, metadata_type): """ if not computed or computed != provided: text = f'Checksum failed for mirror {mirror.id}, not refreshing {metadata_type} metadata' - error_message.send(sender=None, text=text) + error_message(text=text) text = f'Found checksum: {computed}\nExpected checksum: {provided}' - error_message.send(sender=None, text=text) + error_message(text=text) mirror.last_access_ok = False mirror.fail() return False @@ -296,9 +305,9 @@ def clean_repos(): repos = Repository.objects.filter(mirror__isnull=True) rlen = repos.count() if rlen == 0: - info_message.send(sender=None, text='No Repositories with zero Mirrors found.') + info_message(text='No Repositories with zero Mirrors found.') else: - info_message.send(sender=None, text=f'Removing {rlen} empty Repositories.') + info_message(text=f'Removing {rlen} empty Repositories.') repos.delete() @@ -309,13 +318,13 @@ def remove_mirror_trailing_slashes(): mirrors = Mirror.objects.filter(url__endswith='/') mlen = mirrors.count() if mlen == 0: - info_message.send(sender=None, text='No Mirrors with trailing slashes found.') + info_message(text='No Mirrors with trailing slashes found.') else: - info_message.send(sender=None, text=f'Removing trailing slashes from {mlen} Mirrors.') + info_message(text=f'Removing trailing slashes from {mlen} Mirrors.') for mirror in mirrors: mirror.url = mirror.url.rstrip('/') try: mirror.save() except IntegrityError: - warning_message.send(sender=None, text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') + warning_message(text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') mirror.delete() diff --git a/repos/views.py b/repos/views.py index 199c834e..1f0c2bfa 100644 --- a/repos/views.py +++ b/repos/views.py @@ -15,24 +15,27 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect -from django.http import HttpResponse -from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse -from django.db.models import Q from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db import IntegrityError - +from django.db.models import Q +from django.http import HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar +from arch.models import MachineArchitecture from hosts.models import HostRepo -from repos.models import Repository, Mirror, MirrorPackage from operatingsystems.models import OSRelease -from arch.models import MachineArchitecture -from repos.forms import EditRepoForm, LinkRepoForm, CreateRepoForm, EditMirrorForm -from repos.serializers import RepositorySerializer, MirrorSerializer, MirrorPackageSerializer +from repos.forms import ( + CreateRepoForm, EditMirrorForm, EditRepoForm, LinkRepoForm, +) +from repos.models import Mirror, MirrorPackage, Repository +from repos.serializers import ( + MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/requirements.txt b/requirements.txt index 2f264c9b..08ce4573 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.25 +Django==4.2.27 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 @@ -19,3 +19,4 @@ redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 +zstandard==0.25.0 diff --git a/sbin/patchman b/sbin/patchman index c0911434..06bef981 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -17,33 +17,38 @@ # along with Patchman. If not, see +import argparse import os import sys -import argparse +from django import setup as django_setup from django.core.exceptions import MultipleObjectsReturned from django.db.models import Count -from django import setup as django_setup os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') from django.conf import settings # noqa + django_setup() from arch.utils import clean_architectures -from errata.utils import mark_errata_security_updates, enrich_errata, \ - scan_package_updates_for_affected_packages from errata.tasks import update_errata +from errata.utils import ( + enrich_errata, mark_errata_security_updates, + scan_package_updates_for_affected_packages, +) from hosts.models import Host from hosts.utils import clean_tags from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) +from reports.models import Report +from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos -from reports.models import Report -from reports.tasks import clean_reports_with_no_hosts from security.utils import update_cves, update_cwes -from util import set_verbosity, get_datetime_now -from patchman.signals import info_message +from util import get_datetime_now +from util.logging import info_message, set_quiet_mode def get_host(host=None, action='Performing action'): @@ -64,7 +69,7 @@ def get_host(host=None, action='Performing action'): matches = Host.objects.filter(hostname__startswith=host).count() text = f'{matches} Hosts match hostname "{host}"' - info_message.send(sender=None, text=text) + info_message(text=text) return host_obj @@ -84,7 +89,7 @@ def get_hosts(hosts=None, action='Performing action'): host_objs.append(host_obj) else: text = f'{action} for all Hosts\n' - info_message.send(sender=None, text=text) + info_message(text=text) host_objs = Host.objects.all() return host_objs @@ -107,7 +112,7 @@ def get_repos(repo=None, action='Performing action', only_enabled=False): else: repos = Repository.objects.all() - info_message.send(sender=None, text=text) + info_message(text=text) return repos @@ -118,9 +123,9 @@ def refresh_repos(repo=None, force=False): repos = get_repos(repo, 'Refreshing metadata', True) for repo in repos: text = f'Repository {repo.id} : {repo}' - info_message.send(sender=None, text=text) + info_message(text=text) repo.refresh(force) - info_message.send(sender=None, text='') + info_message(text='') def list_repos(repos=None): @@ -151,7 +156,7 @@ def clean_reports(hoststr=None): host.clean_reports() if not hoststr: - clean_reports_with_no_hosts() + remove_reports_with_no_hosts() def host_updates_alt(host=None): @@ -161,10 +166,10 @@ def host_updates_alt(host=None): hosts = get_hosts(host, 'Finding updates') ts = get_datetime_now() for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) if host not in updated_hosts: host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') host.updated_at = ts host.save() @@ -200,10 +205,10 @@ def host_updates_alt(host=None): phost.save() updated_hosts.append(phost) text = f'Added the same updates to {phost}' - info_message.send(sender=None, text=text) + info_message(text=text) else: text = 'Updates already added in this run' - info_message.send(sender=None, text=text) + info_message(text=text) def host_updates(host=None): @@ -211,9 +216,9 @@ def host_updates(host=None): """ hosts = get_hosts(host, 'Finding updates') for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') def diff_hosts(hosts): @@ -236,47 +241,47 @@ def diff_hosts(hosts): repo_diff_AB = reposA.difference(reposB) repo_diff_BA = reposB.difference(reposA) - info_message.send(sender=None, text=f'+ {hostA.hostname}') - info_message.send(sender=None, text=f'- {hostB.hostname}') + info_message(text=f'+ {hostA.hostname}') + info_message(text=f'- {hostB.hostname}') if hostA.os != hostB.os: - info_message.send(sender=None, text='\nOperating Systems') - info_message.send(sender=None, text=f'+ {hostA.os}') - info_message.send(sender=None, text=f'- {hostB.os}') + info_message(text='\nOperating Systems') + info_message(text=f'+ {hostA.os}') + info_message(text=f'- {hostB.os}') else: - info_message.send(sender=None, text='\nNo OS differences') + info_message(text='\nNo OS differences') if hostA.arch != hostB.arch: - info_message.send(sender=None, text='\nArchitecture') - info_message.send(sender=None, text=f'+ {hostA.arch}') - info_message.send(sender=None, text=f'- {hostB.arch}') + info_message(text='\nArchitecture') + info_message(text=f'+ {hostA.arch}') + info_message(text=f'- {hostB.arch}') else: - info_message.send(sender=None, text='\nNo Architecture differences') + info_message(text='\nNo Architecture differences') if hostA.kernel != hostB.kernel: - info_message.send(sender=None, text='\nKernels') - info_message.send(sender=None, text=f'+ {hostA.kernel}') - info_message.send(sender=None, text=f'- {hostB.kernel}') + info_message(text='\nKernels') + info_message(text=f'+ {hostA.kernel}') + info_message(text=f'- {hostB.kernel}') else: - info_message.send(sender=None, text='\nNo Kernel differences') + info_message(text='\nNo Kernel differences') if len(package_diff_AB) != 0 or len(package_diff_BA) != 0: - info_message.send(sender=None, text='\nPackages') + info_message(text='\nPackages') for package in package_diff_AB: - info_message.send(sender=None, text=f'+ {package}') + info_message(text=f'+ {package}') for package in package_diff_BA: - info_message.send(sender=None, text=f'- {package}') + info_message(text=f'- {package}') else: - info_message.send(sender=None, text='\nNo Package differences') + info_message(text='\nNo Package differences') if len(repo_diff_AB) != 0 or len(repo_diff_BA) != 0: - info_message.send(sender=None, text='\nRepositories') + info_message(text='\nRepositories') for repo in repo_diff_AB: - info_message.send(sender=None, text=f'+ {repo}') + info_message(text=f'+ {repo}') for repo in repo_diff_BA: - info_message.send(sender=None, text=f'- {repo}') + info_message(text=f'- {repo}') else: - info_message.send(sender=None, text='\nNo Repo differences') + info_message(text='\nNo Repo differences') def delete_hosts(hosts=None): @@ -286,7 +291,7 @@ def delete_hosts(hosts=None): matching_hosts = get_hosts(hosts) for host in matching_hosts: text = f'Deleting host: {host.hostname}:' - info_message.send(sender=None, text=text) + info_message(text=text) host.delete() @@ -300,7 +305,7 @@ def toggle_host_hro(hosts=None, host_repos_only=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} host_repos_only') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.host_repos_only = host_repos_only host.save() @@ -315,7 +320,7 @@ def toggle_host_check_dns(hosts=None, check_dns=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} check_dns') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.check_dns = check_dns host.save() @@ -347,7 +352,7 @@ def process_reports(host=None, force=False): text = 'Processing Reports for all Hosts' reports = Report.objects.filter(processed=force).order_by('created') - info_message.send(sender=None, text=text) + info_message(text=text) for report in reports: report.process(find_updates=False) @@ -542,7 +547,7 @@ def main(): parser = collect_args() args = parser.parse_args() - set_verbosity(not args.quiet) + set_quiet_mode(args.quiet) showhelp = process_args(args) if showhelp: parser.print_help() diff --git a/scripts/rpm-post-install.sh b/scripts/rpm-post-install.sh index 24ade8af..451f7d48 100644 --- a/scripts/rpm-post-install.sh +++ b/scripts/rpm-post-install.sh @@ -4,8 +4,9 @@ if [ ! -e /etc/httpd/conf.d/patchman.conf ] ; then cp /etc/patchman/apache.conf.example /etc/httpd/conf.d/patchman.conf fi -if ! grep /usr/lib/python3.9/site-packages /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then - sed -i -e "s/^\(Define patchman_pythonpath\).*/\1 \/usr\/lib\/python3.9\/site-packages/" \ +PYTHON_SITEPACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])") +if ! grep "${PYTHON_SITEPACKAGES}" /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then + sed -i -e "s|^\(Define patchman_pythonpath\).*|\1 ${PYTHON_SITEPACKAGES}|" \ /etc/httpd/conf.d/patchman.conf fi @@ -24,15 +25,36 @@ patchman-manage makemigrations patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' -chown -R apache:apache /var/lib/patchman -adduser --system --group patchman-celery -usermod -a -G apache patchman-celery -chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db -chcon --type httpd_sys_rw_content_t /var/lib/patchman/db/patchman.db -semanage port -a -t http_port_t -p tcp 5672 -setsebool -P httpd_can_network_memcache 1 +adduser --system --shell /sbin/nologin patchman +usermod -a -G patchman apache +chown root:patchman /etc/patchman/celery.conf +chmod 640 /etc/patchman/celery.conf +chown -R patchman:patchman /var/lib/patchman +semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/patchman/db(/.*)?" +restorecon -Rv /var/lib/patchman/db setsebool -P httpd_can_network_connect 1 +WORKER_COUNT=1 +if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} +fi + +for i in $(seq 1 "${WORKER_COUNT}"); do + systemctl enable --now "patchman-celery-worker@$i.service" +done + +active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') +for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + systemctl stop "$service" + systemctl disable "$service" + fi +done + +systemctl enable --now patchman-celery-beat.service + echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." echo diff --git a/security/admin.py b/security/admin.py index 196a9468..aedeaea9 100644 --- a/security/admin.py +++ b/security/admin.py @@ -15,8 +15,8 @@ # along with Patchman. If not, see from django.contrib import admin -from security.models import CWE, CVSS, CVE, Reference +from security.models import CVE, CVSS, CWE, Reference admin.site.register(CWE) admin.site.register(CVSS) diff --git a/security/models.py b/security/models.py index 9c097eed..405c8db6 100644 --- a/security/models.py +++ b/security/models.py @@ -16,14 +16,14 @@ import json import re -from cvss import CVSS2, CVSS3, CVSS4 from time import sleep +from cvss import CVSS2, CVSS3, CVSS4 from django.db import models from django.urls import reverse from security.managers import CVEManager -from util import get_url, fetch_content, tz_aware_datetime, error_message +from util import error_message, fetch_content, get_url, tz_aware_datetime class Reference(models.Model): @@ -125,6 +125,8 @@ def add_cvss_score(self, vector_string, score=None, severity=None, version=None) score = cvss_score.base_score if not severity: severity = cvss_score.severities()[0] + if isinstance(severity, str): + severity = severity.capitalize() try: cvss, created = CVSS.objects.get_or_create( version=version, @@ -152,7 +154,7 @@ def fetch_mitre_cve_data(self): mitre_cve_url = f'https://cveawg.mitre.org/api/cve/{self.cve_id}' res = get_url(mitre_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} MITRE data') cve_json = json.loads(data) @@ -162,7 +164,7 @@ def fetch_osv_dev_cve_data(self): osv_dev_cve_url = f'https://api.osv.dev/v1/vulns/{self.cve_id}' res = get_url(osv_dev_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} OSV data') cve_json = json.loads(data) @@ -186,7 +188,7 @@ def fetch_nist_cve_data(self): res = get_url(nist_cve_url) data = fetch_content(res, f'Fetching {self.cve_id} NIST data') if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') cve_json = json.loads(data) self.parse_nist_cve_data(cve_json) @@ -197,7 +199,7 @@ def parse_nist_cve_data(self, cve_json): cve = vulnerability.get('cve') cve_id = cve.get('id') if cve_id != self.cve_id: - error_message.send(sender=None, text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') + error_message(text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') return metrics = cve.get('metrics') for metric, score_data in metrics.items(): diff --git a/security/tasks.py b/security/tasks.py index a04bb1c8..0cfbc2f1 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -15,37 +15,81 @@ # along with Patchman. If not, see from celery import shared_task +from django.core.cache import cache from security.models import CVE, CWE +from util.logging import warning_message -@shared_task +@shared_task(priority=3) def update_cve(cve_id): """ Task to update a CVE """ - cve = CVE.objects.get(id=cve_id) - cve.fetch_cve_data() + cve_id_lock_key = f'update_cve_id_lock_{cve_id}' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 -@shared_task + if cache.add(cve_id_lock_key, 'true', lock_expire): + try: + cve = CVE.objects.get(id=cve_id) + cve.fetch_cve_data() + finally: + cache.delete(cve_id_lock_key) + else: + warning_message(f'Already updating CVE {cve_id}, skipping task.') + + +@shared_task(priority=2) def update_cves(): """ Task to update all CVEs """ - for cve in CVE.objects.all(): - update_cve.delay(cve.id) + lock_key = 'update_cves_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cve in CVE.objects.all(): + update_cve.delay(cve.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CVEs, skipping task.') -@shared_task +@shared_task(priority=3) def update_cwe(cwe_id): """ Task to update a CWE """ - cwe = CWE.objects.get(id=cwe_id) - cwe.fetch_cwe_data() + cwe_id_lock_key = f'update_cwe_id_lock_{cwe_id}' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 -@shared_task + if cache.add(cwe_id_lock_key, 'true', lock_expire): + try: + cwe = CWE.objects.get(id=cwe_id) + cwe.fetch_cwe_data() + finally: + cache.delete(cwe_id_lock_key) + else: + warning_message(f'Already updating CWE {cwe_id}, skipping task.') + + +@shared_task(priority=2) def update_cwes(): - """ Task to update all CWEa + """ Task to update all CWEs """ - for cwe in CWE.objects.all(): - update_cwe.delay(cwe.id) + lock_key = 'update_cwes_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cwe in CWE.objects.all(): + update_cwe.delay(cwe.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CWEs, skipping task.') diff --git a/security/views.py b/security/views.py index 58a686b5..c9e606a6 100644 --- a/security/views.py +++ b/security/views.py @@ -14,17 +14,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from packages.models import Package from operatingsystems.models import OSRelease +from packages.models import Package from security.models import CVE, CWE, Reference -from security.serializers import CVESerializer, CWESerializer, ReferenceSerializer +from security.serializers import ( + CVESerializer, CWESerializer, ReferenceSerializer, +) from util.filterspecs import Filter, FilterBar diff --git a/setup.cfg b/setup.cfg index 7af9ccb0..b1d5ee4e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ requires = /usr/bin/python3 python3-importlib-metadata python3-cvss python3-redis + python3-zstandard redis celery python3-django-celery-beat diff --git a/setup.py b/setup.py index 6ec6d974..a7dbfc68 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,9 @@ # along with Patchman. If not, see import os -from setuptools import setup, find_packages +import sys + +from setuptools import find_packages, setup with open('VERSION.txt', 'r', encoding='utf_8') as v: version = v.readline().strip() @@ -28,8 +30,14 @@ with open('requirements.txt', 'r', encoding='utf_8') as rt: install_requires = rt.read().splitlines() - data_files = [] +if 'bdist_rpm' in sys.argv: + data_files.append( + ('/usr/lib/systemd/system', [ + 'etc/systemd/system/patchman-celery-worker@.service', + 'etc/systemd/system/patchman-celery-beat.service' + ]) + ) for dirpath, dirnames, filenames in os.walk('etc'): # Ignore dirnames that start with '.' diff --git a/util/__init__.py b/util/__init__.py index ac6f8f1b..b85e5e37 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -15,28 +15,40 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import requests import bz2 -import magic -import zlib import lzma import os +import zlib + +import magic +import requests + +try: + # python 3.14+ - can also remove the dependency at that stage + from compression import zstd +except ImportError: + import zstandard as zstd + from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 -from requests.exceptions import HTTPError, Timeout, ConnectionError -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential from time import time -from tqdm import tqdm -from django.utils.timezone import make_aware -from django.utils.dateparse import parse_datetime from django.conf import settings +from django.utils.dateparse import parse_datetime +from django.utils.timezone import make_aware +from requests.exceptions import ConnectionError, HTTPError, Timeout +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) -from patchman.signals import error_message, info_message, debug_message +from util.logging import ( + create_pbar, debug_message, error_message, info_message, quiet_mode, + update_pbar, +) pbar = None -verbose = None +verbose = not quiet_mode Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') http_proxy = os.getenv('http_proxy') @@ -47,40 +59,6 @@ } -def get_verbosity(): - """ Get the global verbosity level - """ - return verbose - - -def set_verbosity(value): - """ Set the global verbosity level - """ - global verbose - verbose = value - - -def create_pbar(ptext, plength, ljust=35, **kwargs): - """ Create a global progress bar if global verbose is True - """ - global pbar - if verbose and plength > 0: - jtext = str(ptext).ljust(ljust) - pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') - return pbar - - -def update_pbar(index, **kwargs): - """ Update the global progress bar if global verbose is True - """ - global pbar - if verbose and pbar: - pbar.update(n=index-pbar.n) - if index >= pbar.total: - pbar.close() - pbar = None - - def fetch_content(response, text='', ljust=35): """ Display a progress bar to fetch the request content if verbose is True. Otherwise, just return the request content @@ -104,7 +82,7 @@ def fetch_content(response, text='', ljust=35): data += chunk return data else: - info_message.send(sender=None, text=text) + info_message(text=text) return response.content @@ -123,16 +101,16 @@ def get_url(url, headers=None, params=None): if not params: params = {} try: - debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') + debug_message(text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) - debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') + debug_message(text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response response.raise_for_status() except requests.exceptions.TooManyRedirects: - error_message.send(sender=None, text=f'Too many redirects - {url}') + error_message(text=f'Too many redirects - {url}') except ConnectionError: - error_message.send(sender=None, text=f'Connection error - {url}') + error_message(text=f'Connection error - {url}') return response @@ -175,7 +153,7 @@ def gunzip(contents): wbits = zlib.MAX_WBITS | 32 return zlib.decompress(contents, wbits) except zlib.error as e: - error_message.send(sender=None, text='gunzip: ' + str(e)) + error_message(text='gunzip: ' + str(e)) def bunzip2(contents): @@ -186,10 +164,10 @@ def bunzip2(contents): return bzip2data except IOError as e: if e == 'invalid data stream': - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) except ValueError as e: if e == "couldn't find end of stream": - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) def unxz(contents): @@ -199,7 +177,17 @@ def unxz(contents): xzdata = lzma.decompress(contents) return xzdata except lzma.LZMAError as e: - error_message.send(sender=None, text='lzma: ' + e) + error_message(text='lzma: ' + e) + + +def unzstd(contents): + """ unzstd contents in memory and return the data + """ + try: + zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() + return zstddata + except zstd.ZstdError as e: + error_message(text=f'zstd: {e}') def extract(data, fmt): @@ -214,6 +202,8 @@ def extract(data, fmt): m = magic.open(magic.MAGIC_MIME) m.load() mime = m.buffer(data).split(';')[0] + if mime == 'application/zstd' or fmt.endswith('zst'): + return unzstd(data) if mime == 'application/x-xz' or fmt.endswith('xz'): return unxz(data) elif mime == 'application/x-bzip2' or fmt.endswith('bz2'): @@ -236,7 +226,7 @@ def get_checksum(data, checksum_type): checksum = get_md5(data) else: text = f'Unknown checksum type: {checksum_type}' - error_message.send(sender=None, text=text) + error_message(text=text) return checksum diff --git a/util/filterspecs.py b/util/filterspecs.py index 722b45df..eac0f747 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -15,10 +15,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.utils.safestring import mark_safe -from django.db.models.query import QuerySet from operator import itemgetter +from django.db.models.query import QuerySet +from django.utils.safestring import mark_safe + def get_query_string(qs): new_qs = [f'{k}={v}' for k, v in list(qs.items())] diff --git a/util/logging.py b/util/logging.py new file mode 100644 index 00000000..bf532e54 --- /dev/null +++ b/util/logging.py @@ -0,0 +1,87 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + + +import logging + +from django.conf import settings +from tqdm import tqdm + +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, warning_message_s, +) + +log_format = '[%(asctime)s] %(levelname)s: %(message)s' +if settings.DEBUG: + logging_level = logging.DEBUG +else: + logging_level = logging.INFO +logging.basicConfig(level=logging_level, format=log_format) +logger = logging.getLogger() +logging.getLogger('git.cmd').setLevel(logging.WARNING) + +quiet_mode = False +pbar = None + + +def get_quiet_mode(): + """ Get the global quiet_mode + """ + return quiet_mode + + +def set_quiet_mode(value): + """ Set the global quiet_mode + """ + global quiet_mode + quiet_mode = value + + +def create_pbar(ptext, plength, ljust=35, **kwargs): + """ Create a global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and plength > 0: + jtext = str(ptext).ljust(ljust) + pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') + return pbar + + +def update_pbar(index, **kwargs): + """ Update the global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and pbar: + pbar.update(n=index-pbar.n) + if index >= pbar.total: + pbar.close() + pbar = None + + +def info_message(text): + info_message_s.send(sender=None, text=text) + + +def warning_message(text): + warning_message_s.send(sender=None, text=text) + + +def debug_message(text): + debug_message_s.send(sender=None, text=text) + + +def error_message(text): + error_message_s.send(sender=None, text=text) diff --git a/util/tasks.py b/util/tasks.py index f650e3e2..12825a8c 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -18,11 +18,13 @@ from arch.utils import clean_architectures from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from repos.utils import clean_repos, remove_mirror_trailing_slashes -@shared_task +@shared_task(priority=1) def clean_database(remove_duplicate_packages=False): """ Task to check the database and remove orphaned objects Runs all clean_* functions to check database consistency diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 2aea1e5e..674e1721 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -15,16 +15,15 @@ # along with Patchman. If not, see import re - -from humanize import naturaltime from datetime import datetime, timedelta from urllib.parse import urlencode +from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template -from django.utils.html import format_html from django.templatetags.static import static -from django.core.paginator import Paginator +from django.utils.html import format_html +from humanize import naturaltime from util import get_setting_of_type diff --git a/util/views.py b/util/views.py index b66db6b0..fd003bca 100644 --- a/util/views.py +++ b/util/views.py @@ -17,16 +17,16 @@ from datetime import datetime, timedelta -from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.contrib.sites.models import Site from django.db.models import F +from django.shortcuts import render from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from repos.models import Repository, Mirror +from operatingsystems.models import OSRelease, OSVariant from packages.models import Package from reports.models import Report +from repos.models import Mirror, Repository from util import get_setting_of_type