340 Improve query performance for related models

[arthanson]

Fixes: #340

Here is a temp management command can use or adapt that shows the issue, If you create a COT with id:1 add a field f1 of text and f2 Multi-object pointing to site. If have demo data then Site id: 24 should be the first. Run the script and it will create 20K objects pointing to that Site. Go to the Site detail page query count before is about 20,000 with a noticeable delay before loading. After change it is down to 53 queries.

from django.core.management.base import BaseCommand
from django.db import transaction
from netbox_custom_objects.models import CustomObjectType
from dcim.models import Site


class Command(BaseCommand):
    help = 'Creates 20,000 test custom objects for performance testing'

    def add_arguments(self, parser):
        parser.add_argument(
            '--custom-object-type-id',
            type=int,
            default=1,
            help='ID of the custom object type to create instances for (default: 1)',
        )
        parser.add_argument(
            '--count',
            type=int,
            default=20000,
            help='Number of test objects to create (default: 20000)',
        )
        parser.add_argument(
            '--text-field',
            type=str,
            default='f1',
            help='Name of the text field to populate (default: f1)',
        )
        parser.add_argument(
            '--multiobject-field',
            type=str,
            default='f2',
            help='Name of the multiobject field to populate (default: f2)',
        )
        parser.add_argument(
            '--site-id',
            type=int,
            default=24,
            help='Site ID to link all objects to (default: 24)',
        )
        parser.add_argument(
            '--batch-size',
            type=int,
            default=1000,
            help='Number of objects to create per batch (default: 1000)',
        )

    def handle(self, *args, **options):
        custom_object_type_id = options['custom_object_type_id']
        count = options['count']
        text_field = options['text_field']
        multiobject_field = options['multiobject_field']
        site_id = options['site_id']
        batch_size = options['batch_size']

        self.stdout.write(
            f"Creating {count} test custom objects for CustomObjectType ID {custom_object_type_id}..."
        )

        try:
            print("--------------------------------")
            print("Starting test custom object creation")
            print("--------------------------------")
            # Get the custom object type
            custom_object_type = CustomObjectType.objects.get(pk=custom_object_type_id)
            self.stdout.write(f"Found CustomObjectType: {custom_object_type.name}")

            # Get the site
            site = Site.objects.get(pk=site_id)
            self.stdout.write(f"Found Site: {site.name} (ID: {site_id})")

            # Get the dynamic model
            model = custom_object_type.get_model()
            self.stdout.write(f"Got model: {model.__name__}")

            # Delete all existing objects
            existing_count = model.objects.count()
            if existing_count > 0:
                self.stdout.write(f"Deleting {existing_count} existing objects...")
                model.objects.all().delete()
                self.stdout.write(self.style.SUCCESS(f"Deleted {existing_count} existing objects"))

            # Verify the fields exist
            model_fields = [f.name for f in model._meta.get_fields()]
            if not hasattr(model(), text_field):
                self.stderr.write(
                    self.style.ERROR(
                        f"Text field '{text_field}' does not exist on this model. "
                        f"Available fields: {model_fields}"
                    )
                )
                return

            if not hasattr(model(), multiobject_field):
                self.stderr.write(
                    self.style.ERROR(
                        f"Multiobject field '{multiobject_field}' does not exist on this model. "
                        f"Available fields: {model_fields}"
                    )
                )
                return

            # Create objects in batches
            created_count = 0
            all_created_objects = []

            for batch_start in range(0, count, batch_size):
                batch_end = min(batch_start + batch_size, count)
                objects_to_create = []

                for i in range(batch_start, batch_end):
                    # Create object with text field set to co1, co2, co3, etc.
                    obj_data = {text_field: f"co{i + 1}"}
                    objects_to_create.append(model(**obj_data))

                # Bulk create the batch
                with transaction.atomic():
                    created_objects = model.objects.bulk_create(objects_to_create)
                    all_created_objects.extend(created_objects)
                    created_count += len(created_objects)

                self.stdout.write(f"Created {created_count}/{count} objects...")

            # Now add the site relationship to all created objects
            self.stdout.write(f"Linking all objects to site {site.name}...")

            # Add relationships in batches for better performance
            for batch_start in range(0, len(all_created_objects), batch_size):
                batch_end = min(batch_start + batch_size, len(all_created_objects))
                batch = all_created_objects[batch_start:batch_end]

                with transaction.atomic():
                    for obj in batch:
                        # Get the multiobject field manager and add the site
                        m2m_field = getattr(obj, multiobject_field)
                        m2m_field.add(site)

                self.stdout.write(f"Linked {batch_end}/{len(all_created_objects)} objects to site...")

            self.stdout.write(
                self.style.SUCCESS(
                    f"Successfully created {created_count} test custom objects "
                    f"and linked them all to site '{site.name}'!"
                )
            )

        except CustomObjectType.DoesNotExist:
            self.stderr.write(
                self.style.ERROR(
                    f"CustomObjectType with ID {custom_object_type_id} does not exist"
                )
            )
        except Site.DoesNotExist:
            self.stderr.write(
                self.style.ERROR(f"Site with ID {site_id} does not exist")
            )
        except Exception as e:
            self.stderr.write(
                self.style.ERROR(f"Error creating test objects: {str(e)}")
            )
            raise