Allow subscription to changes of object matching a Query

adminapi/dataset/__init__.py

@@ -1,6 +1,7 @@
 from distutils.util import strtobool
 from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
 from itertools import chain
+from time import sleep
 from types import GeneratorType
 
 from adminapi.datatype import validate_value, json_to_datatype
@@ -11,14 +12,15 @@
 COMMIT_ENDPOINT = '/dataset/commit'
 QUERY_ENDPOINT = '/dataset/query'
 CREATE_ENDPOINT = '/dataset/create'
+GET_COMMITS_ENDPOINT = '/dataset/get_commits'
 
 
 class DatasetError(Exception):
     pass
 
 
 class BaseQuery(object):
-    def __init__(self, filters=None, restrict=None, order_by=None):
+    def __init__(self, filters=None, restrict=[], order_by=None):
         if filters is None:
             self._filters = None
             self._results = []
@@ -52,7 +54,7 @@ def __repr__(self):
 
     def _get_results(self):
         if self._results is None:
-            self._results = list(self._fetch_results())
+            self._results = self._fetch_results()
         return self._results
 
     def _fetch_results(self):
@@ -194,6 +196,17 @@ def get_free_ip_addrs(self):
                     if host not in used_hosts:
                         yield host
 
+    def subscribe_changes(self, callback, newer_than_mins=None, slot=None):
+        self._results = self._fetch_results()
+        while True:
+            for commit in self._fetch_commits(newer_than_mins, slot):
+                callback(commit)
+                self._save_changes_slot(commit)
+            sleep(1)
+
+    def _apply_commit(self, commit):
+        pass
+
 
 class Query(BaseQuery):
 
@@ -217,14 +230,31 @@ def commit(self):
     def _fetch_results(self):
         request_data = {'filters': self._filters}
         if self._restrict is not None:
-            request_data['restrict'] = self._restrict
+            request_data['restrict'] = list(self._restrict) + ['object_id']
         if self._order_by is not None:
             request_data['order_by'] = self._order_by
 
         response = send_request(QUERY_ENDPOINT, post_params=request_data)
         if response['status'] == 'error':
             _handle_exception(response)
-        return (_format_obj(s) for s in response['result'])
+        return [_format_obj(s) for s in response['result']]
+
+    def _fetch_commits(self, newer_than_mins=None, slot=None):
+        request_data = {'filters': self._filters}
+        if self._restrict is not None:
+            request_data['restrict'] = self._restrict
+        if self._order_by is not None:
+            request_data['order_by'] = self._order_by
+        if newer_than_mins is not None:
+            request_data['newer_than_mins'] = newer_than_mins
+        if slot is not None:
+            request_data['slot'] = slot
+
+        response = send_request(GET_COMMITS_ENDPOINT, post_params=request_data)
+        for result in response['result']:
+            print(result)
+            if False:
+                yield result
 
 
 class DatasetObject(dict):
@@ -502,7 +532,7 @@ def _handle_exception(result):
 
 # XXX: Deprecated, use Query() instead
 def query(**kwargs):
-    return Query(kwargs)
+    return Query(kwargs, None)
 
 
 # XXX: Deprecated, use Query().new_object() instead

serveradmin/access_control/migrations/0001_initial.py

@@ -9,21 +9,18 @@ class Migration(migrations.Migration):
 
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('apps', '__first__'),
+        ('apps', '0001_initial'),
     ]
 
     operations = [
         migrations.CreateModel(
             name='AccessControlGroup',
             fields=[
-                ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
                 ('name', models.CharField(max_length=80, unique=True)),
-                ('create_server_query', models.CharField(max_length=1000)),
-                ('edit_server_query', models.CharField(max_length=1000)),
-                ('commit_server_query', models.CharField(max_length=1000)),
-                ('delete_server_query', models.CharField(max_length=1000)),
-                ('applications', models.ManyToManyField(blank=True, to='apps.Application', related_name='access_control_groups')),
-                ('members', models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL, related_name='access_control_groups')),
+                ('query', models.CharField(max_length=1000)),
+                ('applications', models.ManyToManyField(related_name='access_control_groups', blank=True, to='apps.Application')),
+                ('members', models.ManyToManyField(related_name='access_control_groups', blank=True, to=settings.AUTH_USER_MODEL)),
             ],
             options={
                 'db_table': 'access_control_group',

serveradmin/access_control/migrations/0002_accesscontrolgroup_attributes.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2018-04-25 15:58
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('serverdb', '0001_initial'),
+        ('access_control', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='accesscontrolgroup',
+            name='attributes',
+            field=models.ManyToManyField(blank=True, related_name='access_control_groups', to='serverdb.Attribute'),
+        ),
+    ]

serveradmin/access_control/models.py

@@ -3,14 +3,12 @@
 
 from adminapi.parse import parse_query
 from serveradmin.apps.models import Application
+from serveradmin.serverdb.models import Attribute
 
 
 class AccessControlGroup(Model):
     name = CharField(max_length=80, unique=True)
-    create_server_query = CharField(max_length=1000)
-    edit_server_query = CharField(max_length=1000)
-    commit_server_query = CharField(max_length=1000)
-    delete_server_query = CharField(max_length=1000)
+    query = CharField(max_length=1000)
     members = ManyToManyField(
         User,
         blank=True,
@@ -23,24 +21,24 @@ class AccessControlGroup(Model):
         limit_choices_to={'disabled': False, 'superuser': False},
         related_name='access_control_groups',
     )
+    attributes = ManyToManyField(
+        Attribute,
+        blank=True,
+        limit_choices_to={'readonly': False},
+        related_name='access_control_groups',
+    )
 
     class Meta:
         db_table = 'access_control_group'
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._filters = {}
+        self._filters = None
 
     def __str__(self):
         return self.name
 
-    def get_filters(self, action):
-        if action not in self._filters:
-            query = getattr(self, action + '_server_query')
-            self._filters[action] = parse_query(query)
-        return self._filters[action]
-
-    def match_server(self, action, server):
-        return all(
-            f.matches(server[a]) for a, f in self.get_filters(action).items()
-        )
+    def get_filters(self):
+        if self._filters is None:
+            self._filters = parse_query(self.query)
+        return self._filters

serveradmin/api/urls.py

@@ -6,6 +6,7 @@
     dataset_commit,
     dataset_new_object,
     dataset_create,
+    dataset_get_commits,
     api_call,
 )
 
@@ -15,5 +16,6 @@
     url('^dataset/commit$', dataset_commit),
     url('^dataset/new_object$', dataset_new_object),
     url('^dataset/create$', dataset_create),
+    url('^dataset/get_commits$', dataset_get_commits),
     url('^call$', api_call),
 ]

serveradmin/api/views.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from operator import itemgetter
 
 from django.core.exceptions import (
@@ -13,11 +14,12 @@
 from serveradmin.api import ApiError, AVAILABLE_API_FUNCTIONS
 from serveradmin.api.decorators import api_view
 from serveradmin.api.utils import build_function_description
+from serveradmin.apps.models import ApplicationSlot
+from serveradmin.changes.models import Commit
 from serveradmin.serverdb.query_committer import QueryCommitter
-from serveradmin.serverdb.query_filterer import QueryFilterer
+from serveradmin.serverdb.query_executer import execute_query
 from serveradmin.serverdb.query_materializer import (
-    QueryMaterializer,
-    get_default_attribute_values,
+    get_default_attribute_values
 )
 
 
@@ -71,21 +73,12 @@ def dataset_query(request, app, data):
         for attr, filter_obj in data['filters'].items():
             filters[attr] = filter_from_obj(filter_obj)
 
-        # Empty list means query all attributes to the older versions of
-        # the adminapi.
-        if not data.get('restrict'):
-            restrict = None
-        else:
-            restrict = data['restrict']
-
+        restrict = data.get('restrict')
         order_by = data.get('order_by')
 
-        filterer = QueryFilterer(filters)
-        materializer = QueryMaterializer(filterer, restrict, order_by)
-
         return {
             'status': 'success',
-            'result': list(materializer),
+            'result': execute_query(filters, restrict, order_by),
         }
     except (FilterValueError, ValidationError) as error:
         return {
@@ -221,6 +214,43 @@ def dataset_create(request, app, data):
     }
 
 
+@api_view
+def dataset_get_commits(request, app, data):
+    if request.method != 'POST':
+        raise SuspiciousOperation('Method not allowed')
+    if not isinstance(data, dict):
+        raise SuspiciousOperation('Invalid payload')
+
+    if 'filters' not in data or not isinstance(data['filters'], dict):
+        raise SuspiciousOperation('"filters" must be a dictionary')
+
+    queryset = Commit.objects
+
+    if 'newer_than_mins' in data:
+        if not isinstance(data['newer_than_mins'], int):
+            raise SuspiciousOperation('"newer_than_mins" must be an int')
+        queryset = queryset.filter(change_on__gt=(
+            datetime.now() - timedelta(minutes=data['newer_than_mins'])
+        ))
+
+    if 'slot' in data:
+        if not isinstance(data['slot'], str):
+            raise SuspiciousOperation('"newer_than_mins" must be a string')
+        try:
+            slot = ApplicationSlot.objects.get(
+                application=app, name=data['slot']
+            )
+        except ApplicationSlot.DoesNotExist:
+            pass
+        else:
+            queryset = queryset.filter(id__gt=slot.commit_id)
+
+    return [{
+        'user': c.user.username,
+        'application': c.app.application_id,
+    } for c in queryset]
+
+
 @api_view
 def api_call(request, app, data):
     try:

serveradmin/changes/admin.py

@@ -0,0 +1,12 @@
+from django.contrib.admin import site, ModelAdmin, TabularInline
+
+from serveradmin.changes.models import Commit, Addition, Modification, Deletion
+
+
+site.register(Commit, type('CommitAdmin', (ModelAdmin, ), {
+    'inlines': [
+        type('AdditionInline', (TabularInline, ), {'model': Addition}),
+        type('ModificationInline', (TabularInline, ), {'model': Modification}),
+        type('DeletionInline', (TabularInline, ), {'model': Deletion}),
+    ]
+}))

serveradmin/changes/migrations/0001_initial.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+from django.conf import settings
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('apps', '0001_initial'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Addition',
+            fields=[
+                ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+                ('server_id', models.IntegerField(db_index=True)),
+                ('attributes_json', models.TextField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Commit',
+            fields=[
+                ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+                ('change_on', models.DateTimeField(default=django.utils.timezone.now, db_index=True)),
+                ('app', models.ForeignKey(to='apps.Application', on_delete=django.db.models.deletion.PROTECT, null=True)),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=django.db.models.deletion.PROTECT, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Deletion',
+            fields=[
+                ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+                ('server_id', models.IntegerField(db_index=True)),
+                ('attributes_json', models.TextField()),
+                ('commit', models.ForeignKey(to='changes.Commit')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Modification',
+            fields=[
+                ('id', models.AutoField(serialize=False, auto_created=True, verbose_name='ID', primary_key=True)),
+                ('server_id', models.IntegerField(db_index=True)),
+                ('updates_json', models.TextField()),
+                ('commit', models.ForeignKey(to='changes.Commit')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='addition',
+            name='commit',
+            field=models.ForeignKey(to='changes.Commit'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='modification',
+            unique_together=set([('commit', 'server_id')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='deletion',
+            unique_together=set([('commit', 'server_id')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='addition',
+            unique_together=set([('commit', 'server_id')]),
+        ),
+    ]