Merge pull request #15 from planet-a-ventures/joscha/update-spec-and-fix-list

fix: update spec and fix ignore list

Author: joscha

affinity_pipeline.py

@@ -15,6 +15,7 @@ def load_affinity_data() -> None:
             ListReference(247888, 1869904),
             ListReference(69224, 351112),
             ListReference(126638, 1133940),
+            ListReference(126638, 1899475),
             ListReference(157541, 831583),
         ],
         dev_mode=DEV_MODE,

dlt_source_affinity/model/v2/__init__.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from typing import Any
 
 import dlt
@@ -10,11 +11,13 @@
     JSONResponseCursorPaginator,
 )
 
+from .model.v2 import ValueType
 from .settings import API_BASE, V2_PREFIX
 from .type_adapters import error_adapter
 
 # Share a session (and thus pool) between all rest clients
 session: Session = None
+logger = logging.getLogger("dlt")
 
 
 def get_v2_rest_client(
@@ -89,10 +92,13 @@ def remove_unknown_fields(response: Response, *args: Any, **kwargs: Any) -> None
                             if isinstance(fields, list):
                                 to_remove = []
                                 for field in fields:
-                                    if isinstance(field, dict) and (
-                                        field["value"]["type"] == "connections"
-                                        or field["value"]["type"] == "note"
+                                    if (
+                                        isinstance(field, dict)
+                                        and field["value"]["type"] not in ValueType
                                     ):
+                                        logger.warning(
+                                            f"Removing field with unknown type: {field['value']['type']}"
+                                        )
                                         to_remove.append(field)
                                         changed = True
                                 for field in to_remove: