feat(indexer): Add JSON schema support and apply for it before indexing

Author: stronk7

CHANGELOG.md

@@ -8,12 +8,16 @@ and commits should be formatted using [Conventional Commits](https://www.convent
 
 ## [Unreleased]
 
+### Added
+
+- Indexer: Add JSON schema support and apply for it before indexing
+
 ### Changed
 
 - GitHub: Run workflows also with Python 3.14 (aka, πthon) by @stronk7 ([ffc9b17](https://github.com/moodlehq/wiki-rag/commit/ffc9b17312ee7e190050585d674d3f11510fa8f2))
-- Loader: Improve the generation of page head sections
-- Indexer: Better handling of preamble and contents on indexing
-- Searcher: Improve the "popularity" optimisation
+- Loader: Improve the generation of page head sections by @stronk7 ([a228019](https://github.com/moodlehq/wiki-rag/commit/a228019a03284aa21da8661adfd2c23dac9eee8d))
+- Indexer: Better handling of preamble and contents on indexing by @stronk7 ([996eeec](https://github.com/moodlehq/wiki-rag/commit/996eeecfc9d3f49ee6b9264ae77cea9fa5bb0f34))
+- Searcher: Improve the "popularity" optimisation by @stronk7 ([5e84ba5](https://github.com/moodlehq/wiki-rag/commit/5e84ba5039fdb02a8b5549297036cbe7c3931d03))
 ## [0.11.2] - 2025-10-22
 
 ### Added

wiki_rag/index/main.py

@@ -107,11 +107,11 @@ def main():
     # TODO: Make this to accept CLI argument or, by default, use the last file in the directory.
     input_file = loader_dump_path / input_candidate
 
-    logger.info(f"Loading parsed pages from json: {input_file}, namespaces: {mediawiki_namespaces}")
+    logger.info(f"Loading parsed pages from JSON: {input_file}, namespaces: {mediawiki_namespaces}")
     information = load_parsed_information(input_file)
     # TODO: Multiple site information handling should be implemented here.
     pages = information["sites"][0]["pages"]
-    logger.info(f"Loaded {len(pages)} pages from json file")
+    logger.info(f"Loaded {len(pages)} pages from JSON file")
 
     temp_collection_name = f"{collection_name}_temp"
     logger.info(f'Preparing new temp collection "{temp_collection_name}" schema')

wiki_rag/index/util.py

@@ -9,6 +9,7 @@
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
+from jsonschema import ValidationError, validate
 from langchain_openai import OpenAIEmbeddings
 from pymilvus import (
     CollectionSchema,
@@ -22,6 +23,8 @@
 
 import wiki_rag.index as index
 
+from wiki_rag import ROOT_DIR
+
 logger = logging.getLogger(__name__)
 
 
@@ -54,11 +57,15 @@ def load_parsed_information(input_file: Path) -> dict:
             ]
         }
 
-    # If the loaded information is not a dictionary or is missing "meta" and "sites" properties,
-    # this is wrong and we should error. Raise an exception.
-    if not isinstance(information, dict) or "meta" not in information or "sites" not in information:
-        msg = f"Error with the format from {input_file}: missing 'meta' or 'sites' properties."
-        raise ValueError(msg)
+    # Let's validate the schema as much as we can.
+    schema = json.load(open(ROOT_DIR / "wiki_rag/schema.json"))
+    try:
+        validate(information, schema)
+        logger.debug("Successfully parsed the JSON information")
+    except ValidationError as e:
+        msg = f"Error validating the JSON information from {input_file}: {e}"
+        logger.error(msg)
+        exit(1)
 
     return information
 

wiki_rag/schema.json

@@ -0,0 +1,91 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/moodlehq/wiki-rag/raw/refs/heads/main/wiki_rag/schema.json",
+  "title": "Wiki-RAG data files schema",
+  "type": "object",
+  "required": ["meta", "sites"],
+  "unevaluatedProperties": false,
+  "properties": {
+    "meta": {
+      "type": "object",
+      "required": ["timestamp", "num_sites"],
+      "unevaluatedProperties": false,
+      "properties": {
+        "timestamp": { "type": "string", "format": "date-time" },
+        "num_sites": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "sites": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/site" }
+    }
+  },
+
+  "$defs": {
+    "uuid": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+    },
+    "uuidOrNull": {
+      "type": ["string", "null"],
+      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+    },
+
+    "site": {
+      "type": "object",
+      "required": ["site_url", "num_pages", "pages"],
+      "unevaluatedProperties": false,
+      "properties": {
+        "site_url": { "type": "string", "format": "uri" },
+        "num_pages": { "type": "integer", "minimum": 0 },
+        "pages": { "type": "array", "items": { "$ref": "#/$defs/page" } }
+      }
+    },
+
+    "page": {
+      "type": "object",
+      "required": ["id", "title", "sections"],
+      "unevaluatedProperties": false,
+      "properties": {
+        "id": { "type": "integer" },
+        "title": { "type": "string" },
+        "sections": { "type": "array", "items": { "$ref": "#/$defs/section" } },
+        "categories": { "type": "array", "items": { "type": "string" } },
+        "templates": { "type": "array", "items": { "type": "string" } },
+        "internal_links": { "type": "array", "items": { "type": "string" } },
+        "external_links": { "type": "array", "items": { "type": "string" } },
+        "language_links": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+
+    "section": {
+      "type": "object",
+      "required": [
+        "id", "anchor", "title", "source", "text", "all_links", "wiki_links",
+        "index", "level", "page_id", "doc_id", "doc_title", "doc_hash",
+        "parent", "children", "previous", "next", "relations"
+      ],
+      "unevaluatedProperties": false,
+      "properties": {
+        "id": { "$ref": "#/$defs/uuid" },
+        "anchor": { "type": "string" },
+        "title": { "type": "string" },
+        "source": { "type": "string", "format": "uri" },
+        "text": { "type": "string" },
+        "all_links": { "type": "array", "items": { "type": "string" } },
+        "wiki_links": { "type": "array", "items": { "type": "string" } },
+        "index": { "type": "integer", "minimum": 0 },
+        "level": { "type": "integer", "minimum": 0 },
+        "page_id": { "type": "integer" },
+        "doc_id": { "$ref": "#/$defs/uuid" },
+        "doc_title": { "type": "string" },
+        "doc_hash": { "$ref": "#/$defs/uuid" },
+        "parent": { "$ref": "#/$defs/uuidOrNull" },
+        "children": { "type": "array", "items": { "$ref": "#/$defs/uuidOrNull" } },
+        "previous": { "type": "array", "items": { "$ref": "#/$defs/uuidOrNull" } },
+        "next": { "type": "array", "items": { "$ref": "#/$defs/uuidOrNull" } },
+        "relations": { "type": "array", "items": { "$ref": "#/$defs/uuidOrNull" } }
+      }
+    }
+  }
+}