expose include_orig_elements param in api

.github/workflows/schoolai-build-publish.yml

@@ -0,0 +1,34 @@
+name: SchoolAI Build and Publish
+
+# Builds the SchoolAI fork image and publishes it to our Artifact Registry so
+# the webapp (apps/web RAG pipeline) can run a build.
+#
+# Mirrors the webapp build pattern (see web-app:
+# .github/workflows/webapp-staging-deploy.yml -> build_webapp) by reusing the
+# shared SchoolAI/github-actions-workflows sai-docker-build.yaml workflow.
+
+on:
+  workflow_call:
+  # Keep workflow_dispatch so operators can trigger a manual staging deploy
+  # even though `staging-deploy.yml` is now the primary entrypoint on main.
+  workflow_dispatch:
+
+jobs:
+  build_unstructured_api:
+    uses: SchoolAI/github-actions-workflows/.github/workflows/sai-docker-build.yaml@main
+    with:
+      # amd64 only — matches the cluster nodeSelector (kubernetes.io/arch: amd64)
+      # and how the other SchoolAI services build.
+      build_amd_enabled: true
+      build_arm_enabled: false
+      docker_build_amd_runs_on: sai-7cpu-26gb-amd-perf
+      # Produces images at:
+      #   us-central1-docker.pkg.dev/schoolai-global/docker/unstructured-api:<tag>
+      docker_repo: us-central1-docker.pkg.dev/schoolai-global/docker
+      docker_image_name: unstructured-api
+      dockerfile: ./Dockerfile
+      docker_build_args: |
+        PIPELINE_PACKAGE=general
+      push_image: true
+    secrets:
+      GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY_STAGING }}

prepline_general/api/general.py

@@ -223,6 +223,7 @@ def pipeline_api(
     new_after_n_chars: Optional[int],
     overlap: int,
     overlap_all: bool,
+    include_orig_elements: bool = True,
     # ----------------------
     filename: str = "",
     file_content_type: Optional[str] = None,
@@ -283,6 +284,7 @@ def pipeline_api(
                         "new_after_n_chars": new_after_n_chars,
                         "overlap": overlap,
                         "overlap_all": overlap_all,
+                        "include_orig_elements": include_orig_elements,
                         "starting_page_number": starting_page_number,
                         "include_slide_notes": include_slide_notes,
                     },
@@ -338,6 +340,7 @@ def pipeline_api(
                         "max_characters": max_characters,
                         "overlap": overlap,
                         "overlap_all": overlap_all,
+                        "include_orig_elements": include_orig_elements,
                         "extract_image_block_types": extract_image_block_types,
                         "extract_image_block_to_payload": extract_image_block_to_payload,
                         "unique_element_ids": unique_element_ids,
@@ -368,6 +371,7 @@ def pipeline_api(
             "max_characters": max_characters,
             "overlap": overlap,
             "overlap_all": overlap_all,
+            "include_orig_elements": include_orig_elements,
             "extract_image_block_types": extract_image_block_types,
             "extract_image_block_to_payload": extract_image_block_to_payload,
             "unique_element_ids": unique_element_ids,
@@ -716,6 +720,7 @@ def response_generator(is_multipart: bool):
                 new_after_n_chars=form_params.new_after_n_chars,
                 overlap=form_params.overlap,
                 overlap_all=form_params.overlap_all,
+                include_orig_elements=form_params.include_orig_elements,
                 starting_page_number=form_params.starting_page_number,
                 include_slide_notes=form_params.include_slide_notes,
             )

prepline_general/api/models/form_params.py

@@ -35,6 +35,7 @@ class GeneralFormParams(BaseModel):
     new_after_n_chars: Optional[int]
     overlap: int
     overlap_all: bool
+    include_orig_elements: bool
     starting_page_number: Optional[int] = None
     include_slide_notes: bool
 
@@ -236,6 +237,18 @@ def as_form(
                 examples=[True],
             ),
         ] = False,
+        include_orig_elements: Annotated[
+            bool,
+            Form(
+                title="Include Orig Elements",
+                description="""When `True` (the default), the elements used to form each chunk are
+added to that chunk's `.metadata.orig_elements` as a gzipped+base64 blob. Set to `False` to omit
+them and produce a much smaller payload — useful for large tables, where this blob is duplicated
+into every chunk and can balloon the response size dramatically.""",
+                examples=[False],
+            ),
+            BeforeValidator(SmartValueParser[bool]().value_or_first_element),
+        ] = True,
         starting_page_number: Annotated[
             Optional[int],
             Form(
@@ -283,6 +296,7 @@ def as_form(
             new_after_n_chars=new_after_n_chars,
             overlap=overlap,
             overlap_all=overlap_all,
+            include_orig_elements=include_orig_elements,
             unique_element_ids=unique_element_ids,
             starting_page_number=starting_page_number,
             include_slide_notes=include_slide_notes,