feat(scrape): add json output format to POST /web/scrape (#1213)

Stainless-Generated-From: 0fbc1cc

Author: stlc-bot

lib/context_dev/models/web_scrape_params.rb

@@ -25,6 +25,12 @@ class WebScrapeParams < ContextDev::Internal::Type::BaseModel
       #   @return [ContextDev::Models::WebScrapeParams::ImageParams, nil]
       optional :image_params, -> { ContextDev::WebScrapeParams::ImageParams }, api_name: :imageParams
 
+      # @!attribute json_params
+      #   Required when formats.json is true.
+      #
+      #   @return [ContextDev::Models::WebScrapeParams::JsonParams, nil]
+      optional :json_params, -> { ContextDev::WebScrapeParams::JsonParams }, api_name: :jsonParams
+
       # @!attribute markdown_params
       #   Markdown options. Requires formats.markdown: true.
       #
@@ -86,7 +92,7 @@ class WebScrapeParams < ContextDev::Internal::Type::BaseModel
       #   @return [Symbol, ContextDev::Models::WebScrapeParams::Zdr, nil]
       optional :zdr, enum: -> { ContextDev::WebScrapeParams::Zdr }
 
-      # @!method initialize(formats:, url:, image_params: nil, markdown_params: nil, max_age_ms: nil, parse_params: nil, screenshot_params: nil, shared_params: nil, tags: nil, timeout_opts: nil, zdr: nil, request_options: {})
+      # @!method initialize(formats:, url:, image_params: nil, json_params: nil, markdown_params: nil, max_age_ms: nil, parse_params: nil, screenshot_params: nil, shared_params: nil, tags: nil, timeout_opts: nil, zdr: nil, request_options: {})
       #   Some parameter documentations has been truncated, see
       #   {ContextDev::Models::WebScrapeParams} for more details.
       #
@@ -96,6 +102,8 @@ class WebScrapeParams < ContextDev::Internal::Type::BaseModel
       #
       #   @param image_params [ContextDev::Models::WebScrapeParams::ImageParams] Image options. Requires formats.images: true.
       #
+      #   @param json_params [ContextDev::Models::WebScrapeParams::JsonParams] Required when formats.json is true.
+      #
       #   @param markdown_params [ContextDev::Models::WebScrapeParams::MarkdownParams] Markdown options. Requires formats.markdown: true.
       #
       #   @param max_age_ms [Integer] Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and update
@@ -133,6 +141,15 @@ class Formats < ContextDev::Internal::Type::BaseModel
         #   @return [Boolean, nil]
         optional :images, ContextDev::Internal::Type::Boolean
 
+        # @!attribute json
+        #   Page data extracted by an LLM from the page Markdown into jsonParams.schema;
+        #   values carried only in attributes or CSS classes need formats.parse instead.
+        #   Adds four credits when the page has text to extract; when shared content filters
+        #   leave no text the result is an empty object and only the base price applies.
+        #
+        #   @return [Boolean, nil]
+        optional :json, ContextDev::Internal::Type::Boolean
+
         # @!attribute markdown
         #   Page content as Markdown.
         #
@@ -151,7 +168,10 @@ class Formats < ContextDev::Internal::Type::BaseModel
         #   @return [Boolean, nil]
         optional :screenshot, ContextDev::Internal::Type::Boolean
 
-        # @!method initialize(bytes: nil, html: nil, images: nil, markdown: nil, parse: nil, screenshot: nil)
+        # @!method initialize(bytes: nil, html: nil, images: nil, json: nil, markdown: nil, parse: nil, screenshot: nil)
+        #   Some parameter documentations has been truncated, see
+        #   {ContextDev::Models::WebScrapeParams::Formats} for more details.
+        #
         #   Outputs to return. Enable at least one; omitted formats are false.
         #
         #   @param bytes [Boolean] The original HTTP response body.
@@ -160,6 +180,8 @@ class Formats < ContextDev::Internal::Type::BaseModel
         #
         #   @param images [Boolean] Images found on the page.
         #
+        #   @param json [Boolean] Page data extracted by an LLM from the page Markdown into jsonParams.schema; val
+        #
         #   @param markdown [Boolean] Page content as Markdown.
         #
         #   @param parse [Boolean] Fields selected by parseParams.rules.
@@ -218,6 +240,35 @@ module Enrich
         end
       end
 
+      class JsonParams < ContextDev::Internal::Type::BaseModel
+        # @!attribute schema
+        #   JSON Schema for the returned object. Must describe a top-level object; at most
+        #   50 KB serialized. Optional fields the page does not state are omitted, or null
+        #   when their type allows null, while required non-nullable fields always receive a
+        #   best-effort value, so prefer nullable or optional fields for data a page may
+        #   omit. Zod users can pass the output of z.toJSONSchema().
+        #
+        #   @return [Hash{Symbol=>Object}]
+        required :schema, ContextDev::Internal::Type::HashOf[ContextDev::Internal::Type::Unknown]
+
+        # @!attribute instructions
+        #   Optional guidance on which facts to prioritize or how to interpret schema
+        #   fields.
+        #
+        #   @return [String, nil]
+        optional :instructions, String
+
+        # @!method initialize(schema:, instructions: nil)
+        #   Some parameter documentations has been truncated, see
+        #   {ContextDev::Models::WebScrapeParams::JsonParams} for more details.
+        #
+        #   Required when formats.json is true.
+        #
+        #   @param schema [Hash{Symbol=>Object}] JSON Schema for the returned object. Must describe a top-level object; at most 5
+        #
+        #   @param instructions [String] Optional guidance on which facts to prioritize or how to interpret schema fields
+      end
+
       class MarkdownParams < ContextDev::Internal::Type::BaseModel
         # @!attribute include_images
         #

lib/context_dev/models/web_scrape_response.rb

@@ -31,6 +31,15 @@ class WebScrapeResponse < ContextDev::Internal::Type::BaseModel
       #   @return [ContextDev::Models::WebScrapeResponse::Images]
       required :images, -> { ContextDev::Models::WebScrapeResponse::Images }
 
+      # @!attribute json
+      #   Page data extracted into jsonParams.schema, after shared content filters. Values
+      #   are grounded in the page; optional fields the page does not state are omitted,
+      #   or null when their type allows null. An empty object when the filters leave no
+      #   text.
+      #
+      #   @return [ContextDev::Models::WebScrapeResponse::Json]
+      required :json, -> { ContextDev::Models::WebScrapeResponse::Json }
+
       # @!attribute markdown
       #   Markdown after content filters.
       #
@@ -81,7 +90,7 @@ class WebScrapeResponse < ContextDev::Internal::Type::BaseModel
       #   @return [ContextDev::Models::WebScrapeResponse::KeyMetadata, nil]
       optional :key_metadata, -> { ContextDev::Models::WebScrapeResponse::KeyMetadata }
 
-      # @!method initialize(bytes:, cache_metadata:, html:, images:, markdown:, metadata:, parsed:, request_id:, screenshot:, url:, is_partial: nil, key_metadata: nil)
+      # @!method initialize(bytes:, cache_metadata:, html:, images:, json:, markdown:, metadata:, parsed:, request_id:, screenshot:, url:, is_partial: nil, key_metadata: nil)
       #   Some parameter documentations has been truncated, see
       #   {ContextDev::Models::WebScrapeResponse} for more details.
       #
@@ -93,6 +102,8 @@ class WebScrapeResponse < ContextDev::Internal::Type::BaseModel
       #
       #   @param images [ContextDev::Models::WebScrapeResponse::Images] Images after content filters. Empty when none are found.
       #
+      #   @param json [ContextDev::Models::WebScrapeResponse::Json] Page data extracted into jsonParams.schema, after shared content filters. Values
+      #
       #   @param markdown [ContextDev::Models::WebScrapeResponse::Markdown] Markdown after content filters.
       #
       #   @param metadata [ContextDev::Models::WebScrapeResponse::Metadata] Page details, when available.
@@ -304,6 +315,28 @@ module Classification
         end
       end
 
+      # @see ContextDev::Models::WebScrapeResponse#json
+      class Json < ContextDev::Internal::Type::BaseModel
+        # @!attribute data
+        #
+        #   @return [Hash{Symbol=>Object}, nil]
+        required :data, ContextDev::Internal::Type::HashOf[ContextDev::Internal::Type::Unknown], nil?: true
+
+        # @!attribute requested
+        #
+        #   @return [Boolean]
+        required :requested, ContextDev::Internal::Type::Boolean
+
+        # @!method initialize(data:, requested:)
+        #   Page data extracted into jsonParams.schema, after shared content filters. Values
+        #   are grounded in the page; optional fields the page does not state are omitted,
+        #   or null when their type allows null. An empty object when the filters leave no
+        #   text.
+        #
+        #   @param data [Hash{Symbol=>Object}, nil]
+        #   @param requested [Boolean]
+      end
+
       # @see ContextDev::Models::WebScrapeResponse#markdown
       class Markdown < ContextDev::Internal::Type::BaseModel
         # @!attribute data

lib/context_dev/resources/web.rb

@@ -183,21 +183,25 @@ def map_urls(params)
       #
       # Reuse cached outputs independently and capture missing formats in one page
       # visit. Each cache key includes only the settings that affect that output. HTML
-      # is shared with Markdown and parsed fields. Cached outputs can come from
-      # different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests
-      # use the existing fast acquisition path. One credit per request, including cache
-      # hits, or two with browser actions; PDF OCR adds one credit per recovered page on
+      # is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can
+      # come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only
+      # requests use the existing fast acquisition path. One credit per request,
+      # including cache hits and missing pages, or two with browser actions; JSON
+      # extraction adds four credits and runs an LLM over the page Markdown on every
+      # request that has text to extract; PDF OCR adds one credit per recovered page on
       # fresh extraction. Original response bytes and screenshots are limited to 20 MiB
       # each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
       #
-      # @overload scrape(formats:, url:, image_params: nil, markdown_params: nil, max_age_ms: nil, parse_params: nil, screenshot_params: nil, shared_params: nil, tags: nil, timeout_opts: nil, zdr: nil, request_options: {})
+      # @overload scrape(formats:, url:, image_params: nil, json_params: nil, markdown_params: nil, max_age_ms: nil, parse_params: nil, screenshot_params: nil, shared_params: nil, tags: nil, timeout_opts: nil, zdr: nil, request_options: {})
       #
       # @param formats [ContextDev::Models::WebScrapeParams::Formats] Outputs to return. Enable at least one; omitted formats are false.
       #
       # @param url [String] The URL to scrape.
       #
       # @param image_params [ContextDev::Models::WebScrapeParams::ImageParams] Image options. Requires formats.images: true.
       #
+      # @param json_params [ContextDev::Models::WebScrapeParams::JsonParams] Required when formats.json is true.
+      #
       # @param markdown_params [ContextDev::Models::WebScrapeParams::MarkdownParams] Markdown options. Requires formats.markdown: true.
       #
       # @param max_age_ms [Integer] Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and update

rbi/context_dev/models/web_scrape_params.rbi

@@ -33,6 +33,17 @@ module ContextDev
       end
       attr_writer :image_params
 
+      # Required when formats.json is true.
+      sig { returns(T.nilable(ContextDev::WebScrapeParams::JsonParams)) }
+      attr_reader :json_params
+
+      sig do
+        params(
+          json_params: ContextDev::WebScrapeParams::JsonParams::OrHash
+        ).void
+      end
+      attr_writer :json_params
+
       # Markdown options. Requires formats.markdown: true.
       sig { returns(T.nilable(ContextDev::WebScrapeParams::MarkdownParams)) }
       attr_reader :markdown_params
@@ -126,6 +137,7 @@ module ContextDev
           formats: ContextDev::WebScrapeParams::Formats::OrHash,
           url: String,
           image_params: ContextDev::WebScrapeParams::ImageParams::OrHash,
+          json_params: ContextDev::WebScrapeParams::JsonParams::OrHash,
           markdown_params: ContextDev::WebScrapeParams::MarkdownParams::OrHash,
           max_age_ms: Integer,
           parse_params: ContextDev::WebScrapeParams::ParseParams::OrHash,
@@ -145,6 +157,8 @@ module ContextDev
         url:,
         # Image options. Requires formats.images: true.
         image_params: nil,
+        # Required when formats.json is true.
+        json_params: nil,
         # Markdown options. Requires formats.markdown: true.
         markdown_params: nil,
         # Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and
@@ -182,6 +196,7 @@ module ContextDev
             formats: ContextDev::WebScrapeParams::Formats,
             url: String,
             image_params: ContextDev::WebScrapeParams::ImageParams,
+            json_params: ContextDev::WebScrapeParams::JsonParams,
             markdown_params: ContextDev::WebScrapeParams::MarkdownParams,
             max_age_ms: Integer,
             parse_params: ContextDev::WebScrapeParams::ParseParams,
@@ -227,6 +242,16 @@ module ContextDev
         sig { params(images: T::Boolean).void }
         attr_writer :images
 
+        # Page data extracted by an LLM from the page Markdown into jsonParams.schema;
+        # values carried only in attributes or CSS classes need formats.parse instead.
+        # Adds four credits when the page has text to extract; when shared content filters
+        # leave no text the result is an empty object and only the base price applies.
+        sig { returns(T.nilable(T::Boolean)) }
+        attr_reader :json
+
+        sig { params(json: T::Boolean).void }
+        attr_writer :json
+
         # Page content as Markdown.
         sig { returns(T.nilable(T::Boolean)) }
         attr_reader :markdown
@@ -254,6 +279,7 @@ module ContextDev
             bytes: T::Boolean,
             html: T::Boolean,
             images: T::Boolean,
+            json: T::Boolean,
             markdown: T::Boolean,
             parse: T::Boolean,
             screenshot: T::Boolean
@@ -266,6 +292,11 @@ module ContextDev
           html: nil,
           # Images found on the page.
           images: nil,
+          # Page data extracted by an LLM from the page Markdown into jsonParams.schema;
+          # values carried only in attributes or CSS classes need formats.parse instead.
+          # Adds four credits when the page has text to extract; when shared content filters
+          # leave no text the result is an empty object and only the base price applies.
+          json: nil,
           # Page content as Markdown.
           markdown: nil,
           # Fields selected by parseParams.rules.
@@ -281,6 +312,7 @@ module ContextDev
               bytes: T::Boolean,
               html: T::Boolean,
               images: T::Boolean,
+              json: T::Boolean,
               markdown: T::Boolean,
               parse: T::Boolean,
               screenshot: T::Boolean
@@ -445,6 +477,60 @@ module ContextDev
         end
       end
 
+      class JsonParams < ContextDev::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              ContextDev::WebScrapeParams::JsonParams,
+              ContextDev::Internal::AnyHash
+            )
+          end
+
+        # JSON Schema for the returned object. Must describe a top-level object; at most
+        # 50 KB serialized. Optional fields the page does not state are omitted, or null
+        # when their type allows null, while required non-nullable fields always receive a
+        # best-effort value, so prefer nullable or optional fields for data a page may
+        # omit. Zod users can pass the output of z.toJSONSchema().
+        sig { returns(T::Hash[Symbol, T.anything]) }
+        attr_accessor :schema
+
+        # Optional guidance on which facts to prioritize or how to interpret schema
+        # fields.
+        sig { returns(T.nilable(String)) }
+        attr_reader :instructions
+
+        sig { params(instructions: String).void }
+        attr_writer :instructions
+
+        # Required when formats.json is true.
+        sig do
+          params(
+            schema: T::Hash[Symbol, T.anything],
+            instructions: String
+          ).returns(T.attached_class)
+        end
+        def self.new(
+          # JSON Schema for the returned object. Must describe a top-level object; at most
+          # 50 KB serialized. Optional fields the page does not state are omitted, or null
+          # when their type allows null, while required non-nullable fields always receive a
+          # best-effort value, so prefer nullable or optional fields for data a page may
+          # omit. Zod users can pass the output of z.toJSONSchema().
+          schema:,
+          # Optional guidance on which facts to prioritize or how to interpret schema
+          # fields.
+          instructions: nil
+        )
+        end
+
+        sig do
+          override.returns(
+            { schema: T::Hash[Symbol, T.anything], instructions: String }
+          )
+        end
+        def to_hash
+        end
+      end
+
       class MarkdownParams < ContextDev::Internal::Type::BaseModel
         OrHash =
           T.type_alias do