|
2 | 2 |
|
3 | 3 | from __future__ import annotations |
4 | 4 |
|
| 5 | +from typing import Dict |
5 | 6 | from typing_extensions import Literal |
6 | 7 |
|
7 | 8 | import httpx |
8 | 9 |
|
9 | 10 | from ..types import ( |
10 | 11 | web_search_params, |
| 12 | + web_extract_params, |
11 | 13 | web_screenshot_params, |
12 | 14 | web_web_crawl_md_params, |
13 | 15 | web_extract_fonts_params, |
|
29 | 31 | ) |
30 | 32 | from .._base_client import make_request_options |
31 | 33 | from ..types.web_search_response import WebSearchResponse |
| 34 | +from ..types.web_extract_response import WebExtractResponse |
32 | 35 | from ..types.web_screenshot_response import WebScreenshotResponse |
33 | 36 | from ..types.web_web_crawl_md_response import WebWebCrawlMdResponse |
34 | 37 | from ..types.web_extract_fonts_response import WebExtractFontsResponse |
@@ -61,6 +64,99 @@ def with_streaming_response(self) -> WebResourceWithStreamingResponse: |
61 | 64 | """ |
62 | 65 | return WebResourceWithStreamingResponse(self) |
63 | 66 |
|
| 67 | + def extract( |
| 68 | + self, |
| 69 | + *, |
| 70 | + schema: Dict[str, object], |
| 71 | + url: str, |
| 72 | + fact_check: bool | Omit = omit, |
| 73 | + follow_subdomains: bool | Omit = omit, |
| 74 | + include_frames: bool | Omit = omit, |
| 75 | + instructions: str | Omit = omit, |
| 76 | + max_age_ms: int | Omit = omit, |
| 77 | + pdf: web_extract_params.Pdf | Omit = omit, |
| 78 | + stop_after_ms: int | Omit = omit, |
| 79 | + timeout_ms: int | Omit = omit, |
| 80 | + wait_for_ms: int | Omit = omit, |
| 81 | + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. |
| 82 | + # The extra values given here take precedence over values defined on the client or passed to this method. |
| 83 | + extra_headers: Headers | None = None, |
| 84 | + extra_query: Query | None = None, |
| 85 | + extra_body: Body | None = None, |
| 86 | + timeout: float | httpx.Timeout | None | NotGiven = not_given, |
| 87 | + ) -> WebExtractResponse: |
| 88 | + """ |
| 89 | + Crawl a website, convert pages to Markdown using the scrape cache, and extract |
| 90 | + structured data into the provided JSON Schema. The schema must describe the |
| 91 | + response data object. This endpoint does not accept targeted page-type |
| 92 | + selection. |
| 93 | +
|
| 94 | + Args: |
| 95 | + schema: JSON Schema for the returned data object. TypeScript Zod users can pass a JSON |
| 96 | + Schema generated from a Zod object; Python users can pass the equivalent JSON |
| 97 | + Schema object. |
| 98 | +
|
| 99 | + url: The starting website URL to crawl and extract from. Must include http:// or |
| 100 | + https://. |
| 101 | +
|
| 102 | + fact_check: When true (default), every returned value must be grounded in facts stated on |
| 103 | + the page; fields that cannot be supported by the page are returned as |
| 104 | + null/empty. When false, the model may make reasonable inferences and derivations |
| 105 | + from the page content (e.g. ideal customer, competitor analysis, |
| 106 | + recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, |
| 107 | + metrics) faithful to the source. |
| 108 | +
|
| 109 | + follow_subdomains: When true, follow links on subdomains of the starting URL's domain. |
| 110 | +
|
| 111 | + include_frames: When true, iframe contents are included in Markdown before extraction. |
| 112 | +
|
| 113 | + instructions: Optional extraction guidance, such as which facts to prioritize or how to |
| 114 | + interpret fields in the schema. |
| 115 | +
|
| 116 | + max_age_ms: Return cached scrape results if a prior scrape for the same parameters is |
| 117 | + younger than this many milliseconds. |
| 118 | +
|
| 119 | + stop_after_ms: Soft time budget for the crawl in milliseconds. |
| 120 | +
|
| 121 | + timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer |
| 122 | + than this value, it will be aborted with a 408 status code. Maximum allowed |
| 123 | + value is 300000ms (5 minutes). |
| 124 | +
|
| 125 | + wait_for_ms: Optional browser wait time in milliseconds after initial page load for each |
| 126 | + crawled page. |
| 127 | +
|
| 128 | + extra_headers: Send extra headers |
| 129 | +
|
| 130 | + extra_query: Add additional query parameters to the request |
| 131 | +
|
| 132 | + extra_body: Add additional JSON properties to the request |
| 133 | +
|
| 134 | + timeout: Override the client-level default timeout for this request, in seconds |
| 135 | + """ |
| 136 | + return self._post( |
| 137 | + "/web/extract", |
| 138 | + body=maybe_transform( |
| 139 | + { |
| 140 | + "schema": schema, |
| 141 | + "url": url, |
| 142 | + "fact_check": fact_check, |
| 143 | + "follow_subdomains": follow_subdomains, |
| 144 | + "include_frames": include_frames, |
| 145 | + "instructions": instructions, |
| 146 | + "max_age_ms": max_age_ms, |
| 147 | + "pdf": pdf, |
| 148 | + "stop_after_ms": stop_after_ms, |
| 149 | + "timeout_ms": timeout_ms, |
| 150 | + "wait_for_ms": wait_for_ms, |
| 151 | + }, |
| 152 | + web_extract_params.WebExtractParams, |
| 153 | + ), |
| 154 | + options=make_request_options( |
| 155 | + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout |
| 156 | + ), |
| 157 | + cast_to=WebExtractResponse, |
| 158 | + ) |
| 159 | + |
64 | 160 | def extract_fonts( |
65 | 161 | self, |
66 | 162 | *, |
@@ -758,6 +854,99 @@ def with_streaming_response(self) -> AsyncWebResourceWithStreamingResponse: |
758 | 854 | """ |
759 | 855 | return AsyncWebResourceWithStreamingResponse(self) |
760 | 856 |
|
| 857 | + async def extract( |
| 858 | + self, |
| 859 | + *, |
| 860 | + schema: Dict[str, object], |
| 861 | + url: str, |
| 862 | + fact_check: bool | Omit = omit, |
| 863 | + follow_subdomains: bool | Omit = omit, |
| 864 | + include_frames: bool | Omit = omit, |
| 865 | + instructions: str | Omit = omit, |
| 866 | + max_age_ms: int | Omit = omit, |
| 867 | + pdf: web_extract_params.Pdf | Omit = omit, |
| 868 | + stop_after_ms: int | Omit = omit, |
| 869 | + timeout_ms: int | Omit = omit, |
| 870 | + wait_for_ms: int | Omit = omit, |
| 871 | + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. |
| 872 | + # The extra values given here take precedence over values defined on the client or passed to this method. |
| 873 | + extra_headers: Headers | None = None, |
| 874 | + extra_query: Query | None = None, |
| 875 | + extra_body: Body | None = None, |
| 876 | + timeout: float | httpx.Timeout | None | NotGiven = not_given, |
| 877 | + ) -> WebExtractResponse: |
| 878 | + """ |
| 879 | + Crawl a website, convert pages to Markdown using the scrape cache, and extract |
| 880 | + structured data into the provided JSON Schema. The schema must describe the |
| 881 | + response data object. This endpoint does not accept targeted page-type |
| 882 | + selection. |
| 883 | +
|
| 884 | + Args: |
| 885 | + schema: JSON Schema for the returned data object. TypeScript Zod users can pass a JSON |
| 886 | + Schema generated from a Zod object; Python users can pass the equivalent JSON |
| 887 | + Schema object. |
| 888 | +
|
| 889 | + url: The starting website URL to crawl and extract from. Must include http:// or |
| 890 | + https://. |
| 891 | +
|
| 892 | + fact_check: When true (default), every returned value must be grounded in facts stated on |
| 893 | + the page; fields that cannot be supported by the page are returned as |
| 894 | + null/empty. When false, the model may make reasonable inferences and derivations |
| 895 | + from the page content (e.g. ideal customer, competitor analysis, |
| 896 | + recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, |
| 897 | + metrics) faithful to the source. |
| 898 | +
|
| 899 | + follow_subdomains: When true, follow links on subdomains of the starting URL's domain. |
| 900 | +
|
| 901 | + include_frames: When true, iframe contents are included in Markdown before extraction. |
| 902 | +
|
| 903 | + instructions: Optional extraction guidance, such as which facts to prioritize or how to |
| 904 | + interpret fields in the schema. |
| 905 | +
|
| 906 | + max_age_ms: Return cached scrape results if a prior scrape for the same parameters is |
| 907 | + younger than this many milliseconds. |
| 908 | +
|
| 909 | + stop_after_ms: Soft time budget for the crawl in milliseconds. |
| 910 | +
|
| 911 | + timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer |
| 912 | + than this value, it will be aborted with a 408 status code. Maximum allowed |
| 913 | + value is 300000ms (5 minutes). |
| 914 | +
|
| 915 | + wait_for_ms: Optional browser wait time in milliseconds after initial page load for each |
| 916 | + crawled page. |
| 917 | +
|
| 918 | + extra_headers: Send extra headers |
| 919 | +
|
| 920 | + extra_query: Add additional query parameters to the request |
| 921 | +
|
| 922 | + extra_body: Add additional JSON properties to the request |
| 923 | +
|
| 924 | + timeout: Override the client-level default timeout for this request, in seconds |
| 925 | + """ |
| 926 | + return await self._post( |
| 927 | + "/web/extract", |
| 928 | + body=await async_maybe_transform( |
| 929 | + { |
| 930 | + "schema": schema, |
| 931 | + "url": url, |
| 932 | + "fact_check": fact_check, |
| 933 | + "follow_subdomains": follow_subdomains, |
| 934 | + "include_frames": include_frames, |
| 935 | + "instructions": instructions, |
| 936 | + "max_age_ms": max_age_ms, |
| 937 | + "pdf": pdf, |
| 938 | + "stop_after_ms": stop_after_ms, |
| 939 | + "timeout_ms": timeout_ms, |
| 940 | + "wait_for_ms": wait_for_ms, |
| 941 | + }, |
| 942 | + web_extract_params.WebExtractParams, |
| 943 | + ), |
| 944 | + options=make_request_options( |
| 945 | + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout |
| 946 | + ), |
| 947 | + cast_to=WebExtractResponse, |
| 948 | + ) |
| 949 | + |
761 | 950 | async def extract_fonts( |
762 | 951 | self, |
763 | 952 | *, |
@@ -1439,6 +1628,9 @@ class WebResourceWithRawResponse: |
1439 | 1628 | def __init__(self, web: WebResource) -> None: |
1440 | 1629 | self._web = web |
1441 | 1630 |
|
| 1631 | + self.extract = to_raw_response_wrapper( |
| 1632 | + web.extract, |
| 1633 | + ) |
1442 | 1634 | self.extract_fonts = to_raw_response_wrapper( |
1443 | 1635 | web.extract_fonts, |
1444 | 1636 | ) |
@@ -1472,6 +1664,9 @@ class AsyncWebResourceWithRawResponse: |
1472 | 1664 | def __init__(self, web: AsyncWebResource) -> None: |
1473 | 1665 | self._web = web |
1474 | 1666 |
|
| 1667 | + self.extract = async_to_raw_response_wrapper( |
| 1668 | + web.extract, |
| 1669 | + ) |
1475 | 1670 | self.extract_fonts = async_to_raw_response_wrapper( |
1476 | 1671 | web.extract_fonts, |
1477 | 1672 | ) |
@@ -1505,6 +1700,9 @@ class WebResourceWithStreamingResponse: |
1505 | 1700 | def __init__(self, web: WebResource) -> None: |
1506 | 1701 | self._web = web |
1507 | 1702 |
|
| 1703 | + self.extract = to_streamed_response_wrapper( |
| 1704 | + web.extract, |
| 1705 | + ) |
1508 | 1706 | self.extract_fonts = to_streamed_response_wrapper( |
1509 | 1707 | web.extract_fonts, |
1510 | 1708 | ) |
@@ -1538,6 +1736,9 @@ class AsyncWebResourceWithStreamingResponse: |
1538 | 1736 | def __init__(self, web: AsyncWebResource) -> None: |
1539 | 1737 | self._web = web |
1540 | 1738 |
|
| 1739 | + self.extract = async_to_streamed_response_wrapper( |
| 1740 | + web.extract, |
| 1741 | + ) |
1541 | 1742 | self.extract_fonts = async_to_streamed_response_wrapper( |
1542 | 1743 | web.extract_fonts, |
1543 | 1744 | ) |
|
0 commit comments