Skip to content

Commit 7b4539c

Browse files
author
stlc-bot
committed
feat(web): add raw bytes scraping endpoint (#1081)
Stainless-Generated-From: a7d618c
1 parent 653bc8c commit 7b4539c

7 files changed

Lines changed: 1738 additions & 731 deletions

File tree

‎.stats.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
configured_endpoints: 48
1+
configured_endpoints: 49

‎api.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ from context.dev.types import (
2424
WebScreenshotResponse,
2525
WebSearchResponse,
2626
WebWebCrawlMdResponse,
27+
WebWebScrapeBytesResponse,
2728
WebWebScrapeHTMLResponse,
2829
WebWebScrapeImagesResponse,
2930
WebWebScrapeMdResponse,
@@ -41,6 +42,7 @@ Methods:
4142
- <code title="get /web/screenshot">client.web.<a href="./src/context/dev/resources/web.py">screenshot</a>(\*\*<a href="src/context/dev/types/web_screenshot_params.py">params</a>) -> <a href="./src/context/dev/types/web_screenshot_response.py">WebScreenshotResponse</a></code>
4243
- <code title="post /web/search">client.web.<a href="./src/context/dev/resources/web.py">search</a>(\*\*<a href="src/context/dev/types/web_search_params.py">params</a>) -> <a href="./src/context/dev/types/web_search_response.py">WebSearchResponse</a></code>
4344
- <code title="post /web/crawl">client.web.<a href="./src/context/dev/resources/web.py">web_crawl_md</a>(\*\*<a href="src/context/dev/types/web_web_crawl_md_params.py">params</a>) -> <a href="./src/context/dev/types/web_web_crawl_md_response.py">WebWebCrawlMdResponse</a></code>
45+
- <code title="get /web/scrape/bytes">client.web.<a href="./src/context/dev/resources/web.py">web_scrape_bytes</a>(\*\*<a href="src/context/dev/types/web_web_scrape_bytes_params.py">params</a>) -> <a href="./src/context/dev/types/web_web_scrape_bytes_response.py">WebWebScrapeBytesResponse</a></code>
4446
- <code title="get /web/scrape/html">client.web.<a href="./src/context/dev/resources/web.py">web_scrape_html</a>(\*\*<a href="src/context/dev/types/web_web_scrape_html_params.py">params</a>) -> <a href="./src/context/dev/types/web_web_scrape_html_response.py">WebWebScrapeHTMLResponse</a></code>
4547
- <code title="get /web/scrape/images">client.web.<a href="./src/context/dev/resources/web.py">web_scrape_images</a>(\*\*<a href="src/context/dev/types/web_web_scrape_images_params.py">params</a>) -> <a href="./src/context/dev/types/web_web_scrape_images_response.py">WebWebScrapeImagesResponse</a></code>
4648
- <code title="get /web/scrape/markdown">client.web.<a href="./src/context/dev/resources/web.py">web_scrape_md</a>(\*\*<a href="src/context/dev/types/web_web_scrape_md_params.py">params</a>) -> <a href="./src/context/dev/types/web_web_scrape_md_response.py">WebWebScrapeMdResponse</a></code>

‎src/context/dev/resources/web.py‎

Lines changed: 1322 additions & 730 deletions
Large diffs are not rendered by default.

‎src/context/dev/types/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,14 @@
6464
from .ai_extract_product_response import AIExtractProductResponse as AIExtractProductResponse
6565
from .monitor_get_limits_response import MonitorGetLimitsResponse as MonitorGetLimitsResponse
6666
from .monitor_list_changes_params import MonitorListChangesParams as MonitorListChangesParams
67+
from .web_web_scrape_bytes_params import WebWebScrapeBytesParams as WebWebScrapeBytesParams
6768
from .ai_extract_products_response import AIExtractProductsResponse as AIExtractProductsResponse
6869
from .industry_retrieve_sic_params import IndustryRetrieveSicParams as IndustryRetrieveSicParams
6970
from .web_web_scrape_html_response import WebWebScrapeHTMLResponse as WebWebScrapeHTMLResponse
7071
from .web_web_scrape_images_params import WebWebScrapeImagesParams as WebWebScrapeImagesParams
7172
from .monitor_list_changes_response import MonitorListChangesResponse as MonitorListChangesResponse
7273
from .web_extract_styleguide_params import WebExtractStyleguideParams as WebExtractStyleguideParams
74+
from .web_web_scrape_bytes_response import WebWebScrapeBytesResponse as WebWebScrapeBytesResponse
7375
from .web_web_scrape_sitemap_params import WebWebScrapeSitemapParams as WebWebScrapeSitemapParams
7476
from .industry_retrieve_naics_params import IndustryRetrieveNaicsParams as IndustryRetrieveNaicsParams
7577
from .industry_retrieve_sic_response import IndustryRetrieveSicResponse as IndustryRetrieveSicResponse

‎src/context/dev/types/web_web_scrape_bytes_params.py‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2+
3+
from __future__ import annotations
4+
5+
from typing import Dict
6+
from typing_extensions import Literal, Required, Annotated, TypedDict
7+
8+
from .._types import SequenceNotStr
9+
from .._utils import PropertyInfo
10+
11+
__all__ = ["WebWebScrapeBytesParams"]
12+
13+
14+
class WebWebScrapeBytesParams(TypedDict, total=False):
15+
url: Required[str]
16+
"""Full HTTP(S) URL of the resource to download, such as an image, PDF, or page."""
17+
18+
country: Literal[
19+
"ad",
20+
"ae",
21+
"af",
22+
"ag",
23+
"ai",
24+
"al",
25+
"am",
26+
"ao",
27+
"ar",
28+
"at",
29+
"au",
30+
"aw",
31+
"az",
32+
"ba",
33+
"bb",
34+
"bd",
35+
"be",
36+
"bf",
37+
"bg",
38+
"bh",
39+
"bi",
40+
"bj",
41+
"bm",
42+
"bn",
43+
"bo",
44+
"bq",
45+
"br",
46+
"bs",
47+
"bw",
48+
"by",
49+
"bz",
50+
"ca",
51+
"cd",
52+
"cf",
53+
"cg",
54+
"ch",
55+
"ci",
56+
"cl",
57+
"cm",
58+
"cn",
59+
"co",
60+
"cr",
61+
"cv",
62+
"cw",
63+
"cy",
64+
"cz",
65+
"de",
66+
"dj",
67+
"dk",
68+
"dm",
69+
"do",
70+
"dz",
71+
"ec",
72+
"ee",
73+
"eg",
74+
"es",
75+
"et",
76+
"fi",
77+
"fj",
78+
"fr",
79+
"ga",
80+
"gb",
81+
"gd",
82+
"ge",
83+
"gf",
84+
"gg",
85+
"gh",
86+
"gm",
87+
"gn",
88+
"gp",
89+
"gq",
90+
"gr",
91+
"gt",
92+
"gu",
93+
"gw",
94+
"gy",
95+
"hk",
96+
"hn",
97+
"hr",
98+
"ht",
99+
"hu",
100+
"id",
101+
"ie",
102+
"il",
103+
"im",
104+
"in",
105+
"iq",
106+
"ir",
107+
"is",
108+
"it",
109+
"je",
110+
"jm",
111+
"jo",
112+
"jp",
113+
"ke",
114+
"kg",
115+
"kh",
116+
"kn",
117+
"kr",
118+
"kw",
119+
"ky",
120+
"kz",
121+
"la",
122+
"lb",
123+
"lc",
124+
"lk",
125+
"lr",
126+
"ls",
127+
"lt",
128+
"lu",
129+
"lv",
130+
"ly",
131+
"ma",
132+
"mc",
133+
"md",
134+
"me",
135+
"mf",
136+
"mg",
137+
"mk",
138+
"ml",
139+
"mm",
140+
"mn",
141+
"mo",
142+
"mq",
143+
"mr",
144+
"mt",
145+
"mu",
146+
"mv",
147+
"mw",
148+
"mx",
149+
"my",
150+
"mz",
151+
"na",
152+
"nc",
153+
"ne",
154+
"ng",
155+
"ni",
156+
"nl",
157+
"no",
158+
"np",
159+
"nz",
160+
"om",
161+
"pa",
162+
"pe",
163+
"pf",
164+
"pg",
165+
"ph",
166+
"pk",
167+
"pl",
168+
"pr",
169+
"ps",
170+
"pt",
171+
"py",
172+
"qa",
173+
"re",
174+
"ro",
175+
"rs",
176+
"ru",
177+
"rw",
178+
"sa",
179+
"sc",
180+
"sd",
181+
"se",
182+
"sg",
183+
"si",
184+
"sk",
185+
"sl",
186+
"sm",
187+
"sn",
188+
"so",
189+
"sr",
190+
"ss",
191+
"st",
192+
"sv",
193+
"sx",
194+
"sy",
195+
"sz",
196+
"tc",
197+
"td",
198+
"tg",
199+
"th",
200+
"tj",
201+
"tl",
202+
"tm",
203+
"tn",
204+
"tr",
205+
"tt",
206+
"tw",
207+
"tz",
208+
"ua",
209+
"ug",
210+
"us",
211+
"uy",
212+
"uz",
213+
"vc",
214+
"ve",
215+
"vg",
216+
"vi",
217+
"vn",
218+
"ye",
219+
"yt",
220+
"za",
221+
"zm",
222+
"zw",
223+
]
224+
"""
225+
Fetch the target page through a residential proxy in this country (ISO 3166-1
226+
alpha-2).
227+
"""
228+
229+
headers: Dict[str, str]
230+
"""Optional outbound HTTP headers, such as Referer, Cookie, or Authorization.
231+
232+
Send as a JSON object or deep-object query params such as
233+
headers[Referer]=https://example.com/. Host, Content-Length, and hop-by-hop
234+
transport headers are rejected. Authorization and cookies are removed when a
235+
redirect changes origin.
236+
"""
237+
238+
tags: SequenceNotStr[str]
239+
"""Comma-separated tags for tracking request usage.
240+
241+
Up to 20 tags, each 1-50 characters.
242+
"""
243+
244+
timeout_ms: Annotated[int, PropertyInfo(alias="timeoutMS")]
245+
"""Optional timeout in milliseconds for the request.
246+
247+
If the request takes longer than this value, it will be aborted with a 408
248+
status code. Maximum allowed value is 300000ms (5 minutes).
249+
"""
250+
251+
zdr: Literal["enabled", "disabled"]
252+
"""
253+
Set to enabled to bypass shared caches and omit request and response content
254+
from retained usage logs. Requires zero data retention to be enabled for your
255+
organization (contact support@context.dev), otherwise the request fails with
256+
ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.
257+
"""

‎src/context/dev/types/web_web_scrape_bytes_response.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2+
3+
from typing import Optional
4+
from typing_extensions import Literal
5+
6+
from pydantic import Field as FieldInfo
7+
8+
from .._models import BaseModel
9+
10+
__all__ = ["WebWebScrapeBytesResponse", "KeyMetadata"]
11+
12+
13+
class KeyMetadata(BaseModel):
14+
"""Credit usage, included whenever a valid API key is provided."""
15+
16+
credits_consumed: int
17+
"""Credits used by this request."""
18+
19+
credits_remaining: int
20+
"""Credits remaining for your organization."""
21+
22+
23+
class WebWebScrapeBytesResponse(BaseModel):
24+
bytes: str
25+
"""Base64-encoded resource bytes, without a data URI prefix.
26+
27+
Decode this field to recover the downloaded file.
28+
"""
29+
30+
content_length: int = FieldInfo(alias="contentLength")
31+
"""Number of decoded resource bytes, before base64 encoding."""
32+
33+
content_type: str = FieldInfo(alias="contentType")
34+
"""The Content-Type returned by the origin, including any charset.
35+
36+
Defaults to application/octet-stream when absent.
37+
"""
38+
39+
encoding: Literal["base64"]
40+
41+
final_url: str = FieldInfo(alias="finalUrl")
42+
"""The resource URL after redirects."""
43+
44+
request_id: str
45+
"""Unique id of this API call, also sent in the X-Request-Id response header.
46+
47+
Quote it when contacting support about a failed request.
48+
"""
49+
50+
status_code: int = FieldInfo(alias="statusCode")
51+
"""HTTP status returned by the origin."""
52+
53+
success: Literal[True]
54+
55+
url: str
56+
"""The requested resource URL."""
57+
58+
key_metadata: Optional[KeyMetadata] = None
59+
"""Credit usage, included whenever a valid API key is provided."""

0 commit comments

Comments
 (0)