diff --git a/.release-please-manifest.json b/.release-please-manifest.json index bfc26f9..75ec52f 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.2.0" + ".": "2.3.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index c2cc39a..1ba5348 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 29 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-ecb277557634e2f0838ea196e2b21f11924f65e133c7977e8c2e1a603b8df466.yml -openapi_spec_hash: 798ce4cb042389a1179bf9af6fc1173d -config_hash: ff35e224e809656528c44163aa41bebd +configured_endpoints: 30 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-c90f13e3cf4b9ae6a1d8f22c0e7285ab12d412d3a659a51a247a97adc7f08121.yml +openapi_spec_hash: bf7879be38ebf3397a939c80b261a47a +config_hash: daabb160675d86b354711da1e77e5129 diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ef7ce..70da1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 2.3.0 (2026-07-12) + +Full Changelog: [v2.2.0...v2.3.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.2.0...v2.3.0) + +### Features + +* **api:** api update ([a3f2a36](https://github.com/context-dot-dev/context-go-sdk/commit/a3f2a36a8eec95f86c2413a306163117e81a5aef)) +* **api:** api update ([a329e8f](https://github.com/context-dot-dev/context-go-sdk/commit/a329e8fd82cb7daa96a57d34c64c0fdc173fcfe6)) +* **api:** api update ([94c8657](https://github.com/context-dot-dev/context-go-sdk/commit/94c865717ba0322cfb6978f30451cedbc83024d3)) +* **api:** api update ([a689b94](https://github.com/context-dot-dev/context-go-sdk/commit/a689b94cc1913c2521c6e8932cfe4cb35e07fff1)) +* **api:** api update ([28e740f](https://github.com/context-dot-dev/context-go-sdk/commit/28e740f468b39810f2f5ca477e67bbcab358ae11)) +* **api:** api update ([cea1ee4](https://github.com/context-dot-dev/context-go-sdk/commit/cea1ee4604ca3c5510d57f47afddc0c2f04a13bd)) +* **api:** manual updates ([8d816b0](https://github.com/context-dot-dev/context-go-sdk/commit/8d816b0540dde7b346e76870be8e2a5b68be3b2b)) + ## 2.2.0 (2026-07-10) Full Changelog: [v2.1.0...v2.2.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.1.0...v2.2.0) diff --git a/README.md b/README.md index b76c087..7d81aba 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Or to pin the version: ```sh -go get -u 'github.com/context-dot-dev/context-go-sdk@v2.2.0' +go get -u 'github.com/context-dot-dev/context-go-sdk@v2.3.0' ``` diff --git a/api.md b/api.md index 79da99c..e3e1ca1 100644 --- a/api.md +++ b/api.md @@ -1,3 +1,13 @@ +# Parse + +Response Types: + +- contextdev.ParseHandleResponse + +Methods: + +- client.Parse.Handle(ctx context.Context, body io.Reader, params contextdev.ParseHandleParams) (\*contextdev.ParseHandleResponse, error) + # Web Response Types: @@ -78,6 +88,7 @@ Methods: Response Types: +- contextdev.WebhookDelivery - contextdev.MonitorNewResponse - contextdev.MonitorGetResponse - contextdev.MonitorUpdateResponse diff --git a/client.go b/client.go index 5962fbc..edee3e9 100644 --- a/client.go +++ b/client.go @@ -18,14 +18,16 @@ import ( // directly, and instead use the [NewClient] method instead. type Client struct { options []option.RequestOption + Parse ParseService Web WebService AI AIService Brand BrandService Industry IndustryService Utility UtilityService // Monitor pages, sitemaps, and extracted website data for exact or semantic - // changes. The change.detected webhook payload is documented by the - // MonitorsChangeDetectedWebhookPayload schema. + // changes. Webhook payloads are documented by the + // MonitorsChangeDetectedWebhookPayload and MonitorsRunCompletedWebhookPayload + // schemas. Monitors MonitorService } @@ -59,6 +61,7 @@ func NewClient(opts ...option.RequestOption) (r Client) { r = Client{options: opts} + r.Parse = NewParseService(opts...) r.Web = NewWebService(opts...) r.AI = NewAIService(opts...) r.Brand = NewBrandService(opts...) diff --git a/internal/version.go b/internal/version.go index 0759deb..ebb76be 100644 --- a/internal/version.go +++ b/internal/version.go @@ -2,4 +2,4 @@ package internal -const PackageVersion = "2.2.0" // x-release-please-version +const PackageVersion = "2.3.0" // x-release-please-version diff --git a/monitor.go b/monitor.go index db6c705..2a06cee 100644 --- a/monitor.go +++ b/monitor.go @@ -22,8 +22,9 @@ import ( ) // Monitor pages, sitemaps, and extracted website data for exact or semantic -// changes. The change.detected webhook payload is documented by the -// MonitorsChangeDetectedWebhookPayload schema. +// changes. Webhook payloads are documented by the +// MonitorsChangeDetectedWebhookPayload and MonitorsRunCompletedWebhookPayload +// schemas. // // MonitorService contains methods and other services that help with interacting // with the context.dev API. @@ -167,6 +168,83 @@ func (r *MonitorService) Run(ctx context.Context, monitorID string, opts ...opti return res, err } +type WebhookDelivery struct { + AttemptedAt time.Time `json:"attempted_at" api:"required" format:"date-time"` + Error WebhookDeliveryError `json:"error" api:"required"` + // The event this delivery carried. Deliveries recorded before event selection + // existed report change.detected. + // + // Any of "change.detected", "run.completed". + Event WebhookDeliveryEvent `json:"event" api:"required"` + // Identifier sent in the X-Context-Id header. + EventID string `json:"event_id" api:"required"` + // The endpoint's final HTTP response status, or null when no response was + // received. + HTTPStatus int64 `json:"http_status" api:"required"` + // Delivery outcome. delivered means any 2xx response; rejected means a non-2xx + // response; failed means no HTTP response was received; skipped_unsafe_url means + // the URL failed the public-endpoint safety check. + // + // Any of "delivered", "rejected", "failed", "skipped_unsafe_url". + Status WebhookDeliveryStatus `json:"status" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + AttemptedAt respjson.Field + Error respjson.Field + Event respjson.Field + EventID respjson.Field + HTTPStatus respjson.Field + Status respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r WebhookDelivery) RawJSON() string { return r.JSON.raw } +func (r *WebhookDelivery) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type WebhookDeliveryError struct { + Code string `json:"code" api:"required"` + Message string `json:"message" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Code respjson.Field + Message respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r WebhookDeliveryError) RawJSON() string { return r.JSON.raw } +func (r *WebhookDeliveryError) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// The event this delivery carried. Deliveries recorded before event selection +// existed report change.detected. +type WebhookDeliveryEvent string + +const ( + WebhookDeliveryEventChangeDetected WebhookDeliveryEvent = "change.detected" + WebhookDeliveryEventRunCompleted WebhookDeliveryEvent = "run.completed" +) + +// Delivery outcome. delivered means any 2xx response; rejected means a non-2xx +// response; failed means no HTTP response was received; skipped_unsafe_url means +// the URL failed the public-endpoint safety check. +type WebhookDeliveryStatus string + +const ( + WebhookDeliveryStatusDelivered WebhookDeliveryStatus = "delivered" + WebhookDeliveryStatusRejected WebhookDeliveryStatus = "rejected" + WebhookDeliveryStatusFailed WebhookDeliveryStatus = "failed" + WebhookDeliveryStatusSkippedUnsafeURL WebhookDeliveryStatus = "skipped_unsafe_url" +) + // A web monitor. `mode` is the constant `web`; behavior is described by `target` // (page/sitemap/extract) and `change_detection` (exact/semantic). type MonitorNewResponse struct { @@ -209,6 +287,10 @@ type MonitorNewResponse struct { // User-defined tags for grouping and filtering monitors and their changes. Tags []string `json:"tags"` Webhook MonitorNewResponseWebhook `json:"webhook" api:"nullable"` + // Present while webhook deliveries are failing consecutively; null when deliveries + // are healthy or no webhook is configured. Cleared on the next successful delivery + // and when the webhook URL changes. + WebhookFailure MonitorNewResponseWebhookFailure `json:"webhook_failure" api:"nullable"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field @@ -227,6 +309,7 @@ type MonitorNewResponse struct { NextRunAt respjson.Field Tags respjson.Field Webhook respjson.Field + WebhookFailure respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -736,8 +819,15 @@ func (r *MonitorNewResponseLastError) UnmarshalJSON(data []byte) error { } type MonitorNewResponseWebhook struct { - // Webhook URL called when a change is detected. + // Webhook URL events are delivered to. URL string `json:"url" api:"required" format:"uri"` + // Events delivered to this endpoint. `change.detected` fires only when a run + // detects a change; `run.completed` fires on every completed run — including runs + // that detected no change — and embeds the change when one was detected. Defaults + // to `["change.detected"]` when omitted. + // + // Any of "change.detected", "run.completed". + Events []string `json:"events"` // Signing secret used to verify webhook authenticity. Each delivery includes an // `X-Context-Signature: t=,v1=` header, where the HMAC is SHA-256 over // `"{t}.{rawRequestBody}"` keyed by this secret. Recompute it with a constant-time @@ -747,6 +837,7 @@ type MonitorNewResponseWebhook struct { // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { URL respjson.Field + Events respjson.Field Secret respjson.Field ExtraFields map[string]respjson.Field raw string @@ -759,6 +850,38 @@ func (r *MonitorNewResponseWebhook) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } +// Present while webhook deliveries are failing consecutively; null when deliveries +// are healthy or no webhook is configured. Cleared on the next successful delivery +// and when the webhook URL changes. +type MonitorNewResponseWebhookFailure struct { + // Number of consecutive delivery attempts that did not succeed. + ConsecutiveFailures int64 `json:"consecutive_failures" api:"required"` + LastFailedAt time.Time `json:"last_failed_at" api:"required" format:"date-time"` + // Human-readable description of the most recent failure. + LastMessage string `json:"last_message" api:"required"` + // Outcome of the most recent failed delivery. rejected means a non-2xx response; + // failed means no HTTP response was received; skipped_unsafe_url means the URL + // failed the public-endpoint safety check. + // + // Any of "rejected", "failed", "skipped_unsafe_url". + LastStatus string `json:"last_status" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + ConsecutiveFailures respjson.Field + LastFailedAt respjson.Field + LastMessage respjson.Field + LastStatus respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r MonitorNewResponseWebhookFailure) RawJSON() string { return r.JSON.raw } +func (r *MonitorNewResponseWebhookFailure) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + // A web monitor. `mode` is the constant `web`; behavior is described by `target` // (page/sitemap/extract) and `change_detection` (exact/semantic). type MonitorGetResponse struct { @@ -801,6 +924,10 @@ type MonitorGetResponse struct { // User-defined tags for grouping and filtering monitors and their changes. Tags []string `json:"tags"` Webhook MonitorGetResponseWebhook `json:"webhook" api:"nullable"` + // Present while webhook deliveries are failing consecutively; null when deliveries + // are healthy or no webhook is configured. Cleared on the next successful delivery + // and when the webhook URL changes. + WebhookFailure MonitorGetResponseWebhookFailure `json:"webhook_failure" api:"nullable"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field @@ -819,6 +946,7 @@ type MonitorGetResponse struct { NextRunAt respjson.Field Tags respjson.Field Webhook respjson.Field + WebhookFailure respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -1328,8 +1456,15 @@ func (r *MonitorGetResponseLastError) UnmarshalJSON(data []byte) error { } type MonitorGetResponseWebhook struct { - // Webhook URL called when a change is detected. + // Webhook URL events are delivered to. URL string `json:"url" api:"required" format:"uri"` + // Events delivered to this endpoint. `change.detected` fires only when a run + // detects a change; `run.completed` fires on every completed run — including runs + // that detected no change — and embeds the change when one was detected. Defaults + // to `["change.detected"]` when omitted. + // + // Any of "change.detected", "run.completed". + Events []string `json:"events"` // Signing secret used to verify webhook authenticity. Each delivery includes an // `X-Context-Signature: t=,v1=` header, where the HMAC is SHA-256 over // `"{t}.{rawRequestBody}"` keyed by this secret. Recompute it with a constant-time @@ -1339,6 +1474,7 @@ type MonitorGetResponseWebhook struct { // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { URL respjson.Field + Events respjson.Field Secret respjson.Field ExtraFields map[string]respjson.Field raw string @@ -1351,6 +1487,38 @@ func (r *MonitorGetResponseWebhook) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } +// Present while webhook deliveries are failing consecutively; null when deliveries +// are healthy or no webhook is configured. Cleared on the next successful delivery +// and when the webhook URL changes. +type MonitorGetResponseWebhookFailure struct { + // Number of consecutive delivery attempts that did not succeed. + ConsecutiveFailures int64 `json:"consecutive_failures" api:"required"` + LastFailedAt time.Time `json:"last_failed_at" api:"required" format:"date-time"` + // Human-readable description of the most recent failure. + LastMessage string `json:"last_message" api:"required"` + // Outcome of the most recent failed delivery. rejected means a non-2xx response; + // failed means no HTTP response was received; skipped_unsafe_url means the URL + // failed the public-endpoint safety check. + // + // Any of "rejected", "failed", "skipped_unsafe_url". + LastStatus string `json:"last_status" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + ConsecutiveFailures respjson.Field + LastFailedAt respjson.Field + LastMessage respjson.Field + LastStatus respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r MonitorGetResponseWebhookFailure) RawJSON() string { return r.JSON.raw } +func (r *MonitorGetResponseWebhookFailure) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + // A web monitor. `mode` is the constant `web`; behavior is described by `target` // (page/sitemap/extract) and `change_detection` (exact/semantic). type MonitorUpdateResponse struct { @@ -1393,6 +1561,10 @@ type MonitorUpdateResponse struct { // User-defined tags for grouping and filtering monitors and their changes. Tags []string `json:"tags"` Webhook MonitorUpdateResponseWebhook `json:"webhook" api:"nullable"` + // Present while webhook deliveries are failing consecutively; null when deliveries + // are healthy or no webhook is configured. Cleared on the next successful delivery + // and when the webhook URL changes. + WebhookFailure MonitorUpdateResponseWebhookFailure `json:"webhook_failure" api:"nullable"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field @@ -1411,6 +1583,7 @@ type MonitorUpdateResponse struct { NextRunAt respjson.Field Tags respjson.Field Webhook respjson.Field + WebhookFailure respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -1921,8 +2094,15 @@ func (r *MonitorUpdateResponseLastError) UnmarshalJSON(data []byte) error { } type MonitorUpdateResponseWebhook struct { - // Webhook URL called when a change is detected. + // Webhook URL events are delivered to. URL string `json:"url" api:"required" format:"uri"` + // Events delivered to this endpoint. `change.detected` fires only when a run + // detects a change; `run.completed` fires on every completed run — including runs + // that detected no change — and embeds the change when one was detected. Defaults + // to `["change.detected"]` when omitted. + // + // Any of "change.detected", "run.completed". + Events []string `json:"events"` // Signing secret used to verify webhook authenticity. Each delivery includes an // `X-Context-Signature: t=,v1=` header, where the HMAC is SHA-256 over // `"{t}.{rawRequestBody}"` keyed by this secret. Recompute it with a constant-time @@ -1932,6 +2112,7 @@ type MonitorUpdateResponseWebhook struct { // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { URL respjson.Field + Events respjson.Field Secret respjson.Field ExtraFields map[string]respjson.Field raw string @@ -1944,6 +2125,38 @@ func (r *MonitorUpdateResponseWebhook) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } +// Present while webhook deliveries are failing consecutively; null when deliveries +// are healthy or no webhook is configured. Cleared on the next successful delivery +// and when the webhook URL changes. +type MonitorUpdateResponseWebhookFailure struct { + // Number of consecutive delivery attempts that did not succeed. + ConsecutiveFailures int64 `json:"consecutive_failures" api:"required"` + LastFailedAt time.Time `json:"last_failed_at" api:"required" format:"date-time"` + // Human-readable description of the most recent failure. + LastMessage string `json:"last_message" api:"required"` + // Outcome of the most recent failed delivery. rejected means a non-2xx response; + // failed means no HTTP response was received; skipped_unsafe_url means the URL + // failed the public-endpoint safety check. + // + // Any of "rejected", "failed", "skipped_unsafe_url". + LastStatus string `json:"last_status" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + ConsecutiveFailures respjson.Field + LastFailedAt respjson.Field + LastMessage respjson.Field + LastStatus respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r MonitorUpdateResponseWebhookFailure) RawJSON() string { return r.JSON.raw } +func (r *MonitorUpdateResponseWebhookFailure) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + type MonitorListResponse struct { Data []MonitorListResponseData `json:"data" api:"required"` HasMore bool `json:"has_more" api:"required"` @@ -2006,6 +2219,10 @@ type MonitorListResponseData struct { // User-defined tags for grouping and filtering monitors and their changes. Tags []string `json:"tags"` Webhook MonitorListResponseDataWebhook `json:"webhook" api:"nullable"` + // Present while webhook deliveries are failing consecutively; null when deliveries + // are healthy or no webhook is configured. Cleared on the next successful delivery + // and when the webhook URL changes. + WebhookFailure MonitorListResponseDataWebhookFailure `json:"webhook_failure" api:"nullable"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field @@ -2024,6 +2241,7 @@ type MonitorListResponseData struct { NextRunAt respjson.Field Tags respjson.Field Webhook respjson.Field + WebhookFailure respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -2515,8 +2733,15 @@ func (r *MonitorListResponseDataLastError) UnmarshalJSON(data []byte) error { } type MonitorListResponseDataWebhook struct { - // Webhook URL called when a change is detected. + // Webhook URL events are delivered to. URL string `json:"url" api:"required" format:"uri"` + // Events delivered to this endpoint. `change.detected` fires only when a run + // detects a change; `run.completed` fires on every completed run — including runs + // that detected no change — and embeds the change when one was detected. Defaults + // to `["change.detected"]` when omitted. + // + // Any of "change.detected", "run.completed". + Events []string `json:"events"` // Signing secret used to verify webhook authenticity. Each delivery includes an // `X-Context-Signature: t=,v1=` header, where the HMAC is SHA-256 over // `"{t}.{rawRequestBody}"` keyed by this secret. Recompute it with a constant-time @@ -2526,6 +2751,7 @@ type MonitorListResponseDataWebhook struct { // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { URL respjson.Field + Events respjson.Field Secret respjson.Field ExtraFields map[string]respjson.Field raw string @@ -2538,6 +2764,38 @@ func (r *MonitorListResponseDataWebhook) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } +// Present while webhook deliveries are failing consecutively; null when deliveries +// are healthy or no webhook is configured. Cleared on the next successful delivery +// and when the webhook URL changes. +type MonitorListResponseDataWebhookFailure struct { + // Number of consecutive delivery attempts that did not succeed. + ConsecutiveFailures int64 `json:"consecutive_failures" api:"required"` + LastFailedAt time.Time `json:"last_failed_at" api:"required" format:"date-time"` + // Human-readable description of the most recent failure. + LastMessage string `json:"last_message" api:"required"` + // Outcome of the most recent failed delivery. rejected means a non-2xx response; + // failed means no HTTP response was received; skipped_unsafe_url means the URL + // failed the public-endpoint safety check. + // + // Any of "rejected", "failed", "skipped_unsafe_url". + LastStatus string `json:"last_status" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + ConsecutiveFailures respjson.Field + LastFailedAt respjson.Field + LastMessage respjson.Field + LastStatus respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r MonitorListResponseDataWebhookFailure) RawJSON() string { return r.JSON.raw } +func (r *MonitorListResponseDataWebhookFailure) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + type MonitorDeleteResponse struct { ID string `json:"id" api:"required"` Deleted bool `json:"deleted" api:"required"` @@ -2683,10 +2941,16 @@ type MonitorListAccountRunsResponseData struct { // Any of "insufficient_credits", "monitor_paused", "superseded". SkipReason string `json:"skip_reason" api:"nullable"` StartedAt time.Time `json:"started_at" api:"nullable" format:"date-time"` - // The webhook delivery attempted for a change detected by this run. Omitted when - // no webhook was attempted, including historical runs created before delivery - // tracking was added. - WebhookDelivery MonitorListAccountRunsResponseDataWebhookDelivery `json:"webhook_delivery"` + // All webhook deliveries attempted by this run — one per subscribed event that + // fired. Omitted when no webhook was attempted, including runs created before + // event selection was added. + WebhookDeliveries []WebhookDelivery `json:"webhook_deliveries"` + // Deprecated: use `webhook_deliveries`, which records every attempt now that a run + // can deliver multiple events. Omitted when no webhook was attempted, including + // historical runs created before delivery tracking was added. + // + // Deprecated: deprecated + WebhookDelivery WebhookDelivery `json:"webhook_delivery"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field @@ -2703,6 +2967,7 @@ type MonitorListAccountRunsResponseData struct { Error respjson.Field SkipReason respjson.Field StartedAt respjson.Field + WebhookDeliveries respjson.Field WebhookDelivery respjson.Field ExtraFields map[string]respjson.Field raw string @@ -2733,59 +2998,6 @@ func (r *MonitorListAccountRunsResponseDataError) UnmarshalJSON(data []byte) err return apijson.UnmarshalRoot(data, r) } -// The webhook delivery attempted for a change detected by this run. Omitted when -// no webhook was attempted, including historical runs created before delivery -// tracking was added. -type MonitorListAccountRunsResponseDataWebhookDelivery struct { - AttemptedAt time.Time `json:"attempted_at" api:"required" format:"date-time"` - Error MonitorListAccountRunsResponseDataWebhookDeliveryError `json:"error" api:"required"` - // Identifier sent in the X-Context-Id header. - EventID string `json:"event_id" api:"required"` - // The endpoint's final HTTP response status, or null when no response was - // received. - HTTPStatus int64 `json:"http_status" api:"required"` - // Delivery outcome. delivered means any 2xx response; rejected means a non-2xx - // response; failed means no HTTP response was received; skipped_unsafe_url means - // the URL failed the public-endpoint safety check. - // - // Any of "delivered", "rejected", "failed", "skipped_unsafe_url". - Status string `json:"status" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - AttemptedAt respjson.Field - Error respjson.Field - EventID respjson.Field - HTTPStatus respjson.Field - Status respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r MonitorListAccountRunsResponseDataWebhookDelivery) RawJSON() string { return r.JSON.raw } -func (r *MonitorListAccountRunsResponseDataWebhookDelivery) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -type MonitorListAccountRunsResponseDataWebhookDeliveryError struct { - Code string `json:"code" api:"required"` - Message string `json:"message" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Code respjson.Field - Message respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r MonitorListAccountRunsResponseDataWebhookDeliveryError) RawJSON() string { return r.JSON.raw } -func (r *MonitorListAccountRunsResponseDataWebhookDeliveryError) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - type MonitorListChangesResponse struct { Data []MonitorListChangesResponseData `json:"data" api:"required"` HasMore bool `json:"has_more" api:"required"` @@ -2913,10 +3125,16 @@ type MonitorListRunsResponseData struct { // Any of "insufficient_credits", "monitor_paused", "superseded". SkipReason string `json:"skip_reason" api:"nullable"` StartedAt time.Time `json:"started_at" api:"nullable" format:"date-time"` - // The webhook delivery attempted for a change detected by this run. Omitted when - // no webhook was attempted, including historical runs created before delivery - // tracking was added. - WebhookDelivery MonitorListRunsResponseDataWebhookDelivery `json:"webhook_delivery"` + // All webhook deliveries attempted by this run — one per subscribed event that + // fired. Omitted when no webhook was attempted, including runs created before + // event selection was added. + WebhookDeliveries []WebhookDelivery `json:"webhook_deliveries"` + // Deprecated: use `webhook_deliveries`, which records every attempt now that a run + // can deliver multiple events. Omitted when no webhook was attempted, including + // historical runs created before delivery tracking was added. + // + // Deprecated: deprecated + WebhookDelivery WebhookDelivery `json:"webhook_delivery"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field @@ -2933,6 +3151,7 @@ type MonitorListRunsResponseData struct { Error respjson.Field SkipReason respjson.Field StartedAt respjson.Field + WebhookDeliveries respjson.Field WebhookDelivery respjson.Field ExtraFields map[string]respjson.Field raw string @@ -2963,64 +3182,11 @@ func (r *MonitorListRunsResponseDataError) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// The webhook delivery attempted for a change detected by this run. Omitted when -// no webhook was attempted, including historical runs created before delivery -// tracking was added. -type MonitorListRunsResponseDataWebhookDelivery struct { - AttemptedAt time.Time `json:"attempted_at" api:"required" format:"date-time"` - Error MonitorListRunsResponseDataWebhookDeliveryError `json:"error" api:"required"` - // Identifier sent in the X-Context-Id header. - EventID string `json:"event_id" api:"required"` - // The endpoint's final HTTP response status, or null when no response was - // received. - HTTPStatus int64 `json:"http_status" api:"required"` - // Delivery outcome. delivered means any 2xx response; rejected means a non-2xx - // response; failed means no HTTP response was received; skipped_unsafe_url means - // the URL failed the public-endpoint safety check. - // - // Any of "delivered", "rejected", "failed", "skipped_unsafe_url". - Status string `json:"status" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - AttemptedAt respjson.Field - Error respjson.Field - EventID respjson.Field - HTTPStatus respjson.Field - Status respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r MonitorListRunsResponseDataWebhookDelivery) RawJSON() string { return r.JSON.raw } -func (r *MonitorListRunsResponseDataWebhookDelivery) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -type MonitorListRunsResponseDataWebhookDeliveryError struct { - Code string `json:"code" api:"required"` - Message string `json:"message" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Code respjson.Field - Message respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r MonitorListRunsResponseDataWebhookDeliveryError) RawJSON() string { return r.JSON.raw } -func (r *MonitorListRunsResponseDataWebhookDeliveryError) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - // A detected change. `mode` is the constant `web`; `target_type` and // `change_detection_type` describe the change, and which optional fields are // present depends on them (page: `diff` + excerpts; sitemap: // `added_urls`/`removed_urls`; semantic: -// `query`/`confidence`/`importance`/`evidence`/`matched_urls`). +// `confidence`/`importance`/`evidence`/`matched_urls`). type MonitorGetChangeResponse struct { ID string `json:"id" api:"required"` // Any of "exact", "semantic". @@ -3035,6 +3201,8 @@ type MonitorGetChangeResponse struct { // The run that detected this change. RunID string `json:"run_id" api:"required"` Summary string `json:"summary" api:"required"` + // User-defined tags for grouping and filtering monitors and their changes. + Tags []string `json:"tags" api:"required"` // Any of "page", "sitemap", "extract". TargetType MonitorGetChangeResponseTargetType `json:"target_type" api:"required"` Title string `json:"title" api:"required"` @@ -3056,8 +3224,6 @@ type MonitorGetChangeResponse struct { RemovedURLCount int64 `json:"removed_url_count"` // At most 500 URLs are included; the corresponding count field is always exact. RemovedURLs []string `json:"removed_urls" format:"uri"` - // User-defined tags for grouping and filtering monitors and their changes. - Tags []string `json:"tags"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field @@ -3067,6 +3233,7 @@ type MonitorGetChangeResponse struct { MonitorID respjson.Field RunID respjson.Field Summary respjson.Field + Tags respjson.Field TargetType respjson.Field Title respjson.Field URL respjson.Field @@ -3082,7 +3249,6 @@ type MonitorGetChangeResponse struct { MatchedURLs respjson.Field RemovedURLCount respjson.Field RemovedURLs respjson.Field - Tags respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -3426,8 +3592,15 @@ const ( // The property URL is required. type MonitorNewParamsWebhook struct { - // Webhook URL called when a change is detected. + // Webhook URL events are delivered to. URL string `json:"url" api:"required" format:"uri"` + // Events delivered to this endpoint. `change.detected` fires only when a run + // detects a change; `run.completed` fires on every completed run — including runs + // that detected no change — and embeds the change when one was detected. Defaults + // to `["change.detected"]` when omitted. + // + // Any of "change.detected", "run.completed". + Events []string `json:"events,omitzero"` paramObj } @@ -3694,8 +3867,15 @@ func (r *MonitorUpdateParamsTargetExtract) UnmarshalJSON(data []byte) error { // // The property URL is required. type MonitorUpdateParamsWebhook struct { - // Webhook URL called when a change is detected. + // Webhook URL events are delivered to. URL string `json:"url" api:"required" format:"uri"` + // Events delivered to this endpoint. `change.detected` fires only when a run + // detects a change; `run.completed` fires on every completed run — including runs + // that detected no change — and embeds the change when one was detected. Defaults + // to `["change.detected"]` when omitted. + // + // Any of "change.detected", "run.completed". + Events []string `json:"events,omitzero"` paramObj } diff --git a/monitor_test.go b/monitor_test.go index fbded67..d6689a9 100644 --- a/monitor_test.go +++ b/monitor_test.go @@ -46,7 +46,8 @@ func TestMonitorNewWithOptionalParams(t *testing.T) { Mode: contextdev.MonitorNewParamsModeWeb, Tags: []string{"pricing", "competitor"}, Webhook: contextdev.MonitorNewParamsWebhook{ - URL: "https://example.com/webhook", + URL: "https://example.com/webhook", + Events: []string{"change.detected", "run.completed"}, }, }) if err != nil { @@ -116,7 +117,8 @@ func TestMonitorUpdateWithOptionalParams(t *testing.T) { }, }, Webhook: contextdev.MonitorUpdateParamsWebhook{ - URL: "https://example.com/webhook", + URL: "https://example.com/webhook", + Events: []string{"change.detected", "run.completed"}, }, }, ) diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..1d3c18d --- /dev/null +++ b/parse.go @@ -0,0 +1,210 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +package contextdev + +import ( + "bytes" + "context" + "io" + "mime/multipart" + "net/http" + "net/url" + "slices" + + "github.com/context-dot-dev/context-go-sdk/v2/internal/apiform" + "github.com/context-dot-dev/context-go-sdk/v2/internal/apijson" + "github.com/context-dot-dev/context-go-sdk/v2/internal/apiquery" + "github.com/context-dot-dev/context-go-sdk/v2/internal/requestconfig" + "github.com/context-dot-dev/context-go-sdk/v2/option" + "github.com/context-dot-dev/context-go-sdk/v2/packages/param" + "github.com/context-dot-dev/context-go-sdk/v2/packages/respjson" +) + +// ParseService contains methods and other services that help with interacting with +// the context.dev API. +// +// Note, unlike clients, this service does not read variables from the environment +// automatically. You should not instantiate this service directly, and instead use +// the [NewParseService] method instead. +type ParseService struct { + options []option.RequestOption +} + +// NewParseService generates a new service that applies the given options to each +// request. These options are applied after the parent client's options (if there +// is one), and before any request-specific options. +func NewParseService(opts ...option.RequestOption) (r ParseService) { + r = ParseService{} + r.options = opts + return +} + +// Converts raw text, source code, web/data, PDF, Microsoft Office, and image bytes +// into LLM-usable Markdown. +func (r *ParseService) Handle(ctx context.Context, body io.Reader, params ParseHandleParams, opts ...option.RequestOption) (res *ParseHandleResponse, err error) { + opts = slices.Concat(r.options, opts) + opts = append([]option.RequestOption{option.WithRequestBody("application/octet-stream", body)}, opts...) + path := "parse" + err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, nil, &res, opts...) + return res, err +} + +type ParseHandleResponse struct { + // Input bytes converted to GitHub Flavored Markdown + Markdown string `json:"markdown" api:"required"` + // Indicates success + // + // Any of true. + Success bool `json:"success" api:"required"` + // Detected content type used for parsing + // + // Any of "html", "xml", "json", "jsonl", "text", "csv", "tsv", "markdown", "yaml", + // "python", "java", "javascript", "php", "shell", "ruby", "typescript", "rtf", + // "srt", "css", "scss", "less", "stylus", "sass", "svg", "pdf", "docx", "doc", + // "xlsx", "xls", "pptx", "ppt", "jpg", "png", "gif", "bmp", "tiff", "webp", "ppm", + // "pbm", "pgm", "pnm". + Type ParseHandleResponseType `json:"type" api:"required"` + // Metadata about the API key used for the request. Included in every response + // whenever a valid API key is provided, even when the response status is not 200. + KeyMetadata ParseHandleResponseKeyMetadata `json:"key_metadata"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Markdown respjson.Field + Success respjson.Field + Type respjson.Field + KeyMetadata respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r ParseHandleResponse) RawJSON() string { return r.JSON.raw } +func (r *ParseHandleResponse) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Detected content type used for parsing +type ParseHandleResponseType string + +const ( + ParseHandleResponseTypeHTML ParseHandleResponseType = "html" + ParseHandleResponseTypeXml ParseHandleResponseType = "xml" + ParseHandleResponseTypeJson ParseHandleResponseType = "json" + ParseHandleResponseTypeJSONL ParseHandleResponseType = "jsonl" + ParseHandleResponseTypeText ParseHandleResponseType = "text" + ParseHandleResponseTypeCsv ParseHandleResponseType = "csv" + ParseHandleResponseTypeTsv ParseHandleResponseType = "tsv" + ParseHandleResponseTypeMarkdown ParseHandleResponseType = "markdown" + ParseHandleResponseTypeYaml ParseHandleResponseType = "yaml" + ParseHandleResponseTypePython ParseHandleResponseType = "python" + ParseHandleResponseTypeJava ParseHandleResponseType = "java" + ParseHandleResponseTypeJavascript ParseHandleResponseType = "javascript" + ParseHandleResponseTypePhp ParseHandleResponseType = "php" + ParseHandleResponseTypeShell ParseHandleResponseType = "shell" + ParseHandleResponseTypeRuby ParseHandleResponseType = "ruby" + ParseHandleResponseTypeTypescript ParseHandleResponseType = "typescript" + ParseHandleResponseTypeRtf ParseHandleResponseType = "rtf" + ParseHandleResponseTypeSrt ParseHandleResponseType = "srt" + ParseHandleResponseTypeCss ParseHandleResponseType = "css" + ParseHandleResponseTypeScss ParseHandleResponseType = "scss" + ParseHandleResponseTypeLess ParseHandleResponseType = "less" + ParseHandleResponseTypeStylus ParseHandleResponseType = "stylus" + ParseHandleResponseTypeSass ParseHandleResponseType = "sass" + ParseHandleResponseTypeSvg ParseHandleResponseType = "svg" + ParseHandleResponseTypePdf ParseHandleResponseType = "pdf" + ParseHandleResponseTypeDocx ParseHandleResponseType = "docx" + ParseHandleResponseTypeDoc ParseHandleResponseType = "doc" + ParseHandleResponseTypeXlsx ParseHandleResponseType = "xlsx" + ParseHandleResponseTypeXls ParseHandleResponseType = "xls" + ParseHandleResponseTypePptx ParseHandleResponseType = "pptx" + ParseHandleResponseTypePpt ParseHandleResponseType = "ppt" + ParseHandleResponseTypeJpg ParseHandleResponseType = "jpg" + ParseHandleResponseTypePng ParseHandleResponseType = "png" + ParseHandleResponseTypeGif ParseHandleResponseType = "gif" + ParseHandleResponseTypeBmp ParseHandleResponseType = "bmp" + ParseHandleResponseTypeTiff ParseHandleResponseType = "tiff" + ParseHandleResponseTypeWebp ParseHandleResponseType = "webp" + ParseHandleResponseTypePpm ParseHandleResponseType = "ppm" + ParseHandleResponseTypePbm ParseHandleResponseType = "pbm" + ParseHandleResponseTypePgm ParseHandleResponseType = "pgm" + ParseHandleResponseTypePnm ParseHandleResponseType = "pnm" +) + +// Metadata about the API key used for the request. Included in every response +// whenever a valid API key is provided, even when the response status is not 200. +type ParseHandleResponseKeyMetadata struct { + // The number of credits consumed by this request. + CreditsConsumed int64 `json:"credits_consumed" api:"required"` + // The number of credits remaining for your organization after this request. + CreditsRemaining int64 `json:"credits_remaining" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + CreditsConsumed respjson.Field + CreditsRemaining respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r ParseHandleResponseKeyMetadata) RawJSON() string { return r.JSON.raw } +func (r *ParseHandleResponseKeyMetadata) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type ParseHandleParams struct { + // Optional HTTP(S) source document URL used to resolve relative links and image + // references. Relative references remain relative when omitted. + BaseURL param.Opt[string] `query:"baseUrl,omitzero" format:"uri" json:"-"` + // Optional file extension hint, such as pdf, docx, xlsx, pptx, html, json, csv, + // md, py, rtf, jpg, png, or txt. + Extension param.Opt[string] `query:"extension,omitzero" json:"-"` + // Optional filename hint used to infer the extension when extension is omitted. + Filename param.Opt[string] `query:"filename,omitzero" json:"-"` + // Include image references in Markdown output + IncludeImages param.Opt[bool] `query:"includeImages,omitzero" json:"-"` + // Preserve hyperlinks in Markdown output + IncludeLinks param.Opt[bool] `query:"includeLinks,omitzero" json:"-"` + // When true for PDF inputs, detect and OCR images embedded in the selected pages, + // inserting recognized text at each image's position in page reading order while + // preserving the PDF text layer. pdfStart/pdfEnd limit the inclusive page range. + // This is separate from automatic scanned-PDF OCR fallback. + Ocr param.Opt[bool] `query:"ocr,omitzero" json:"-"` + // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. + // Must be greater than or equal to pdfStart when both are provided. + PdfEnd param.Opt[int64] `query:"pdfEnd,omitzero" json:"-"` + // First 1-based PDF page to parse. When omitted, parsing starts at the first page. + PdfStart param.Opt[int64] `query:"pdfStart,omitzero" json:"-"` + // Shorten base64-encoded image data in the Markdown output + ShortenBase64Images param.Opt[bool] `query:"shortenBase64Images,omitzero" json:"-"` + // Extract only the main content from HTML-like inputs + UseMainContentOnly param.Opt[bool] `query:"useMainContentOnly,omitzero" json:"-"` + paramObj +} + +func (r ParseHandleParams) MarshalMultipart() (data []byte, contentType string, err error) { + buf := bytes.NewBuffer(nil) + writer := multipart.NewWriter(buf) + err = apiform.MarshalRoot(r, writer) + if err == nil { + err = apiform.WriteExtras(writer, r.ExtraFields()) + } + if err != nil { + writer.Close() + return nil, "", err + } + err = writer.Close() + if err != nil { + return nil, "", err + } + return buf.Bytes(), writer.FormDataContentType(), nil +} + +// URLQuery serializes [ParseHandleParams]'s query parameters as `url.Values`. +func (r ParseHandleParams) URLQuery() (v url.Values, err error) { + return apiquery.MarshalWithSettings(r, apiquery.QuerySettings{ + ArrayFormat: apiquery.ArrayQueryFormatComma, + NestedFormat: apiquery.NestedQueryFormatBrackets, + }) +} diff --git a/parse_test.go b/parse_test.go new file mode 100644 index 0000000..bbc0f4d --- /dev/null +++ b/parse_test.go @@ -0,0 +1,54 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +package contextdev_test + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "testing" + + "github.com/context-dot-dev/context-go-sdk/v2" + "github.com/context-dot-dev/context-go-sdk/v2/internal/testutil" + "github.com/context-dot-dev/context-go-sdk/v2/option" +) + +func TestParseHandleWithOptionalParams(t *testing.T) { + t.Skip("Mock server tests are disabled") + baseURL := "http://localhost:4010" + if envURL, ok := os.LookupEnv("TEST_API_BASE_URL"); ok { + baseURL = envURL + } + if !testutil.CheckTestServer(t, baseURL) { + return + } + client := contextdev.NewClient( + option.WithBaseURL(baseURL), + option.WithAPIKey("My API Key"), + ) + _, err := client.Parse.Handle( + context.TODO(), + io.Reader(bytes.NewBuffer([]byte("Example data"))), + contextdev.ParseHandleParams{ + BaseURL: contextdev.String("https://example.com"), + Extension: contextdev.String("extension"), + Filename: contextdev.String("filename"), + IncludeImages: contextdev.Bool(true), + IncludeLinks: contextdev.Bool(true), + Ocr: contextdev.Bool(true), + PdfEnd: contextdev.Int(1), + PdfStart: contextdev.Int(1), + ShortenBase64Images: contextdev.Bool(true), + UseMainContentOnly: contextdev.Bool(true), + }, + ) + if err != nil { + var apierr *contextdev.Error + if errors.As(err, &apierr) { + t.Log(string(apierr.DumpRequest(true))) + } + t.Fatalf("err should be nil: %s", err.Error()) + } +} diff --git a/web.go b/web.go index 45f3900..60d030a 100644 --- a/web.go +++ b/web.go @@ -116,7 +116,22 @@ func (r *WebService) WebScrapeImages(ctx context.Context, query WebWebScrapeImag return res, err } -// Scrapes the given URL into LLM usable Markdown. +// Scrapes the given URL into LLM usable Markdown. Inspect key_metadata on JSON +// responses from a recognized API key; use error_code to distinguish stable +// failure categories. +// +// ### Billing & errors +// +// | HTTP status | Billed? | Meaning | +// | ----------- | -------------- | ---------------------------------------------------------------------------------------- | +// | 200 | Yes — 1 credit | Successful scrape, including a zero-length result when includeSelectors matched nothing | +// | 400 | No | Invalid input, skipped PDF, or the page could not be scraped | +// | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | +// | 404 | No | Target page returned or fingerprinted as not found | +// | 408 | No | Request timed out | +// | 415 | No | Unsupported content type | +// | 429 | No | Per-minute rate limit exceeded; honor Retry-After | +// | 500 | No | Internal error | func (r *WebService) WebScrapeMd(ctx context.Context, query WebWebScrapeMdParams, opts ...option.RequestOption) (res *WebWebScrapeMdResponse, err error) { opts = slices.Concat(r.options, opts) path := "web/scrape/markdown" @@ -1996,6 +2011,10 @@ func (r *WebWebScrapeImagesResponseKeyMetadata) UnmarshalJSON(data []byte) error } type WebWebScrapeMdResponse struct { + // UTF-8 byte length of the returned Markdown. Use 0 to identify an empty result + // and compare small values against your workload's minimum useful-content + // threshold. + ContentLength int64 `json:"contentLength" api:"required"` // Page content converted to GitHub Flavored Markdown Markdown string `json:"markdown" api:"required"` // Metadata extracted from the scraped page HTML. @@ -2011,13 +2030,14 @@ type WebWebScrapeMdResponse struct { KeyMetadata WebWebScrapeMdResponseKeyMetadata `json:"key_metadata"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - Markdown respjson.Field - Metadata respjson.Field - Success respjson.Field - URL respjson.Field - KeyMetadata respjson.Field - ExtraFields map[string]respjson.Field - raw string + ContentLength respjson.Field + Markdown respjson.Field + Metadata respjson.Field + Success respjson.Field + URL respjson.Field + KeyMetadata respjson.Field + ExtraFields map[string]respjson.Field + raw string } `json:"-"` } @@ -3313,8 +3333,8 @@ type WebWebCrawlMdParams struct { // omitted, the entire document is kept. Examples: "article.main", "#content", // "[role=main]". IncludeSelectors []string `json:"includeSelectors,omitzero"` - // PDF parsing controls. Use start/end to limit text extraction and OCR to an - // inclusive 1-based page range. + // PDF parsing controls. Use start/end to limit text extraction and embedded-image + // detection/OCR to an inclusive 1-based page range. Pdf WebWebCrawlMdParamsPdf `json:"pdf,omitzero"` paramObj } @@ -3539,12 +3559,16 @@ const ( WebWebCrawlMdParamsCountryZw WebWebCrawlMdParamsCountry = "zw" ) -// PDF parsing controls. Use start/end to limit text extraction and OCR to an -// inclusive 1-based page range. +// PDF parsing controls. Use start/end to limit text extraction and embedded-image +// detection/OCR to an inclusive 1-based page range. type WebWebCrawlMdParamsPdf struct { // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. // Must be greater than or equal to start when both are provided. End param.Opt[int64] `json:"end,omitzero"` + // When true, detect and OCR images embedded in the selected PDF pages, inserting + // recognized text at each image's position in page reading order while preserving + // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + Ocr param.Opt[bool] `json:"ocr,omitzero"` // When true, PDF pages are fetched and parsed. When false, PDF pages are skipped // entirely (not included in results and not counted as failures). ShouldParse param.Opt[bool] `json:"shouldParse,omitzero"` @@ -3616,8 +3640,8 @@ type WebWebScrapeHTMLParams struct { // kept and everything else is dropped. When omitted, the entire document is kept. // Examples: "article.main", "#content", "[role=main]". IncludeSelectors []string `query:"includeSelectors,omitzero" json:"-"` - // PDF parsing controls. Use start/end to limit text extraction and OCR to an - // inclusive 1-based page range. + // PDF parsing controls. Use start/end to limit text extraction and embedded-image + // detection/OCR to an inclusive 1-based page range. Pdf WebWebScrapeHTMLParamsPdf `query:"pdf,omitzero" json:"-"` paramObj } @@ -3841,12 +3865,16 @@ const ( WebWebScrapeHTMLParamsCountryZw WebWebScrapeHTMLParamsCountry = "zw" ) -// PDF parsing controls. Use start/end to limit text extraction and OCR to an -// inclusive 1-based page range. +// PDF parsing controls. Use start/end to limit text extraction and embedded-image +// detection/OCR to an inclusive 1-based page range. type WebWebScrapeHTMLParamsPdf struct { // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. // Must be greater than or equal to start when both are provided. End param.Opt[int64] `query:"end,omitzero" json:"-"` + // When true, detect and OCR images embedded in the selected PDF pages, inserting + // recognized text at each image's position in page reading order while preserving + // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + Ocr param.Opt[bool] `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and // a 400 WEBSITE_ACCESS_ERROR is returned. ShouldParse param.Opt[bool] `query:"shouldParse,omitzero" json:"-"` @@ -3987,8 +4015,8 @@ type WebWebScrapeMdParams struct { // descendants) are kept before conversion to Markdown. When omitted, the entire // document is kept. Examples: "article.main", "#content", "[role=main]". IncludeSelectors []string `query:"includeSelectors,omitzero" json:"-"` - // PDF parsing controls. Use start/end to limit text extraction and OCR to an - // inclusive 1-based page range. + // PDF parsing controls. Use start/end to limit text extraction and embedded-image + // detection/OCR to an inclusive 1-based page range. Pdf WebWebScrapeMdParamsPdf `query:"pdf,omitzero" json:"-"` paramObj } @@ -4212,12 +4240,16 @@ const ( WebWebScrapeMdParamsCountryZw WebWebScrapeMdParamsCountry = "zw" ) -// PDF parsing controls. Use start/end to limit text extraction and OCR to an -// inclusive 1-based page range. +// PDF parsing controls. Use start/end to limit text extraction and embedded-image +// detection/OCR to an inclusive 1-based page range. type WebWebScrapeMdParamsPdf struct { // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. // Must be greater than or equal to start when both are provided. End param.Opt[int64] `query:"end,omitzero" json:"-"` + // When true, detect and OCR images embedded in the selected PDF pages, inserting + // recognized text at each image's position in page reading order while preserving + // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + Ocr param.Opt[bool] `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and // a 400 WEBSITE_ACCESS_ERROR is returned. ShouldParse param.Opt[bool] `query:"shouldParse,omitzero" json:"-"` diff --git a/web_test.go b/web_test.go index ce1ddaa..8981173 100644 --- a/web_test.go +++ b/web_test.go @@ -257,6 +257,7 @@ func TestWebWebCrawlMdWithOptionalParams(t *testing.T) { MaxPages: contextdev.Int(1), Pdf: contextdev.WebWebCrawlMdParamsPdf{ End: contextdev.Int(1), + Ocr: contextdev.Bool(true), ShouldParse: contextdev.Bool(true), Start: contextdev.Int(1), }, @@ -302,6 +303,7 @@ func TestWebWebScrapeHTMLWithOptionalParams(t *testing.T) { MaxAgeMs: contextdev.Int(0), Pdf: contextdev.WebWebScrapeHTMLParamsPdf{ End: contextdev.Int(1), + Ocr: contextdev.Bool(true), ShouldParse: contextdev.Bool(true), Start: contextdev.Int(1), }, @@ -384,6 +386,7 @@ func TestWebWebScrapeMdWithOptionalParams(t *testing.T) { MaxAgeMs: contextdev.Int(0), Pdf: contextdev.WebWebScrapeMdParamsPdf{ End: contextdev.Int(1), + Ocr: contextdev.Bool(true), ShouldParse: contextdev.Bool(true), Start: contextdev.Int(1), },