diff --git a/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-csharp.mdx b/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-csharp.mdx index fd9d11719f..bf57270e21 100644 --- a/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-csharp.mdx +++ b/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-csharp.mdx @@ -49,7 +49,7 @@ import CodeBlock from '@theme/CodeBlock'; * Pre-made embeddings that are automatically generated from your document content by RavenDB's [Embeddings generation tasks](../../../ai-integration/generating-embeddings/overview.mdx) using external service providers. -* Note: Vector search queries cannot be used with [Subscription queries](../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-query). +* Note: Vector search queries cannot be used with [Subscription queries](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-query). * When executing a dynamic vector search query, RavenDB creates a [Corax Auto-Index](../../../ai-integration/vector-search/vector-search-using-dynamic-query.mdx#corax-auto-indexes) to process the query, and the results are retrieved from that index. diff --git a/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-nodejs.mdx b/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-nodejs.mdx index d92914b44d..8b82580da8 100644 --- a/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-nodejs.mdx +++ b/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-nodejs.mdx @@ -49,7 +49,7 @@ import CodeBlock from '@theme/CodeBlock'; * Pre-made embeddings that are automatically generated from your document content by RavenDB's [Embeddings generation tasks](../../../ai-integration/generating-embeddings/overview.mdx) using external service providers. -* Note: Vector search queries cannot be used with [Subscription queries](../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-query). +* Note: Vector search queries cannot be used with [Subscription queries](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-query). * When executing a dynamic vector search query, RavenDB creates a [Corax Auto-Index](../../../ai-integration/vector-search/vector-search-using-dynamic-query.mdx#corax-auto-indexes) to process the query, and the results are retrieved from that index. diff --git a/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-python.mdx b/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-python.mdx index 49733ade25..27bd7d4514 100644 --- a/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-python.mdx +++ b/docs/ai-integration/vector-search/content/_vector-search-using-dynamic-query-python.mdx @@ -49,7 +49,7 @@ import CodeBlock from '@theme/CodeBlock'; * Pre-made embeddings that are automatically generated from your document content by RavenDB's [Embeddings generation tasks](../../../ai-integration/generating-embeddings/overview.mdx) using external service providers. -* Note: Vector search queries cannot be used with [Subscription queries](../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-query). +* Note: Vector search queries cannot be used with [Subscription queries](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-query). * When executing a dynamic vector search query, RavenDB creates a [Corax Auto-Index](../../../ai-integration/vector-search/vector-search-using-dynamic-query.mdx#corax-auto-indexes) to process the query, and the results are retrieved from that index. diff --git a/docs/backup/overview.mdx b/docs/backup/overview.mdx index 6bacfe21af..56bd993214 100644 --- a/docs/backup/overview.mdx +++ b/docs/backup/overview.mdx @@ -161,7 +161,7 @@ Backups contain both database-level and cluster-level content, as detailed below | [Identities](../client-api/document-identifiers/working-with-document-identifiers#identities) | ✔ | ✔ | | [Indexes](../indexes/creating-and-deploying) | Indexes are not preserved,
but rebuilt after [restoring](../backup/restore)
from backed up index definitions. | ✔ | | [Ongoing Tasks configuration](../studio/database/tasks/ongoing-tasks/general-info.mdx) | ✔ | ✔ | -| [Subscriptions](../client-api/data-subscriptions/what-are-data-subscriptions) | ✔ | ✔ | +| [Subscriptions](../data-subscriptions/overview) | ✔ | ✔ | diff --git a/docs/client-api/changes/content/_what-is-changes-api-csharp.mdx b/docs/client-api/changes/content/_what-is-changes-api-csharp.mdx index 240dac83d6..fe85782823 100644 --- a/docs/client-api/changes/content/_what-is-changes-api-csharp.mdx +++ b/docs/client-api/changes/content/_what-is-changes-api-csharp.mdx @@ -179,7 +179,7 @@ To get more method overloads, especially ones supporting **delegates**, please a ## Changes API -vs- Data Subscriptions -**Changes API** and [Data Subscription](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) +**Changes API** and [Data Subscription](../../../data-subscriptions/overview.mdx) are services that a RavenDB Server provides subscribing clients. Both services respond to events that take place on the server, by sending updates to their subscribers. @@ -205,6 +205,6 @@ to their subscribers. | | Data Subscriptions | Changes API | |------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| What can the server Track | [Documents](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#documents-processing)
[Revisions](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx)
[Counters](../../../client-api/data-subscriptions/creation/examples.mdx#including-counters)
Time Series | [Documents](../../../client-api/changes/how-to-subscribe-to-document-changes.mdx)
[Indexes](../../../client-api/changes/how-to-subscribe-to-index-changes.mdx)
[Operations](../../../client-api/changes/how-to-subscribe-to-operation-changes.mdx)
[Counters](../../../client-api/changes/how-to-subscribe-to-counter-changes.mdx)
[Time Series](../../../client-api/changes/how-to-subscribe-to-time-series-changes.mdx) | +| What can the server Track | [Documents](../../../data-subscriptions/overview.mdx#how-documents-are-processed)
[Revisions](../../../data-subscriptions/revisions-support.mdx)
[Counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters)
Time Series | [Documents](../../../client-api/changes/how-to-subscribe-to-document-changes.mdx)
[Indexes](../../../client-api/changes/how-to-subscribe-to-index-changes.mdx)
[Operations](../../../client-api/changes/how-to-subscribe-to-operation-changes.mdx)
[Counters](../../../client-api/changes/how-to-subscribe-to-counter-changes.mdx)
[Time Series](../../../client-api/changes/how-to-subscribe-to-time-series-changes.mdx) | | What can the server Deliver | Documents
Revisions
Counters
Time Series | Notifications | | Management | Managed by the Server | Managed by the Client | diff --git a/docs/client-api/changes/content/_what-is-changes-api-java.mdx b/docs/client-api/changes/content/_what-is-changes-api-java.mdx index df488dd95d..2bd9562b4e 100644 --- a/docs/client-api/changes/content/_what-is-changes-api-java.mdx +++ b/docs/client-api/changes/content/_what-is-changes-api-java.mdx @@ -122,7 +122,7 @@ idle and unloaded, regardless of [the configuration value for database idle time ## Changes API -vs- Data Subscriptions -**Changes API** and [Data Subscription](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) +**Changes API** and [Data Subscription](../../../data-subscriptions/overview.mdx) are services that a RavenDB Server provides subscribing clients. Both services respond to events that take place on the server, by sending updates to their subscribers. @@ -148,6 +148,6 @@ to their subscribers. | | Data Subscriptions | Changes API | |------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| What can the server Track | [Documents](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#documents-processing)
[Revisions](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx)
[Counters](../../../client-api/data-subscriptions/creation/examples.mdx#including-counters)
Time Series | [Documents](../../../client-api/changes/how-to-subscribe-to-document-changes.mdx)
[Indexes](../../../client-api/changes/how-to-subscribe-to-index-changes.mdx)
[Operations](../../../client-api/changes/how-to-subscribe-to-operation-changes.mdx)
[Counters](../../../client-api/changes/how-to-subscribe-to-counter-changes.mdx)
[Time Series](../../../client-api/changes/how-to-subscribe-to-time-series-changes.mdx) | +| What can the server Track | [Documents](../../../data-subscriptions/overview.mdx#how-documents-are-processed)
[Revisions](../../../data-subscriptions/revisions-support.mdx)
[Counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters)
Time Series | [Documents](../../../client-api/changes/how-to-subscribe-to-document-changes.mdx)
[Indexes](../../../client-api/changes/how-to-subscribe-to-index-changes.mdx)
[Operations](../../../client-api/changes/how-to-subscribe-to-operation-changes.mdx)
[Counters](../../../client-api/changes/how-to-subscribe-to-counter-changes.mdx)
[Time Series](../../../client-api/changes/how-to-subscribe-to-time-series-changes.mdx) | | What can the server Deliver | Documents
Revisions
Counters
Time Series | Notifications | | Management | Managed by the Server | Managed by the Client | diff --git a/docs/client-api/changes/content/_what-is-changes-api-nodejs.mdx b/docs/client-api/changes/content/_what-is-changes-api-nodejs.mdx index 2dc3ced34c..342a0ef4d4 100644 --- a/docs/client-api/changes/content/_what-is-changes-api-nodejs.mdx +++ b/docs/client-api/changes/content/_what-is-changes-api-nodejs.mdx @@ -121,7 +121,7 @@ To get more method overloads, especially ones supporting **delegates**, please a ## Changes API -vs- Data Subscriptions -**Changes API** and [Data Subscription](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) +**Changes API** and [Data Subscription](../../../data-subscriptions/overview.mdx) are services that a RavenDB Server provides subscribing clients. Both services respond to events that take place on the server, by sending updates to their subscribers. @@ -147,6 +147,6 @@ to their subscribers. | | Data Subscriptions | Changes API | |------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| What can the server Track | [Documents](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#documents-processing)
[Revisions](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx)
[Counters](../../../client-api/data-subscriptions/creation/examples.mdx#including-counters)
Time Series | [Documents](../../../client-api/changes/how-to-subscribe-to-document-changes.mdx)
[Indexes](../../../client-api/changes/how-to-subscribe-to-index-changes.mdx)
[Operations](../../../client-api/changes/how-to-subscribe-to-operation-changes.mdx)
[Counters](../../../client-api/changes/how-to-subscribe-to-counter-changes.mdx)
[Time Series](../../../client-api/changes/how-to-subscribe-to-time-series-changes.mdx) | +| What can the server Track | [Documents](../../../data-subscriptions/overview.mdx#how-documents-are-processed)
[Revisions](../../../data-subscriptions/revisions-support.mdx)
[Counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters)
Time Series | [Documents](../../../client-api/changes/how-to-subscribe-to-document-changes.mdx)
[Indexes](../../../client-api/changes/how-to-subscribe-to-index-changes.mdx)
[Operations](../../../client-api/changes/how-to-subscribe-to-operation-changes.mdx)
[Counters](../../../client-api/changes/how-to-subscribe-to-counter-changes.mdx)
[Time Series](../../../client-api/changes/how-to-subscribe-to-time-series-changes.mdx) | | What can the server Deliver | Documents
Revisions
Counters
Time Series | Notifications | | Management | Managed by the Server | Managed by the Client | diff --git a/docs/client-api/changes/what-is-changes-api.mdx b/docs/client-api/changes/what-is-changes-api.mdx index 176f6505e7..fd59adfc09 100644 --- a/docs/client-api/changes/what-is-changes-api.mdx +++ b/docs/client-api/changes/what-is-changes-api.mdx @@ -30,7 +30,7 @@ see_also: source: "docs" path: "Client API > Changes API" - title: "What Are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" + link: "data-subscriptions/overview" source: "docs" path: "Client API > Data Subscriptions" --- diff --git a/docs/client-api/commands/content/_overview-csharp.mdx b/docs/client-api/commands/content/_overview-csharp.mdx index 6c14ce14a6..53f150b889 100644 --- a/docs/client-api/commands/content/_overview-csharp.mdx +++ b/docs/client-api/commands/content/_overview-csharp.mdx @@ -45,7 +45,7 @@ import CodeBlock from '@theme/CodeBlock'; ## Execute command - using the Store Request Executor This example shows how to execute the low-level `CreateSubscriptionCommand` via the **Store**. -(For examples of creating a subscription using higher-level methods, see [subscription creation examples](../../../client-api/data-subscriptions/creation/examples.mdx)). +(For examples of creating a subscription using higher-level methods, see [subscription creation examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx)). diff --git a/docs/client-api/commands/content/_overview-nodejs.mdx b/docs/client-api/commands/content/_overview-nodejs.mdx index 8e2fe72792..2b662db5b2 100644 --- a/docs/client-api/commands/content/_overview-nodejs.mdx +++ b/docs/client-api/commands/content/_overview-nodejs.mdx @@ -45,7 +45,7 @@ import CodeBlock from '@theme/CodeBlock'; ## Execute command - using the Store Request Executor This example shows how to execute the low-level `CreateSubscriptionCommand` via the **Store**. -(For examples of creating a subscription using higher-level methods, see [subscription creation examples](../../../client-api/data-subscriptions/creation/examples.mdx)). +(For examples of creating a subscription using higher-level methods, see [subscription creation examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx)). diff --git a/docs/client-api/content/_what-is-a-document-store-csharp.mdx b/docs/client-api/content/_what-is-a-document-store-csharp.mdx index 8bbf10e71d..1d770e9369 100644 --- a/docs/client-api/content/_what-is-a-document-store-csharp.mdx +++ b/docs/client-api/content/_what-is-a-document-store-csharp.mdx @@ -25,7 +25,7 @@ and any configurations & customizations that you may have applied. * [Changes API](../../client-api/changes/what-is-changes-api.mdx) - Receive messages from the server * [Aggressive caching](../../client-api/how-to/setup-aggressive-caching.mdx) - Configure caching behavior * [Events](../../client-api/session/how-to/subscribe-to-events.mdx) - Perform custom actions in response to the Session's operations - * [Data Subscriptions](../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) - Define & manage data processing on the client side + * [Data Subscriptions](../../data-subscriptions/overview.mdx) - Define & manage data processing on the client side diff --git a/docs/client-api/data-subscriptions/_category_.json b/docs/client-api/data-subscriptions/_category_.json deleted file mode 100644 index 129df78e9f..0000000000 --- a/docs/client-api/data-subscriptions/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "position": 9, - "label": Data Subscriptions, -} \ No newline at end of file diff --git a/docs/client-api/data-subscriptions/advanced-topics/_category_.json b/docs/client-api/data-subscriptions/advanced-topics/_category_.json deleted file mode 100644 index c206697952..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "position": 3, - "label": Advanced topics, -} \ No newline at end of file diff --git a/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-csharp.mdx b/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-csharp.mdx deleted file mode 100644 index c0683651da..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-csharp.mdx +++ /dev/null @@ -1,206 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* This article covers data subscriptions maintenance operations. - -* In this page: - * [DocumentSubscriptions class](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#documentsubscriptions-class) - * [Delete subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#delete-subscription) - * [Disabling subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#disable-subscription) - * [Enable subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#enable-subscription) - * [Update subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#update-subscription) - * [Drop Connection](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#drop-connection) - * [Get subscription state](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#get-subscription-state) - - -## DocumentSubscriptions class - -The `DocumentSubscriptions` class is the class that manages all interaction with the data subscriptions. -The class is available through `DocumentStore`'s `Subscriptions` property. - -| Method Signature | Return type | Description | -|---------------------------------------------------------------------------------------------------------------|---------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| **Create<T>(SubscriptionCreationOptions<T> options, string database)** | `string` | Create a new data subscription. | -| **Create(SubscriptionCreationOptions criteria, string database)** | `string` | Create a new data subscription. | -| **Create(SubscriptionCreationOptions criteria, string database)** | `string` | Create a new data subscription. | -| **CreateAsync<T>(SubscriptionCreationOptions<T> options, string database)** | `Task` | Create a new data subscription. | -| **CreateAsync<T>(Expression<Func<T, bool>> predicate, SubscriptionCreationOptions options, string database)** | `Task` | Create a new data subscription. | -| **Delete(string name, string database)** | `void` | Delete subscription. | -| **DeleteAsync(string name, string database)** | `Task` | Delete subscription. | -| **DropConnection(string name, string database)** | `void` | Drop all existing subscription connections with workers. | -| **DropConnectionAsync(string name, string database)** | `Task` | Drop all existing subscription connections with workers. | -| **DropSubscriptionWorker<T>(SubscriptionWorker<T> worker, string database = null)** | `void` | Drop an existing subscription connection with a worker | -| **Enable(string name, string database)** | `void` | Enable existing subscription. | -| **EnableAsync(string name, string database)** | `Task` | Enable existing subscription. | -| **Disable(string name, string database)** | `void` | Disable existing subscription. | -| **DisableAsync(string name, string database)** | `Task` | Disable existing subscription. | -| **GetSubscriptions(int start, int take, string database)** | `List` | Returns subscriptions list. | -| **GetSubscriptionsAsync(int start, int take, string database)** | `Task>` | Returns subscriptions list. | -| **GetSubscriptionState(string subscriptionName, string database)** | `SubscriptionState ` | Get specific subscription state. | -| **GetSubscriptionStateAsync(string subscriptionName, string database)** | `Task ` | Get specific subscription state. | -| **GetSubscriptionWorker<T>(string subscriptionName, string database)** | `SubscriptionWorker` | Generate a subscription worker, using default configurations, that processes documents deserialized to `T` type . | -| **GetSubscriptionWorker(string subscriptionName, string database)** | `SubscriptionWorker` | Generate a subscription worker, using default configurations, that processes documents in its raw `BlittableJsonReader`, wrapped by dynamic object. | -| **GetSubscriptionWorker(SubscriptionWorkerOptions options, string database)** | `SubscriptionWorker` | Generate a subscription worker, using default configurations, that processes documents deserialized to `T` type . | -| **GetSubscriptionWorker(SubscriptionWorkerOptions options, string database)** | `SubscriptionWorker` | Generate a subscription worker, using default configurations, that processes documents in its raw `BlittableJsonReader`, wrapped by dynamic object. | -| **Update(SubscriptionUpdateOptions options, string database = null)** | `string` | Update an existing data subscription. | -| **UpdateAsync(SubscriptionUpdateOptions options, string database = null, CancellationToken token = default)** | `Task` | Update an existing data subscription. | - - - -## Delete subscription - -Subscriptions can be entirely deleted from the system. - -This operation can be very useful in ad-hoc subscription scenarios when a lot of subscriptions tasks information may accumulate, making tasks management very hard. - - - -{`void Delete(string name, string database = null); -Task DeleteAsync(string name, string database = null, CancellationToken token = default); -`} - - - -usage: - - - -{`store.Subscriptions.Delete(subscriptionName); -`} - - - - - -## Disable subscription - -Existing subscription tasks can be disabled from the client. - - - -{`void Disable(string name, string database = null); -Task DisableAsync(string name, string database = null, CancellationToken token = default); -`} - - - -usage: - - - -{`store.Subscriptions.Disable(subscriptionName); -`} - - - - - -## Enable subscription - -Existing subscription tasks can be enabled from the client. -This operation can be useful for already disabled subscriptions. - - - -{`void Enable(string name, string database = null); -Task EnableAsync(string name, string database = null, CancellationToken token = default); -`} - - - -usage: - - - -{`store.Subscriptions.Enable(subscriptionName); -`} - - - - - -## Update subscription - -See [examples](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription) -and [API description](../../../../client-api/data-subscriptions/creation/api-overview.mdx#update-subscription). - - - -{`string Update(SubscriptionUpdateOptions options, string database = null); - -Task UpdateAsync(SubscriptionUpdateOptions options, string database = null, - CancellationToken token = default); -`} - - - - - -## Drop connection - -Active subscription connections established by workers can be dropped remotely from the client. -Once dropped, the worker will not attempt to reconnect to the server. - - - -{`void DropConnection(string name, string database = null); -Task DropConnectionAsync(string name, string database = null, CancellationToken token = default); -`} - - - -usage: - - - -{`store.Subscriptions.DropConnection(subscriptionName); -`} - - - - - -## Get subscription state - - - -{`SubscriptionState GetSubscriptionState(string subscriptionName, string database = null); -Task GetSubscriptionStateAsync(string subscriptionName, string database = null, CancellationToken token = default); -`} - - - -usage: - - - -{`var subscriptionState = store.Subscriptions.GetSubscriptionState(subscriptionName); -`} - - - - - -##### SubscriptionState - -| Member | Type | Description | -|-------------------------------------------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Query** | `string` | Subscription's RQL like query. | -| **LastBatchAckTime** | `DateTime?` | Last time a batch processing progress was acknowledged. | -| **NodeTag** | `string` | Processing server's node tag. | -| **MentorNode** | `string` | The mentor node that was manually set. | -| **SubscriptionName** | `string` | The subscription's name, which is also its unique identifier. | -| **SubscriptionId** | `long` | Subscription's internal identifier (cluster's operation etag during subscription creation). | -| **ChangeVectorForNextBatchStartingPoint** | `string` | The Change Vector from which the subscription will begin sending documents.<br.This value is updated on batch acknowledgement and can also be set manually. | -| **Disabled** | `bool` | If `true`, subscription will not allow workers to connect. | -| **LastClientConnectionTime** | `DateTime?` | Time when last client was connected (value sustained after disconnection). | - - - - - - diff --git a/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-java.mdx b/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-java.mdx deleted file mode 100644 index 994b10206c..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-java.mdx +++ /dev/null @@ -1,160 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* This page covers data subscriptions maintenance operations: - * [Deleting subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#deleting-subscription) - * [Dropping connection](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#dropping-connection) - * [Disabling subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#disabling-subscription) - * [Updating subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#updating-subscription) - * [Getting subscription status](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#getting-subscription-status) - - -## Deleting subscription - -Subscriptions can be entirely deleted from the system. - -This operation can be very useful in ad-hoc subscription scenarios when a lot of subscriptions tasks information may accumulate, making tasks management very hard. - - - -{`void delete(String name); - -void delete(String name, String database); -`} - - - -usage: - - - -{`store.subscriptions().delete(subscriptionName); -`} - - - - - -## Dropping connection - -Subscription connections with workers can be dropped remotely. -A dropped worker will not try to reconnect to the server. - - - -{`void dropConnection(String name); - -void dropConnection(String name, String database); -`} - - - -usage: - - - -{`store.subscriptions().dropConnection(subscriptionName); -`} - - - - - -## Disabling subscription - - -This operation can only be performed through the management studio - - - - -## Updating subscription - - -This operation can only be performed through the management studio - - - - -## Getting subscription status - - - -{`SubscriptionState getSubscriptionState(String subscriptionName); - -SubscriptionState getSubscriptionState(String subscriptionName, String database); -`} - - - -usage: - - - -{`SubscriptionState subscriptionState = store.subscriptions().getSubscriptionState(subscriptionName); -`} - - - - - -| Member | Type | Description | -|--------|:-----|-------------| -| **query** | `String` | Subscription's RQL like query. | -| **lastBatchAckTime** | `Date` | Last time a batch processing progress was acknowledged. | -| **nodeTag** | `String` | Processing server's node tag | -| **mentorNode** | `String` | The mentor node that was manually set. | -| **subscriptionName** | `String` | Subscription's name, and also it's unique identifier | -| **subscriptionId** | `long` | Subscription's internal identifier (cluster's operation etag during subscription creation) | -| **changeVectorForNextBatchStartingPoint** | `String` | Change vector, starting from which the subscription will send documents. This value is updated manually, or automatically on batch acknowledgment | -| **disabled** | `boolean` | If true, subscription will not allow workers to connect | -| **lastClientConnectionTime** | `Date` | Time when last client was connected (value sustained after disconnection) | - - - - - -## DocumentSubscriptions class - -The `DocumentSubscriptions` class is the class that manages all interaction with the data subscriptions. -The class is available through `DocumentStore`'s `subscriptions()` method. - -| Method Signature| Return type | Description | -|--------|:---|-------------| -| **create(SubscriptionCreationOptions options)** | `String` | Creates a new data subscription. | -| **create(SubscriptionCreationOptions options, String database)** | `String` | Creates a new data subscription. | -| **create(SubscriptionCreationOptions options)** | `String` | Creates a new data subscription. | -| **create(Class<T> clazz)** | `String` | Creates a new data subscription. | -| **create(Class<T> clazz, SubscriptionCreationOptions options)** | `String` | Creates a new data subscription. | -| **create(Class<T> clazz, SubscriptionCreationOptions options, String database)** | `String` | Creates a new data subscription. | -| **createForRevisions(Class<T> clazz)** | `String` | Creates a new data subscription. | -| **createForRevisions(Class<T> clazz, SubscriptionCreationOptions options)** | `String` | Creates a new data subscription. | -| **createForRevisions(Class<T> clazz, SubscriptionCreationOptions options, String database)** | `String` | Creates a new data subscription. | -| **delete(String name)** | `void` | Deletes subscription. | -| **delete(String name, String database)** | `void` | Deletes subscription. | -| **dropConnection(String name)** | `void` | Drops existing subscription connection with worker. | -| **dropConnection(String name, String database)** | `void` | Drops existing subscription connection with worker. | -| **getSubscriptions(int start, int take)** | `List` | Returns subscriptions list. | -| **getSubscriptions(int start, int take, String database)** | `List` | Returns subscriptions list. | -| **getSubscriptionState(String subscriptionName)** | `SubscriptionState ` | Get specific subscription state. | -| **getSubscriptionState(String subscriptionName, String database)** | `SubscriptionState ` | Get specific subscription state. | -| **getSubscriptionWorker(string subscriptionName)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents in it's raw `ObjectNode` type . | -| **getSubscriptionWorker(string subscriptionName, String database)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents in it's raw `ObjectNode` type . | -| **getSubscriptionWorker(SubscriptionWorkerOptions options)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents in it's raw `ObjectNode` type . | -| **getSubscriptionWorker(SubscriptionWorkerOptions options, String database)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents in it's raw `ObjectNode` type . | -| **getSubscriptionWorker<T>(Class<T> clazz, String subscriptionName)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents deserialized to `T` type . | -| **getSubscriptionWorker<T>(Class<T> clazz, String subscriptionName, String database)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents deserialized to `T` type . | -| **getSubscriptionWorker<T>(Class<T> clazz, SubscriptionWorkerOptions options)** | `SubscriptionWorker` | Generates a subscription worker, using provided configuration, that processes documents deserialized to `T` type . | -| **getSubscriptionWorker<T>(Class<T> clazz, SubscriptionWorkerOptions options, String database)** | `SubscriptionWorker` | Generates a subscription worker, using provided configuration, that processes documents deserialized to `T` type . | -| **getSubscriptionWorkerForRevisions<T>(Class<T> clazz, String subscriptionName)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents deserialized to `T` type . | -| **getSubscriptionWorkerForRevisions<T>(Class<T> clazz, String subscriptionName, String database)** | `SubscriptionWorker` | Generates a subscription worker, using default configurations, that processes documents deserialized to `T` type . | -| **getSubscriptionWorkerForRevisions<T>(Class<T> clazz, SubscriptionWorkerOptions options)** | `SubscriptionWorker` | Generates a subscription worker, using provided configuration, that processes documents deserialized to `T` type . | -| **getSubscriptionWorkerForRevisions<T>(Class<T> clazz, SubscriptionWorkerOptions options, String database)** | `SubscriptionWorker` | Generates a subscription worker, using provided configuration, that processes documents deserialized to `T` type . | - - - - diff --git a/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-nodejs.mdx b/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-nodejs.mdx deleted file mode 100644 index 83379c7a73..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/content/_maintenance-operations-nodejs.mdx +++ /dev/null @@ -1,252 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* This article covers data subscriptions maintenance operations. - -* In this page: - * [Delete subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#delete-subscription) - * [Disable subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#disable-subscription) - * [Enable subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#enable-subscription) - * [Update subscription](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#update-subscription) - * [Drop connection](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#drop-connection) - * [Get subscriptions](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#get-subscriptions) - * [Get subscription state](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#get-subscription-state) - * [DocumentSubscriptions class](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#documentsubscriptions-class) - - -## Delete subscription - -Subscription tasks can be entirely deleted from the system. - - - -{`await documentStore.subscriptions.delete("subscriptionNameToDelete"); -`} - - - - -{`// Available overloads: -delete(name); -delete(name, database); -`} - - - - - -## Disable subscription - -Existing subscription tasks can be disabled from the client. - - - -{`await documentStore.subscriptions.disable("subscriptionNameToDisable"); -`} - - - - -{`// Available overloads: -disable(name); -disable(name, database); -`} - - - - - -## Enable subscription - -Existing subscription tasks can be enabled from the client. -This operation can be useful for already disabled subscriptions. - - - -{`await documentStore.subscriptions.enable("subscriptionNameToEnable"); -`} - - - - -{`// Available overloads: -enable(name); -enable(name, database); -`} - - - - - -## Update subscription - -See [examples](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription) -and [API description](../../../../client-api/data-subscriptions/creation/api-overview.mdx#update-subscription). - - - -{`const updateOptions = \{ - id: "", - query: "" - // ... -\} -await documentStore.subscriptions.update(updateOptions); -`} - - - - -{`// Available overloads: -update(options); -update(options, database); -`} - - - - - -## Drop connection - -Active subscription connections established by workers can be dropped remotely from the client. -Once dropped, the worker will not attempt to reconnect to the server. - - - -{`// Drop all connections to the subscription: -// ========================================= - -await documentStore.subscriptions.dropConnection("subscriptionName"); - -// Drop specific worker connection: -// =============================== - -const workerOptions = \{ - subscriptionName: "subscriptionName", - // ... -\}; - -const worker = documentStore.subscriptions.getSubscriptionWorker(workerOptions); - - -worker.on("batch", (batch, callback) => \{ - // worker processing logic -\}); - -await documentStore.subscriptions.dropConnection(worker); -`} - - - - -{`// Available overloads: -dropConnection(options); -dropConnection(options, database); -dropSubscriptionWorker(worker); -dropSubscriptionWorker(worker, database); -`} - - - - - -## Get subscriptions - -Get a list of all existing subscription tasks in the database. - - - -{`const subscriptions = await documentStore.subscriptions.getSubscriptions(0, 10); -`} - - - - -{`// Available overloads: -getSubscriptions(start, take); -getSubscriptions(start, take, database); -`} - - - - - -## Get subscription state - - - -{`const subscriptionState = - await documentStore.subscriptions.getSubscriptionState("subscriptionName"); -`} - - - - -{`// Available overloads: -getSubscriptionState(subscriptionName); -getSubscriptionState(subscriptionName, database); -`} - - - - - -##### SubscriptionState - -| Member | Type | Description | -|-------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **query** | `string` | Subscription's RQL like query. | -| **lastBatchAckTime** | `string` | Last time a batch processing progress was acknowledged. | -| **nodeTag** | `string` | Processing server's node tag. | -| **mentorNode** | `string` | The mentor node that was manually set. | -| **subscriptionName** | `string` | The subscription's name, which is also its unique identifier. | -| **subscriptionId** | `number` | Subscription's internal identifier (cluster's operation etag during subscription creation). | -| **changeVectorForNextBatchStartingPoint** | `string` | The Change Vector from which the subscription will begin sending documents.
This value is updated on batch acknowledgement and can also be set manually. | -| **disabled** | `boolean` | If `true`, subscription will not allow workers to connect. | -| **lastClientConnectionTime** | `string` | Time when last client was connected (value sustained after disconnection). | - - - - - -## DocumentSubscriptions class - -The `DocumentSubscriptions` class manages all interaction with the data subscriptions. -The class is available through the `subscriptions` property in the `documentStore`. - -| Method Signature | Return type | Description | -|----------------------------------------------------------|--------------------------------|--------------------------------------------------------------| -| **create(options)** | `Promise` | Create a new data subscription. | -| **create(options, database)** | `Promise` | Create a new data subscription. | -| **create(documentType)** | `Promise` | Create a new data subscription. | -| **create(optionsOrDocumentType, database)** | `Promise` | Create a new data subscription. | -| **createForRevisions(options)** | `Promise` | Create a new data subscription. | -| **createForRevisions(options, database)** | `Promise` | Create a new data subscription. | -| **delete(name)** | `Promise` | Delete subscription. | -| **delete(name, database)** | `Promise` | Delete subscription. | -| **dropConnection(name)** | `Promise` | Drop all existing subscription connections with workers. | -| **dropConnection(name, database)** | `Promise` | Drop all existing subscription connections with workers. | -| **dropSubscriptionWorker(worker, database)** | `Promise` | Drop an existing subscription connection with a worker. | -| **enable(name)** | `Promise` | Enable existing subscription. | -| **enable(name, database)** | `Promise` | Enable existing subscription. | -| **disable(name)** | `Promise` | Disable existing subscription. | -| **disable(name, database)** | `Promise` | Disable existing subscription. | -| **update(updateOptions)** | `Promise` | Update an existing data subscription. | -| **update(updateOptions, database)** | `Promise` | Update an existing data subscription. | -| **getSubscriptions(start, take)** | `Promise` | Returns subscriptions list. | -| **getSubscriptions(start, take, database)** | `Promise` | Returns subscriptions list. | -| **getSubscriptionState(subscriptionName)** | `Promise ` | Get the state of a specific subscription. | -| **getSubscriptionState(subscriptionName, database)** | `Promise ` | Get the state of a specific subscription. | -| **getSubscriptionWorker(options)** | `SubscriptionWorker` | Generate a subscription worker. | -| **getSubscriptionWorker(options, database)** | `SubscriptionWorker` | Generate a subscription worker. | -| **getSubscriptionWorker(subscriptionName)** | `SubscriptionWorker` | Generate a subscription worker. | -| **getSubscriptionWorker(subscriptionName, database)** | `SubscriptionWorker` | Generate a subscription worker. | -| **getSubscriptionWorkerForRevisions(options)** | `SubscriptionWorker` | Generate a subscription worker for a revisions subscription. | -| **getSubscriptionWorkerForRevisions(options, database)** | `SubscriptionWorker` | Generate a subscription worker for a revisions subscription. | - - - - diff --git a/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-csharp.mdx b/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-csharp.mdx deleted file mode 100644 index 25f8ad1e10..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-csharp.mdx +++ /dev/null @@ -1,312 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* When the [Revisions feature](../../../../document-extensions/revisions/overview.mdx) is enabled, a document revision is created with each change made to the document. - Each revision contains a snapshot of the document at the time of modification, forming a complete audit trail. - -* The **Data Subscription** feature supports subscribing not only to documents but also to their **revisions**. - This functionality allows the subscribed client to track changes made to documents over time. - -* The revisions support is specified within the subscription definition. - See how to create and consume it in the examples below. - -* In this page: - * [Regular subscription vs Revisions subscription](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#regular-subscription-vs-revisions-subscription) - * [Revisions processing order](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#revisions-processing-order) - * [Simple creation and consumption](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#simple-creation-and-consumption) - * [Filtering revisions](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#filtering-revisions) - * [Projecting fields from revisions](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#projecting-fields-from-revisions) - - -## Regular subscription vs Revisions subscription - - - -##### Regular subscription -* **Processed items**: - The subscription processes **documents** from the defined collection. - Only the latest version of the document is processed, even if the document has revisions. -* **Query access scope**: - The subscription query running on the server has access only to the latest/current version of the documents. -* **Data sent to client**: - Each item in the batch sent to the client contains a single document (or a projection of it), - as defined in the subscription. - - - - -##### Revisions subscription -* **Processed items**: - The subscription processes all **revisions** of documents from the defined collection, - including revisions of deleted documents from the revision bin if they have not been purged. -* **Query access scope**: - For each revision, the subscription query running on the server has access to both the currently processed revision and its previous revision. -* **Data sent to client**: - By default, unless the subscription query is [projecting specific fields](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#projecting-fields-from-revisions), - each item in the batch sent to the client contains both the processed revision (`Result.Current`) and its preceding revision (`Result.Previous`). - If the document has just been created, the previous revision will be `null`. - - -* In order for the revisions subscription to work, - [Revisions must be configured](../../../../document-extensions/revisions/overview.mdx#defining-a-revisions-configuration) and enabled for the collection the subscription manages. - -* A document that has no revisions will Not be processed, - so make sure that your revisions configuration does not purge revisions before the subscription has a chance to process them. - - - - - - -## Revisions processing order - -In the revisions subscription, revisions are processed in pairs of subsequent entries. -For example, consider the following User document: - - - -{`\{ - Name: "James", - Age: "21" -\} -`} - - - -We update this User document in two consecutive operations: - -* Update the 'Age' field to the value of 22 -* Update the 'Age' field to the value of 23 - -The subscription worker in the client will receive pairs of revisions ( _Previous_ & _Current_ ) -within each item in the batch in the following order: - -| Batch item | Previous | Current | -|------------|--------------------------------|--------------------------------| -| item #1 | `null` | `{ Name: "James", Age: "21" }` | -| item #2 | `{ Name: "James", Age: "21" }` | `{ Name: "James", Age: "22" }` | -| item #3 | `{ Name: "James", Age: "22" }` | `{ Name: "James", Age: "23" }` | - - - -## Simple creation and consumption - -Here we set up a basic revisions subscription that will deliver pairs of consecutive _Order_ document revisions to the client: - -**Create subscription**: - - - - -{`subscriptionName = store.Subscriptions.Create( - // Use > as the type for the processed items - // e.g. > - new SubscriptionCreationOptions>()); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - // Add (Revisions = true) to your subscription RQL - Query = @"From Orders (Revisions = true)" -}); -`} - - - - -**Consume subscription**: - - - -{`SubscriptionWorker> revisionsWorker = - // Specify > as the type of the processed items - store.Subscriptions.GetSubscriptionWorker>(subscriptionName); - -await revisionsWorker.Run((SubscriptionBatch> batch) => -\{ - foreach (var item in batch.Items) - \{ - // Access the previous revision via 'Result.Previous' - var previousRevision = item.Result.Previous; - - // Access the current revision via 'Result.Current' - var currentRevision = item.Result.Current; - - // Provide your own processing logic: - ProcessOrderRevisions(previousRevision, currentRevision); - \} -\}); -`} - - - - - -## Filtering revisions - -Here we set up a revisions subscription that will send the client only document revisions in which the order was shipped to Mexico. - -**Create subscription**: - - - - -{`subscriptionName = store.Subscriptions.Create( - // Specify > as the type of the processed items - new SubscriptionCreationOptions>() - { - // Provide filtering logic - // Only revisions that where shipped to Mexico will be sent to subscribed clients - Filter = revision => revision.Current.ShipTo.Country == "Mexico", - }); -`} - - - - -{`subscriptionName = await store.Subscriptions.CreateAsync(new SubscriptionCreationOptions() -{ - Query = @"declare function isSentToMexico(doc) { - return doc.Current.ShipTo.Country == 'Mexico' - } - - from 'Orders' (Revisions = true) as doc - where isSentToMexico(doc) == true" -}); -`} - - - - -**Consume subscription**: - - - -{`SubscriptionWorker> worker = - store.Subscriptions.GetSubscriptionWorker>(subscriptionName); - -await worker.Run(batch => -\{ - foreach (var item in batch.Items) - \{ - Console.WriteLine($@" - This is a revision of document \{item.Id\}. - The order in this revision was shipped at \{item.Result.Current.ShippedAt\}."); - \} -\}); -`} - - - - - -## Projecting fields from revisions - -Here we define a revisions subscription that will filter the revisions and send projected data to the client. - -**Create subscription**: - - - - -{`subscriptionName = store.Subscriptions.Create( - // Specify > as the type of the processed items within the query - new SubscriptionCreationOptions>() - { - // Filter revisions by the revenue delta. - // The subscription will only process revisions where the revenue - // is higher than in the preceding revision by 2500. - Filter = revision => - revision.Previous != null && - revision.Current.Lines.Sum(x => x.PricePerUnit * x.Quantity) > - revision.Previous.Lines.Sum(x => x.PricePerUnit * x.Quantity) + 2500, - - // Define the projected fields that will be sent to the client - Projection = revision => new OrderRevenues() - { - PreviousRevenue = - revision.Previous.Lines.Sum(x => x.PricePerUnit * x.Quantity), - - CurrentRevenue = - revision.Current.Lines.Sum(x => x.PricePerUnit * x.Quantity) - } - }); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"declare function isRevenueDeltaAboveThreshold(doc, threshold) { - return doc.Previous !== null && doc.Current.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0) > doc.Previous.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0) + threshold - } - - from 'Orders' (Revisions = true) as doc - where isRevenueDeltaAboveThreshold(doc, 2500) - - select { - PreviousRevenue: doc.Previous.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0), - - CurrentRevenue: doc.Current.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0) - }" -}); -`} - - - - -{`public class OrderRevenues -{ - public decimal PreviousRevenue { get; set; } - public decimal CurrentRevenue { get; set; } -} -`} - - - - -**Consume subscription**: - -Since the revision fields are projected into the `OrderRevenues` class in the subscription definition, -each item received in the batch has the format of this projected class instead of the default `Result.Previous` and `Result.Current` fields, -as was demonstrated in the [simple example](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#simple-creation-and-consumption). - - - -{`SubscriptionWorker revenuesComparisonWorker = - // Use the projected class type 'OrderRevenues' for the items the worker will process - store.Subscriptions.GetSubscriptionWorker(subscriptionName); - -await revenuesComparisonWorker.Run(batch => -\{ - foreach (var item in batch.Items) - \{ - // Access the projected content: - Console.WriteLine($@"Revenue for order with ID: \{item.Id\} - has grown from \{item.Result.PreviousRevenue\} - to \{item.Result.CurrentRevenue\}"); - \} -\}); -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-java.mdx b/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-java.mdx deleted file mode 100644 index f0278bbcda..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-java.mdx +++ /dev/null @@ -1,147 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* The **Data Subscription** feature supports subscribing not only to documents, but also to [document revisions](../../../../document-extensions/revisions/overview.mdx). - -* The revisions support is defined within the subscription. - A [Revisions Configuration](../../../../document-extensions/revisions/client-api/operations/configure-revisions.mdx) must be defined for the subscribed collection. - -* While a regular subscription processes a single document, a Revisions subscription processes **pairs of subsequent document revisions**. - - Using this functionality allows you to keep track of each change made in a document, as well as compare pairs of subsequent versions of the document. - - Both revisions are accessible for filtering and projection. - -* In this page: - * [Revisions processing order](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#revisions-processing-order) - * [Simple declaration and usage](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#simple-declaration-and-usage) - * [Revisions processing and projection](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#revisions-processing-and-projection) - - -## Revisions processing order - -The Revisions feature allows the tracking of changes made in a document, by storing the audit trail of its changes over time. -An audit trail entry is called a **Document Revision**, and is comprised of a document snapshot. -Read more about revisions [here](../../../../document-extensions/revisions/overview.mdx). - -In a data subscription, revisions will be processed in pairs of subsequent entries. -For example, consider the following User document: - -`{ - Name:'James', - Age:'21' -}` - -We update the User document twice, in separate operations: - -* We update the 'Age' field to the value of 22 -* We update the 'Age' field to the value of 23 - -The data subscriptions revisions processing mechanism will receive pairs of revisions in the following order: - -| # | Previous | Current | -|---|------------------------------|------------------------------| -| 1 | `null` | `{ Name:'James', Age:'21' }` | -| 2 | `{ Name:'James', Age:'21' }` | `{ Name:'James', Age:'22' }` | -| 3 | `{ Name:'James', Age:'22' }` | `{ Name:'James', Age:'23' }` | - - -The revisions subscription will be able to function properly only if the revisions it needs to process are available. -Please make sure that your revisions configuration doesn't purge revisions before the subscription had the chance to process them. - - - - -## Simple declaration and usage - -Here we declare a simple revisions subscription that will send pairs of subsequent document revisions to the client: - -Creation: - - - -{`name = store.subscriptions().createForRevisions(Order.class); -`} - - - - -{`SubscriptionCreationOptions options = new SubscriptionCreationOptions(); -options.setQuery("from orders (Revisions = true)"); -name = store.subscriptions().createForRevisions(Order.class, options); -`} - - - - -Consumption: - - -{`SubscriptionWorker> revisionWorker = store - .subscriptions().getSubscriptionWorkerForRevisions(Order.class, name); -revisionWorker.run(x -> \{ - for (SubscriptionBatch.Item> documentsPair : x.getItems()) \{ - - Order prev = documentsPair.getResult().getPrevious(); - Order current = documentsPair.getResult().getCurrent(); - - processOrderChanges(prev, current); - \} -\}); -`} - - - - - -## Revisions processing and projection - -Here we declare a revisions subscription that will filter and project data from revisions pairs: - -Creation: - - -{`SubscriptionCreationOptions options = new SubscriptionCreationOptions(); -options.setQuery("declare function getOrderLinesSum(doc) \{" + - " var sum = 0;" + - " for (var i in doc.Lines) \{ sum += doc.Lines[i]; \} " + - " return sum;" + - "\}" + - "" + - " from orders (Revisions = true) " + - " where getOrderLinesSum(this.Current) > getOrderLinesSum(this.Previous) " + - " select \{" + - " previousRevenue: getOrderLinesSum(this.Previous)," + - " currentRevenue: getOrderLinesSum(this.Current)" + - "\}"); - -name = store.subscriptions().create(options); -`} - - - -Consumption: - - -{`SubscriptionWorker> revisionWorker = store - .subscriptions().getSubscriptionWorkerForRevisions(Order.class, name); -revisionWorker.run(x -> \{ - for (SubscriptionBatch.Item> documentsPair : x.getItems()) \{ - - Order prev = documentsPair.getResult().getPrevious(); - Order current = documentsPair.getResult().getCurrent(); - - processOrderChanges(prev, current); - \} -\}); -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-nodejs.mdx b/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-nodejs.mdx deleted file mode 100644 index ccb9d4e4bb..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/content/_subscription-with-revisioning-nodejs.mdx +++ /dev/null @@ -1,287 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* When the [Revisions feature](../../../../document-extensions/revisions/overview.mdx) is enabled, a document revision is created with each change made to the document. - Each revision contains a snapshot of the document at the time of modification, forming a complete audit trail. - -* The **Data Subscription** feature supports subscribing not only to documents but also to their **revisions**. - This functionality allows the subscribed client to track changes made to documents over time. - -* The revisions support is specified within the subscription definition. - See how to create and consume it in the examples below. - -* In this page: - * [Regular subscription vs Revisions subscription](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#regular-subscription-vs-revisions-subscription) - * [Revisions processing order](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#revisions-processing-order) - * [Simple creation and consumption](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#simple-creation-and-consumption) - * [Filtering revisions](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#filtering-revisions) - * [Projecting fields from revisions](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#projecting-fields-from-revisions) - - -## Regular subscription vs Revisions subscription - - - -##### Regular subscription -* **Processed items**: - The subscription processes **documents** from the defined collection. - Only the latest version of the document is processed, even if the document has revisions. -* **Query access scope**: - The subscription query running on the server has access only to the latest/current version of the documents. -* **Data sent to client**: - Each item in the batch sent to the client contains a single document (or a projection of it), - as defined in the subscription. - - - - -##### Revisions subscription -* **Processed items**: - The subscription processes all **revisions** of documents from the defined collection, - including revisions of deleted documents from the revision bin if they have not been purged. -* **Query access scope**: - For each revision, the subscription query running on the server has access to both the currently processed revision and its previous revision. -* **Data sent to client**: - By default, unless the subscription query is projecting specific fields, - each item in the batch sent to the client contains both the processed revision (`result.current`) and its preceding revision (`result.previous`). - If the document has just been created, the previous revision will be `null`. - - -* In order for the revisions subscription to work, - [Revisions must be configured](../../../../document-extensions/revisions/overview.mdx#defining-a-revisions-configuration) and enabled for the collection the subscription manages. - -* A document that has no revisions will Not be processed, - so make sure that your revisions configuration does not purge revisions before the subscription has a chance to process them. - - - - - - -## Revisions processing order - -In the revisions subscription, revisions are processed in pairs of subsequent entries. -For example, consider the following User document: - - - -{`\{ - Name: "James", - Age: "21" -\} -`} - - - -We update this User document in two consecutive operations: - -* Update the 'Age' field to the value of 22 -* Update the 'Age' field to the value of 23 - -The subscription worker in the client will receive pairs of revisions ( _previous_ & _current_ ) -within each item in the batch in the following order: - -| Batch item | Previous | Current | -|------------|--------------------------------|--------------------------------| -| item #1 | `null` | `{ Name: "James", Age: "21" }` | -| item #2 | `{ Name: "James", Age: "21" }` | `{ Name: "James", Age: "22" }` | -| item #3 | `{ Name: "James", Age: "22" }` | `{ Name: "James", Age: "23" }` | - - - -## Simple creation and consumption - -Here we set up a basic revisions subscription that will deliver pairs of consecutive _Order_ document revisions to the client: - -**Create subscription**: - - - -{`const subscriptionName = await documentStore.subscriptions.create(\{ - // Add (Revisions = true) to your subscription RQL - query: "From Orders (Revisions = true)" -\}); -`} - - - -**Consume subscription**: - - - -{`const workerOptions = \{ subscriptionName \}; - -const worker = - // Use method \`getSubscriptionWorkerForRevisions\` - documentStore.subscriptions.getSubscriptionWorkerForRevisions(workerOptions); - -worker.on("batch", (batch, callback) => \{ - try \{ - for (const item of batch.items) \{ - - // Access the previous revision via 'result.previous' - const previousRevision = item.result.previous; - - // Access the current revision via 'result.current' - const currentRevision = item.result.current; - \} - callback(); - - \} catch (err) \{ - callback(err); - \} -\}); -`} - - - - - -## Filtering revisions - -Here we set up a revisions subscription that will send the client only document revisions in which the order was shipped to Mexico. - -**Create subscription**: - - - -{`const subscriptionName = await documentStore.subscriptions.create(\{ - // Provide filtering logic - // Only revisions that where shipped to Mexico will be sent to subscribed clients - query: \`declare function isSentToMexico(doc) \{ - return doc.Current.ShipTo.Country == 'Mexico' - \} - - from 'Orders' (Revisions = true) as doc - where isSentToMexico(doc) == true\` -\}); -`} - - - -**Consume subscription**: - - - -{`const workerOptions = \{ subscriptionName \}; - -const worker = - documentStore.subscriptions.getSubscriptionWorkerForRevisions(workerOptions); - -worker.on("batch", (batch, callback) => \{ - try \{ - for (const item of batch.items) \{ - console.log(\` - This is a revision of document $\{item.id\}. - The order in this revision was shipped at $\{item.result.current.ShippedAt\}. - \`); - \} - callback(); - - \} catch (err) \{ - callback(err); - \} -\}); -`} - - - - - -## Projecting fields from revisions - -Here we define a revisions subscription that will filter the revisions and send projected data to the client. - -**Create subscription**: - - - - -{`const subscriptionName = await documentStore.subscriptions.create({ - // Filter revisions by the revenue delta. - // The subscription will only process revisions where the revenue - // is higher than in the preceding revision by 2500. - - query: \`declare function isRevenueDeltaAboveThreshold(doc, threshold) { - return doc.Previous !== null && doc.Current.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0) > doc.Previous.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0) + threshold - } - - from 'Orders' (Revisions = true) as doc - where isRevenueDeltaAboveThreshold(doc, 2500) - - // Define the projected fields that will be sent to the client: - select { - previousRevenue: doc.Previous.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0), - - currentRevenue: doc.Current.Lines.map(function(x) { - return x.PricePerUnit * x.Quantity; - }).reduce((a, b) => a + b, 0) - }\` -}); -`} - - - - -{`class OrderRevenues { - constructor() { - this.previousRevenue; - this.currentRevenue; - } -} -`} - - - - -**Consume subscription**: - -Since the revision fields are projected into the `OrderRevenues` class in the subscription definition, -each item received in the batch has the format of this projected class instead of the default `result.previous` and `result.current` fields, -as was demonstrated in the [simple example](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx#simple-creation-and-consumption). - - - -{`const workerOptions = \{ - subscriptionName: subscriptionName, - documentType: OrderRevenues -\}; - -const worker = - // Note: in this case, where each resulting item in the batch is a projected object - // and not the revision itself, we use method \`getSubscriptionWorker\` - documentStore.subscriptions.getSubscriptionWorker(workerOptions); - -worker.on("batch", (batch, callback) => \{ - try \{ - for (const item of batch.items) \{ - // Access the projected content: - console.log(\` - Revenue for order with ID: $\{item.id\} - has grown from $\{item.result.previousRevenue\} - to $\{item.result.currentRevenue\} - \`); - \} - callback(); - - \} catch (err) \{ - callback(err); - \} -\}); -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx b/docs/client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx deleted file mode 100644 index ea499c8a05..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: "Data Subscriptions: Maintenance Operations" -sidebar_label: Maintenance Operations -description: "Manage RavenDB data subscriptions with maintenance operations to get subscription state, drop connections, and list active subscriptions." -sidebar_position: 1 -supported_languages: ["csharp", "java", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import MaintenanceOperationsCsharp from './content/_maintenance-operations-csharp.mdx'; -import MaintenanceOperationsJava from './content/_maintenance-operations-java.mdx'; -import MaintenanceOperationsNodejs from './content/_maintenance-operations-nodejs.mdx'; - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx b/docs/client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx deleted file mode 100644 index e6e8a2b115..0000000000 --- a/docs/client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: "Data Subscriptions: Revisions Support" -sidebar_label: Revisions Support -description: "Create RavenDB data subscriptions that process document revisions, receiving both previous and current versions of changed documents." -sidebar_position: 0 -supported_languages: ["csharp", "java", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" - - title: "Revisions API Overview" - link: "document-extensions/revisions/client-api/overview" - source: "docs" - path: "Document Extensions > Revisions > Client API" - - title: "Revisions Overview" - link: "document-extensions/revisions/overview" - source: "docs" - path: "Document Extensions > Revisions" - - title: "JavaScript Engine" - link: "server/kb/javascript-engine" - source: "docs" - path: "Server > Knowledge Base" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import SubscriptionWithRevisioningCsharp from './content/_subscription-with-revisioning-csharp.mdx'; -import SubscriptionWithRevisioningJava from './content/_subscription-with-revisioning-java.mdx'; -import SubscriptionWithRevisioningNodejs from './content/_subscription-with-revisioning-nodejs.mdx'; - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/assets/SubscriptionsDocumentProcessing.png b/docs/client-api/data-subscriptions/assets/SubscriptionsDocumentProcessing.png deleted file mode 100644 index f1fc7883d2..0000000000 Binary files a/docs/client-api/data-subscriptions/assets/SubscriptionsDocumentProcessing.png and /dev/null differ diff --git a/docs/client-api/data-subscriptions/concurrent-subscriptions.mdx b/docs/client-api/data-subscriptions/concurrent-subscriptions.mdx deleted file mode 100644 index 8838143157..0000000000 --- a/docs/client-api/data-subscriptions/concurrent-subscriptions.mdx +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "Concurrent Subscriptions" -sidebar_label: Concurrent Subscriptions -description: "Run multiple concurrent workers on a single RavenDB data subscription to parallelize document processing across clients." -sidebar_position: 4 -supported_languages: ["csharp", "nodejs"] -see_also: - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" - - title: "Maintenance Operations" - link: "client-api/data-subscriptions/advanced-topics/maintenance-operations" - source: "docs" - path: "Client API > Data Subscriptions > Advanced topics" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import ConcurrentSubscriptionsCsharp from './content/_concurrent-subscriptions-csharp.mdx'; -import ConcurrentSubscriptionsNodejs from './content/_concurrent-subscriptions-nodejs.mdx'; - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/_category_.json b/docs/client-api/data-subscriptions/consumption/_category_.json deleted file mode 100644 index 4f2eeb38b4..0000000000 --- a/docs/client-api/data-subscriptions/consumption/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "position": 2, - "label": Consumption, -} \ No newline at end of file diff --git a/docs/client-api/data-subscriptions/consumption/api-overview.mdx b/docs/client-api/data-subscriptions/consumption/api-overview.mdx deleted file mode 100644 index 05101b5843..0000000000 --- a/docs/client-api/data-subscriptions/consumption/api-overview.mdx +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "Consume Subscriptions API" -sidebar_label: API Overview -description: "API reference for consuming RavenDB data subscriptions, covering SubscriptionWorker options, batch handling, and error strategies." -sidebar_position: 2 -supported_languages: ["csharp", "java", "python", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import ApiOverviewCsharp from './content/_api-overview-csharp.mdx'; -import ApiOverviewJava from './content/_api-overview-java.mdx'; -import ApiOverviewPython from './content/_api-overview-python.mdx'; -import ApiOverviewNodejs from './content/_api-overview-nodejs.mdx'; - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_api-overview-csharp.mdx b/docs/client-api/data-subscriptions/consumption/content/_api-overview-csharp.mdx deleted file mode 100644 index d172cdbad9..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_api-overview-csharp.mdx +++ /dev/null @@ -1,262 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Create the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) - * [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) - * [Run the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker) - * [SubscriptionBatch<T>](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionbatcht) - * [SubscriptionBatch<T>.Item](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionbatchtitem) - * [SubscriptionWorker<T>](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkert) - - -## Create the subscription worker - -A subscription worker can be created using the following `GetSubscriptionWorker` methods available through the `Subscriptions` property of the `DocumentStore`. - -Note: Simply creating the worker is insufficient; -after creating the worker, you need to [run the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker) to initiate document processing. - - - -{`SubscriptionWorker GetSubscriptionWorker( - string subscriptionName, string database = null); - -SubscriptionWorker GetSubscriptionWorker( - SubscriptionWorkerOptions options, string database = null); - -SubscriptionWorker GetSubscriptionWorker( - string subscriptionName, string database = null) where T : class; - -SubscriptionWorker GetSubscriptionWorker( - SubscriptionWorkerOptions options, string database = null) where T : class; -`} - - - -| Parameter | Type | Description | -|----------------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------| -| **subscriptionName** | `string` | The name of the subscription to which the worker will connect. | -| **options** | `SubscriptionWorkerOptions` | Options that affect how the worker interacts with the subscription. These options do not alter the definition of the subscription itself. | -| **database** | `string` | The name of the database where the subscription task resides.
If `null`, the default database configured in DocumentStore will be used. | - -| Return value | | -|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `SubscriptionWorker` | The subscription worker that has been created.
Initially, it is idle and will only start processing documents when the `Run` function is called. | - - - -## SubscriptionWorkerOptions - - - -{`public class SubscriptionWorkerOptions -\{ - public string SubscriptionName \{ get; \} - public int MaxDocsPerBatch \{ get; set; \} - public int SendBufferSizeInBytes \{ get; set; \} - public int ReceiveBufferSizeInBytes \{ get; set; \} - public bool IgnoreSubscriberErrors \{ get; set; \} - public bool CloseWhenNoDocsLeft \{ get; set; \} - public TimeSpan TimeToWaitBeforeConnectionRetry \{ get; set; \} - public TimeSpan ConnectionStreamTimeout \{ get; set; \} - public TimeSpan MaxErroneousPeriod \{ get; set; \} - public SubscriptionOpeningStrategy Strategy \{ get; set; \} -\} -`} - - - -When creating a worker with `SubscriptionWorkerOptions`, the only mandatory property is `SubscriptionName`. -All other parameters are optional and will default to their respective default values if not specified. - -| Member | Type | Description | -|-------------------------------------|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **SubscriptionName** | `string` | The name of the subscription to which the worker will connect. | -| **MaxDocsPerBatch** | `int` | The maximum number of documents that the server will try to retrieve and send to the client in a batch. If the server doesn't find as many documents as specified, it will send the documents it has found without waiting. Default: 4096. | -| **SendBufferSizeInBytes** | `int` | The size in bytes of the TCP socket buffer used for _sending_ data.
Default: 32,768 bytes (32 KiB). | -| **ReceiveBufferSizeInBytes** | `int` | The size in bytes of the TCP socket buffer used for _receiving_ data.
Default: 4096 (4 KiB). | -| **IgnoreSubscriberErrors** | `bool` | Determines if subscription processing is aborted when the worker's batch-handling code throws an unhandled exception.

`true` – subscription processing will continue.

`false` (Default) – subscription processing will be aborted. | -| **CloseWhenNoDocsLeft** | `bool` | Determines whether the subscription connection closes when no new documents are available.

`true` – The subscription worker processes all available documents and stops when none remain, at which point the `Run` method throws a `SubscriptionClosedException`.
Useful for ad-hoc, one-time processing.

`false` (Default) – The subscription worker remains active, waiting for new documents. | -| **TimeToWaitBeforeConnectionRetry** | `TimeSpan` | The time to wait before attempting to reconnect after a non-aborting failure during subscription processing. Default: 5 seconds. | -| **MaxErroneousPeriod** | `TimeSpan` | The maximum amount of time a subscription connection can remain in an erroneous state before it is terminated. Default: 5 minutes. | -| **Strategy** | `SubscriptionOpeningStrategy` | This enum configures how the server handles connection attempts from workers to a specific subscription task.
Default: `OpenIfFree`. | - -Learn more about `SubscriptionOpeningStrategy` in [worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies). - - - -{`public enum SubscriptionOpeningStrategy -\{ - // Connect if no other worker is connected - OpenIfFree, - - // Take over the connection - TakeOver, - - // Wait for currently connected worker to disconnect - WaitForFree, - - // Connect concurrently - Concurrent -\} -`} - - - - - -## Run the subscription worker - -After [creating](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) a subscription worker, the subscription worker is still not processing any documents. -To start processing, you need to call the `Run` method of the [SubscriptionWorker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkert). - -The `Run` function takes a delegate, which is your client-side code responsible for processing the received document batches. - - - -{`Task Run(Action> processDocuments, - CancellationToken ct = default(CancellationToken)); - -Task Run(Func, Task> processDocuments, - CancellationToken ct = default(CancellationToken)); -`} - - - -| Parameter | Type | Description | -|----------------------|------------------------------------|----------------------------------------------------------------| -| **processDocuments** | `Action>` | Delegate for sync batches processing. | -| **processDocuments** | `Func, Task>` | Delegate for async batches processing. | -| **ct** | `CancellationToken` | Cancellation token used in order to halt the worker operation. | - -| Return value | | -|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `Task` | Task that is alive as long as the subscription worker is processing or tries processing.
If the processing is aborted, the task exits with an exception. | - - - -## SubscriptionBatch<T> - -| Member | Type | Description | -|--------------------------|-----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Items** | `List.Item>` | List of items in the batch.
See [SubscriptionBatch<T>.Item](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionbatchtitem) below. | -| **NumberOfItemsInBatch** | `int` | Number of items in the batch. | - -| Method Signature | Return value | Description | -|------------------------|-------------------------|-------------------------------------------------------------------------------------------------------------------| -| **OpenSession()** | `IDocumentSession` | Open a new document session that tracks all items and their included items within the current batch. | -| **OpenAsyncSession()** | `IAsyncDocumentSession` | Open a new asynchronous document session that tracks all items and their included items within the current batch. | - - - -##### Subscription worker connectivity - -As long as there is no exception, the worker will continue addressing the same server that the first batch was received from. -If the worker fails to reach that node, it will try to [failover](../../../../client-api/configuration/load-balance/overview.mdx) to another node from the session's topology list. -The node that the worker succeeded connecting to, will inform the worker which node is currently responsible for data subscriptions. - - - - - -## SubscriptionBatch<T>.Item - -This class represents a single item in a subscription batch results. - - - -{`public struct Item -\{ - public T Result \{ get; internal set; \} - public string ExceptionMessage \{ get; internal set; \} - public string Id \{ get; internal set; \} - public string ChangeVector \{ get; internal set; \} - public bool Projection \{ get; internal set; \} - public bool Revision \{ get; internal set; \} - public BlittableJsonReaderObject RawResult \{ get; internal set; \} - public BlittableJsonReaderObject RawMetadata \{ get; internal set; \} - public IMetadataDictionary Metadata \{ get; internal set; \} -\} -`} - - - -| Member | Type | Description | -|----------------------|-----------------------------|-------------------------------------------------------------------------------------------------------| -| **Result** | `T` | The current batch item.
If `T` is `BlittableJsonReaderObject`, no deserialization will take place. | -| **ExceptionMessage** | `string` | The exception message thrown during current document processing in the server side. | -| **Id** | `string` | The document ID of the underlying document for the current batch item. | -| **ChangeVector** | `string` | The change vector of the underlying document for the current batch item. | -| **RawResult** | `BlittableJsonReaderObject` | Current batch item before serialization to `T`. | -| **RawMetadata** | `BlittableJsonReaderObject` | Current batch item's underlying document metadata. | -| **Metadata** | `IMetadataDictionary` | Current batch item's underlying metadata values. | - - -This class should only be used within the subscription's `Run` delegate. -Using it outside this scope may cause unexpected behavior. - - - - -## SubscriptionWorker<T> - - - -##### Methods - -| Method Signature | Return Type | Description | -|------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Dispose()** | `void` | Aborts subscription worker operation ungracefully by waiting for the task returned by the `Run` function to finish running. | -| **DisposeAsync()** | `Task` | Async version of `Dispose()`. | -| **Dispose(bool waitForSubscriptionTask)** | `void` | Aborts the subscription worker, but allows deciding whether to wait for the `Run` function task or not. | -| **DisposeAsync(bool waitForSubscriptionTask)** | `Task` | Async version of `DisposeAsync(bool waitForSubscriptionTask)`. | -| **Run (multiple overloads)** | `Task` | Call `Run` to begin the worker's batch processing.
Pass the batch processing delegates to this method
(see [above](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker)). | - - - - - -##### Events - -| Event | Event type | Description | -|-----------------------------------|:--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **AfterAcknowledgment** | `AfterAcknowledgmentAction` | Triggered after each time the server acknowledges the progress of batch processing. | -| **OnSubscriptionConnectionRetry** | `Action` | Triggered when the subscription worker attempts to reconnect to the server after a failure.
The event receives as a parameter the exception that interrupted the processing. | -| **OnDisposed** | `Action>` | Triggered after the subscription worker is disposed. | - - - -##### AfterAcknowledgmentAction - -| Parameter | | | -|-------------|------------------------|------------------------------------------| -| **batch** | `SubscriptionBatch` | The batch process which was acknowledged | - -| Return value | | -|----------------|---------------------------------------------------------------------------------------------------------| -| `Task` | Task for which the worker will wait for the event processing to be finished (for async functions, etc.) | - - - - - - - -##### Properties - -| Member | Type | Description | -|-------------------------------|----------|-----------------------------------------------------------------------| -| **CurrentNodeTag** | `string` | The node tag of the current RavenDB server handling the subscription. | -| **SubscriptionName** | `string` | The name of the currently processed subscription. | -| **WorkerId** | `string` | The worker ID. | - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_api-overview-java.mdx b/docs/client-api/data-subscriptions/consumption/content/_api-overview-java.mdx deleted file mode 100644 index 57f4671bf4..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_api-overview-java.mdx +++ /dev/null @@ -1,175 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Create the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) - * [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) - * [Run the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker) - * [SubscriptionBatch<T>](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionbatcht) - * [SubscriptionWorker<T>](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkert) - - -## Create the subscription worker - -Subscription worker generation is accessible through the `DocumentStore`'s `subscriptions()` method, of type `DocumentSubscriptions`: - - -{`SubscriptionWorker getSubscriptionWorker(SubscriptionWorkerOptions options); -SubscriptionWorker getSubscriptionWorker(SubscriptionWorkerOptions options, String database); - -SubscriptionWorker getSubscriptionWorker(String subscriptionName); -SubscriptionWorker getSubscriptionWorker(String subscriptionName, String database); - - SubscriptionWorker getSubscriptionWorker(Class clazz, SubscriptionWorkerOptions options); - SubscriptionWorker getSubscriptionWorker(Class clazz, SubscriptionWorkerOptions options, String database); - - SubscriptionWorker getSubscriptionWorker(Class clazz, String subscriptionName); - SubscriptionWorker getSubscriptionWorker(Class clazz, String subscriptionName, String database); - - SubscriptionWorker> getSubscriptionWorkerForRevisions(Class clazz, SubscriptionWorkerOptions options); - SubscriptionWorker> getSubscriptionWorkerForRevisions(Class clazz, SubscriptionWorkerOptions options, String database); - - SubscriptionWorker> getSubscriptionWorkerForRevisions(Class clazz, String subscriptionName); - SubscriptionWorker> getSubscriptionWorkerForRevisions(Class clazz, String subscriptionName, String database); -`} - - - -| Parameter | | | -|----------------------|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **subscriptionName** | `String` | The subscription's name. This parameter appears in more simple overloads allowing to start processing without creating a `SubscriptionCreationOptions` instance, relying on the default values | -| **options** | `SubscriptionWorkerOptions` | Options that affect how the worker interacts with the subscription. These options do not alter the definition of the subscription itself. | -| **database** | `String` | The name of the database where the subscription task resides. If `null`, the default database configured in DocumentStore will be used. | - -| Return value | | -|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `SubscriptionWorker` | The subscription worker that has been created.
Initially, it is idle and will only start processing documents when the `run` function is called. | - - - - -## SubscriptionWorkerOptions - -When creating a worker with `SubscriptionWorkerOptions`, the only mandatory property is `subscriptionName`. -All other parameters are optional and will default to their respective default values if not specified. - - -| Member | Type | Description | -|-------------------------------------|-----------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **subscriptionName** | `String` | The name of the subscription to which the worker will connect. | -| **timeToWaitBeforeConnectionRetry** | `Duration` | The time to wait before attempting to reconnect after a non-aborting failure during subscription processing. Default: 5 seconds. | -| **ignoreSubscriberErrors** | `boolean` | Determines if subscription processing is aborted when the worker's batch-handling code throws an unhandled exception.

`true` – subscription processing will continue.

`false` (Default) – subscription processing will be aborted. | -| **strategy** | `SubscriptionOpeningStrategy`
(enum) | Configures how the server handles connection attempts from workers to a specific subscription task.
Learn more in [worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies).
Default: `OPEN_IF_FREE`. | -| **maxDocsPerBatch** | `int` | The maximum number of documents that the server will try to retrieve and send to the client in a batch. If the server doesn't find as many documents as specified, it will send the documents it has found without waiting. Default: 4096. | -| **closeWhenNoDocsLeft** | `boolean` | Determines whether the subscription connection closes when no new documents are available.

`true` – The subscription worker processes all available documents and stops when none remain, at which point the `run` method throws a `SubscriptionClosedException`.
Useful for ad-hoc, one-time processing.

`false` (Default) – The subscription worker remains active, waiting for new documents. | -| **sendBufferSizeInBytes** | `int` | The size in bytes of the TCP socket buffer used for _sending_ data.
Default: 32,768 bytes (32 KiB). | -| **receiveBufferSizeInBytes** | `int` | The size in bytes of the TCP socket buffer used for _receiving_ data.
Default: 4096 (4 KiB). | - - - -## Run the subscription worker - -After [creating](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) a subscription worker, the subscription worker is still not processing any documents. -To start processing, you need to call the `run` method of the [SubscriptionWorker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkert). - -The `run` function takes a delegate, which is your client-side code responsible for processing the received document batches. - - - -{`CompletableFuture run(Consumer> processDocuments); -`} - - - -| Parameter | | | -|----------------------|----------------------------------|--------------------------------------| -| **processDocuments** | `Consumer>` | Delegate for sync batches processing | - -| Return value | | -|---------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `CompletableFuture` | Task that is alive as long as the subscription worker is processing or tries processing. If the processing is aborted, the future exits with an exception | - - - - -## SubscriptionBatch<T> - -| Member | Type | Description | -|--------------------------|-----------------------------------|-------------------------------| -| **items** | `List.Item>` | List of items in the batch. | -| **numberOfItemsInBatch** | `int` | Number of items in the batch. | - -| Method Signature | Return value | Description | -|--------------------|--------------------|--------------------------------------------------------------------------------------| -| **openSession()** | `IDocumentSession` | New document session, that tracks all items and included items of the current batch. | - - - - -As long as there is no exception, the worker will continue addressing the same -server that the first batch was received from. -If the worker fails to reach that node, it will try to failover to another node -from the session's topology list. -The node that the worker succeeded connecting to, will inform the worker which -node is currently responsible for data subscriptions. - - - - - - - -if T is `ObjectNode`, no deserialization will take place - - -| Member | Type | Description | -|----------------------|-----------------------|----------------------------------------------------------------------------------------| -| **result** | `T` | Current batch item. | -| **exceptionMessage** | `String` | Message of the exception thrown during current document processing in the server side. | -| **id** | `String` | Current batch item's underlying document ID. | -| **changeVector** | `String` | Current batch item's underlying document change vector of the current document. | -| **rawResult** | `ObjectNode` | Current batch item before serialization to `T`. | -| **rawMetadata** | `ObjectNode` | Current batch item's underlying document metadata. | -| **metadata** | `IMetadataDictionary` | Current batch item's underlying metadata values. | - - - - - -## SubscriptionWorker<T> - - - -| Method Signature | Return Type | Description | -|------------------------------|---------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **close()** | `void` | Aborts subscription worker operation ungracefully by waiting for the task returned by the `run` function to finish running. | -| **run (multiple overloads)** | `CompletableFuture` | Call `run` to begin the worker's batch processing.
Pass the batch processing delegates to this method
(see [above](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker)). | - - - - - -| Event | Type\Return type | Description | -|------------------------------------|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **addAfterAcknowledgmentListener** | `Consumer>` (event) | Event that is risen after each the server acknowledges batch processing progress. | -| **onSubscriptionConnectionRetry** | `Consumer` (event) | Event that is fired when the subscription worker tries to reconnect to the server after a failure. The event receives as a parameter the exception that interrupted the processing. | -| **onClosed** | `Consumer>` (event) | Event that is fired after the subscription worker was disposed. | - - - - - -| Member | Type\Return type | Description | -|----------------------|--------------------|-----------------------------------------------------------------------| -| **currentNodeTag** | `String` | The node tag of the current RavenDB server handling the subscription. | -| **subscriptionName** | `String` | The name of the currently processed subscription. | - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_api-overview-nodejs.mdx b/docs/client-api/data-subscriptions/consumption/content/_api-overview-nodejs.mdx deleted file mode 100644 index 31a3deb17d..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_api-overview-nodejs.mdx +++ /dev/null @@ -1,212 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Create the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) - * [Subscription worker options](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-worker-options) - * [Run the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker) - * [Subscription batch](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-batch) - * [Subscription batch item](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-batch-item) - * [Subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-worker) - - -## Create the subscription worker - -A subscription worker can be created using the following `getSubscriptionWorker` methods available through the `subscriptions` property of the `documentStore`. - -Note: Simply creating the worker is insufficient; -after creating the worker, you need to [run the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker) to initiate document processing. - - - -{`await documentStore.subscriptions.getSubscriptionWorker(subscriptionName); -await documentStore.subscriptions.getSubscriptionWorker(subscriptionName, database); - -await documentStore.subscriptions.getSubscriptionWorker(options); -await documentStore.subscriptions.getSubscriptionWorker(options, database); -`} - - - -| Parameter | Type | Description | -|----------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **subscriptionName** | `string` | The name of the subscription to which the worker will connect. | -| **database** | `string` | The name of the database where the subscription task resides.
If `null`, the default database configured in DocumentStore will be used. | -| **options** | `object` | [Subscription worker options](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-worker-options) object that affect how the worker interacts with the subscription. These options do not alter the definition of the subscription itself. | - -| Return value | | -|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SubscriptionWorker` | The [subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-worker) that has been created.
The worker will start processing documents once you define the worker's `on` method,
which listens to the `batch` event. | - - - -## Subscription worker options - - - -{`// The SubscriptionWorkerOptions object: -// ===================================== -\{ - subscriptionName; - documentType; - ignoreSubscriberErrors; - closeWhenNoDocsLeft; - maxDocsPerBatch; - timeToWaitBeforeConnectionRetry; - maxErroneousPeriod; - strategy; -\} -`} - - - -When creating a worker with subscription worker options, the only mandatory property is `subscriptionName`. -All other parameters are optional and will default to their respective default values if not specified. - -| Member | Type | Description | -|-------------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **subscriptionName** | `string` | The name of the subscription to which the worker will connect. | -| **documentType** | `object` | The class type of the subscription documents. | -| **ignoreSubscriberErrors** | `boolean` | Determines if subscription processing is aborted when the worker's batch-handling code throws an unhandled exception.

`true` – subscription processing will continue.

`false` (default) – subscription processing will be aborted. | -| **closeWhenNoDocsLeft** | `boolean` | Determines whether the subscription connection closes when no new documents are available.

`true` – The subscription worker processes all available documents and stops when none remain, at which point the `SubscriptionClosedException` will be thrown.
Useful for ad-hoc, one-time processing.

`false` (default) – The subscription worker remains active, waiting for new documents. | -| **maxDocsPerBatch** | `number` | The maximum number of documents that the server will try to retrieve and send to the client in a batch. If the server doesn't find as many documents as specified, it will send the documents it has found without waiting. Default: 4096. | -| **timeToWaitBeforeConnectionRetry** | `number` | The time (in ms) to wait before attempting to reconnect after a non-aborting failure during subscription processing. Default: 5 seconds. | -| **maxErroneousPeriod** | `number` | The maximum amount of time (in ms) a subscription connection can remain in an erroneous state before it is terminated. Default: 5 minutes. | -| **strategy** | `string` | The strategy configures how the server handles connection attempts from workers to a specific subscription task.

Available options:
`OpenIfFree` (default), `TakeOver`, `WaitForFree`, or `Concurrent`.

Learn more in [worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies). | - - - -## Run the subscription worker - -After [creating](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) a subscription worker, the subscription worker is still not processing any documents. -To initiate processing, you need to define an event handler and attach it to the worker's `batch` event listener. - -This handler contains your client-side code responsible for processing the document batches received from the server. -Whenever a new batch of documents is ready, the provided handler will be triggered. - - - -{`subscriptionWorker.on("batch", (batch, callback) => \{ - // Process incoming items: - // ======================= - - // 'batch': - // Contains the documents to be processed. - - // callback(): - // Needs to be called after processing the batch - // to notify the worker that you're done processing. -\}); -`} - - - - - -## Subscription batch - -The subscription batch class contains the following public properties & methods: - -| Property | Type | Description | -|-------------------------------|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| **items** | `Item[]` | List of items in the batch.
See [subscription batch item](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-batch-item). | - -| Method | Return type | Description | -|-------------------------------|-------------|--------------------------------------------------------------------------------------------------------------------------| -| **getNumberOfItemsInBatch()** | `number` | Get the number of items in the batch. | -| **getNumberOfIncludes()** | `number` | Get the number of included documents in the batch. | -| **openSession()** | `object` | Open a new document session that tracks all items and their included items within the current batch. | -| **openSession(options)** | `object` | Open a new document session - can pass [session options](../../../../client-api/session/opening-a-session.mdx#session-options). | - - - -##### Subscription worker connectivity - -As long as there is no exception, the worker will continue addressing the same server that the first batch was received from. -If the worker fails to reach that node, it will try to [failover](../../../../client-api/configuration/load-balance/overview.mdx) to another node from the session's topology list. -The node that the worker succeeded connecting to, will inform the worker which node is currently responsible for data subscriptions. - - - - - -## Subscription batch item - -This class represents a single item in a subscription batch result. - - - -{`class Item -\{ - result; - exceptionMessage; - id; - changeVector; - projection; - revision; - rawResult; - rawMetadata; - metadata; -\} -`} - - - -| Member | Type | Description | -|----------------------|-----------|-------------------------------------------------------------------------------------| -| **result** | `object` | The current batch item. | -| **exceptionMessage** | `string` | The exception message thrown during current document processing in the server side. | -| **id** | `string` | The document ID of the underlying document for the current batch item. | -| **changeVector** | `string` | The change vector of the underlying document for the current batch item. | -| **rawResult** | `object` | Current batch item - no types reconstructed. | -| **rawMetadata** | `object` | Current batch item's underlying document metadata. | -| **metadata** | `object` | Current batch item's underlying metadata values. | - - - -## Subscription worker - - - -##### Methods - -| Method | Return type | Description | -|-------------------|---------------|---------------------------------------------------| -| **dispose()** | `void` | Aborts subscription worker operation. | -| **on()** | `object` | Method used to set up event listeners & handlers. | -| **getWorkerId()** | `string` | Get the worker ID. | - - - - - -##### Events - -| Event | Listener signature | Description | -|-----------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **"batch"** | `(batch, callback) => void` | Emitted when a batch of documents is sent from the server to the client.

Once processing is done, `callback` *must be called* in order to continue batches' emission. | -| **"afterAcknowledgment"** | `(batch, callback) => void` | Emitted after each time the server acknowledges the progress of batch processing. | -| **"connectionRetry"** | `(error) => void` | Emitted when the worker attempts to reconnect to the server after a failure. | -| **"error"** | `(error) => void` | Emitted on subscription errors. | -| **"end"** | `(error) => void` | Emitted when subscription is finished.
No more batches are going to be emitted. | - - - - - -##### Properties - -| Member | Type | Description | -|----------------------|----------|-----------------------------------------------------------------------| -| **currentNodeTag** | `string` | The node tag of the current RavenDB server handling the subscription. | -| **subscriptionName** | `string` | The name of the currently processed subscription. | - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_api-overview-python.mdx b/docs/client-api/data-subscriptions/consumption/content/_api-overview-python.mdx deleted file mode 100644 index 9bca678b40..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_api-overview-python.mdx +++ /dev/null @@ -1,207 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Create the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) - * [`SubscriptionWorkerOptions`](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) - * [Run the subscription worker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker) - * [`SubscriptionBatch[_T]`](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionbatch_t) - * [`SubscriptionWorker[_T]`](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworker_t) - - -## Create the subscription worker - -Create a subscription worker using `get_subscription_worker` or `get_subscription_worker_by_name`. - -* Use the `get_subscription_worker` method to specify the subscription options while creating the worker. -* Use the `get_subscription_worker_by_name` method to create the worker using the default options. - - - -{`def get_subscription_worker( - self, options: SubscriptionWorkerOptions, object_type: Optional[Type[_T]] = None, database: Optional[str] = None -) -> SubscriptionWorker[_T]: ... - -def get_subscription_worker_by_name( - self, - subscription_name: Optional[str] = None, - object_type: Optional[Type[_T]] = None, - database: Optional[str] = None, -) -> SubscriptionWorker[_T]: ... -`} - - - -| Parameter | | | -|----------------------------------|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| -| **options** | `SubscriptionWorkerOptions` | Options that affect how the worker interacts with the subscription. These options do not alter the definition of the subscription itself. | -| **object_type** (Optional) | `Type[_T]` | Defines the object type (class) for the items that will be included in the received `SubscriptionBatch` object. | -| **database** (Optional) | `str` | The name of the database where the subscription task resides. If `None`, the default database configured in DocumentStore will be used. | -| **subscription_name** (Optional) | `str` | The subscription's name. Used when the worker is generated without creating a `SubscriptionCreationOptions` instance, relying on the default values. | - -| Return value | | -|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `SubscriptionWorker` | The subscription worker that has been created.
Initially, it is idle and will only start processing documents when the `run` function is called. | - - - -## `SubscriptionWorkerOptions` - -When creating a worker with `SubscriptionWorkerOptions`, the only mandatory property is `subscription_name`. -All other parameters are optional and will default to their respective default values if not specified. - -| Member | Type | Description | -|------------------------------------------|-----------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **subscription_name** | `str` | The name of the subscription to which the worker will connect. | -| **time_to_wait_before_connection_retry** | `timedelta` | The time to wait before attempting to reconnect after a non-aborting failure during subscription processing. Default: 5 seconds. | -| **ignore_subscriber_errors** | `bool` | Determines if subscription processing is aborted when the worker's batch-handling code throws an unhandled exception.

`True` – subscription processing will continue.

`False` (Default) – subscription processing will be aborted. | -| **max_docs_per_batch** | `int` | The maximum number of documents that the server will try to retrieve and send to the client in a batch. If the server doesn't find as many documents as specified, it will send the documents it has found without waiting. Default: 4096. | -| **close_when_no_docs_left** | `bool` | Determines whether the subscription connection closes when no new documents are available.

`True` – The subscription worker processes all available documents and stops when none remain, at which point the `run` method throws a `SubscriptionClosedException`.
Useful for ad-hoc, one-time processing.

`False` (Default) – The subscription worker remains active, waiting for new documents. | -| **send_buffer_size_in_bytes** | `int` | The size in bytes of the TCP socket buffer used for _sending_ data.
Default: 32,768 bytes (32 KiB). | -| **receive_buffer_size_in_bytes** | `int` | The size in bytes of the TCP socket buffer used for _receiving_ data.
Default: 4096 (4 KiB). | -| **strategy** | `SubscriptionOpeningStrategy`
(enum) | Configures how the server handles connection attempts from workers to a specific subscription task.
Learn more in [worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies).
Default: `OPEN_IF_FREE`. | - - - -Learn more about `SubscriptionOpeningStrategy` in [worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies). - -| `SubscriptionOpeningStrategy` | | -|---------------------------------|---------------------------------------------------| -| `OPEN_IF_FREE` | Connect if no other worker is connected | -| `WAIT_FOR_FREE` | Wait for currently connected worker to disconnect | -| `TAKE_OVER` | Take over the connection | -| `CONCURRENT` | Connect concurrently | - - - - - -## Run the subscription worker - -After [creating](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker) a subscription worker, the subscription worker is still not processing any documents. -To start processing, you need to call the `run` function of the [SubscriptionWorker](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworker[_t]). - -The `run` function receives the client-side code as a function that will process the received document batches. - - - -{`def run(self, process_documents: Optional[Callable[[SubscriptionBatch[_T]], Any]]) -> Future: ... -`} - - - -| Parameter | | | -|----------------------------------|--------------------------------------------|-----------------------------------| -| **process_documents** (Optional) | `[Callable[[SubscriptionBatch[_T]], Any]]` | Delegate to sync batch processing | - - - - -## `SubscriptionBatch[_T]` - -| Member | Type | Description | -|------------------------------|------------------------------------|------------------------------| -| **items** | `SubscriptionBatch[_T].Item` array | List of items in the batch | -| **number_of_items_in_batch** | `int` | Number of items in the batch | - - - -{`def number_of_items_in_batch(self) -> int: - return 0 if self.items is None else len(self.items) -`} - - - - - -As long as there is no exception, the worker will continue addressing the same -server that the first batch was received from. -If the worker fails to reach that node, it will try to -[failover](../../../../client-api/configuration/load-balance/overview.mdx) to another node -from the session's topology list. -The node that the worker succeeds connecting to, will inform the worker which -node is currently responsible for data subscriptions. - - - - -{`class Item(Generic[_T_Item]): - """ - Represents a single item in a subscription batch results. - This class should be used only inside the subscription's run delegate, - using it outside this scope might cause unexpected behavior. - """ -`} - - - - -{`class SubscriptionBatch(Generic[_T]): - -def __init__(self): - self._result: Optional[_T_Item] = None - self._exception_message: Optional[str] = None - self._key: Optional[str] = None - self._change_vector: Optional[str] = None - self._projection: Optional[bool] = None - self._revision: Optional[bool] = None - self.raw_result: Optional[Dict] = None - self.raw_metadata: Optional[Dict] = None - self._metadata: Optional[MetadataAsDictionary] = None -`} - - - -| `SubscriptionBatch[_T].item` Member | Type | Description | -|---------------------------------------|------------------------|---------------------------------------------------------------------------------------| -| **\_result** (Optional) | `_T_Item` | Current batch item | -| **\_exception_message** (Optional) | `str` | Message of the exception thrown during current document processing in the server side | -| **\_key** (Optional) | `str` | Current batch item underlying document ID | -| **\_change_vector** (Optional) | `str` | Current batch item underlying document change vector of the current document | -| **\_projection** (Optional) | `bool` | indicates whether the value id a projection | -| **raw_result** (Optional) | `Dict` | Current batch item before serialization to `T` | -| **raw_metadata** (Optional) | `Dict` | Current batch item underlying document metadata | -| **\_metadata** (Optional) | `MetadataAsDictionary` | Current batch item underlying metadata values | - - -Usage of `raw_result`, `raw_metadata`, and `_metadata` values outside of the document processing delegate -is not supported. - - - - -## `SubscriptionWorker[_T]` -### Methods: - -| Method | Return Type | Description | -|-------------------------------------------------|----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `close(bool wait_for_subscription_task = True)` | `void` | Aborts subscription worker operation ungracefully by waiting for the task returned by the `run` function to finish running. | -| `run` | `Future[None]` | Call `run` to begin the worker's batch processing.
Pass the batch processing delegates to this method
(see [above](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#run-the-subscription-worker)). | -### Events: - -| Event | Type\Return type | Description | -|---------------------------|-------------------------------------------|----------------------------------------------------------------------------------| -| **after\_acknowledgment** | `Callable[[SubscriptionBatch[_T]], None]` | Event invoked after each time the server acknowledges batch processing progress. | - -| `after_acknowledgment` Parameters | | | -|------------------------------------|-------------------------|------------------------------------------| -| **batch** | `SubscriptionBatch[_T]` | The batch process which was acknowledged | - -| Return value | | -|----------------|--------------------------------------------------------------| -| `Future[None]` | The worker waits for the task to finish the event processing | - -### Properties: - -| Member | Type | Description | -|-----------------------|---------|-----------------------------------------------------------------------| -| **current_node_tag** | `str` | The node tag of the current RavenDB server handling the subscription. | -| **subscription_name** | `str` | The name of the currently processed subscription. | - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_examples-csharp.mdx b/docs/client-api/data-subscriptions/consumption/content/_examples-csharp.mdx deleted file mode 100644 index 65e0901c3f..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_examples-csharp.mdx +++ /dev/null @@ -1,450 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Client with full exception handling and processing retries](../../../../client-api/data-subscriptions/consumption/examples.mdx#client-with-full-exception-handling-and-processing-retries) - * [Worker with a specified batch size](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-with-a-specified-batch-size) - * [Worker that operates with a session](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-operates-with-a-session) - * [Worker that processes dynamic objects](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-processes-dynamic-objects) - * [Worker that processes a blittable object](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-processes-a-blittable-object) - * [Subscription that ends when no documents are left](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-ends-when-no-documents-are-left) - * [Subscription that uses included documents](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-uses-included-documents) - * [Subscription workers with failover on other nodes](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-workers-with-failover-on-other-nodes) - * [Primary and secondary workers](../../../../client-api/data-subscriptions/consumption/examples.mdx#primary-and-secondary-workers) - - -## Client with full exception handling and processing retries - -Here we implement a client that handles exceptions thrown by the worker. -If the exception is recoverable, the client retries creating the worker. - - - -{`while (true) -\{ - // Create the worker: - // ================== - var options = new SubscriptionWorkerOptions(subscriptionName); - - // Configure the worker: - // Allow a downtime of up to 2 hours, - // and wait 2 minutes before reconnecting - options.MaxErroneousPeriod = TimeSpan.FromHours(2); - options.TimeToWaitBeforeConnectionRetry = TimeSpan.FromMinutes(2); - - subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(options); - - try - \{ - // Subscribe to connection retry events - // and log any exceptions that occur during processing - subscriptionWorker.OnSubscriptionConnectionRetry += exception => - \{ - Logger.Error("Error during subscription processing: " + subscriptionName, - exception); - \}; - - // Run the worker: - // =============== - await subscriptionWorker.Run(batch => - \{ - foreach (var item in batch.Items) - \{ - // Forcefully stop subscription processing if the ID is "companies/2-A" - // and throw an exception to let external logic handle the specific case - if (item.Result.Company == "companies/2-A") - \{ - // The custom exception thrown from here - // will be wrapped by \`SubscriberErrorException\` - throw new UnsupportedCompanyException( - "Company ID can't be 'companies/2-A', pleases fix"); - \} - - // Process the order document - provide your own logic - ProcessOrder(item.Result); - \} - \}, cancellationToken); - - // The Run method will stop if the subscription worker is disposed, - // exiting the while loop - return; - \} - catch (Exception e) - \{ - Logger.Error("Failure in subscription: " + subscriptionName, e); - - // The following exceptions are Not recoverable - if (e is DatabaseDoesNotExistException || - e is SubscriptionDoesNotExistException || - e is SubscriptionInvalidStateException || - e is AuthorizationException) - throw; - - - if (e is SubscriptionClosedException) - // Subscription probably closed explicitly by admin - return; - - if (e is SubscriberErrorException se) - \{ - // For UnsupportedCompanyException we want to throw an exception, - // otherwise, continue processing - if (se.InnerException != null && se.InnerException is UnsupportedCompanyException) - \{ - throw; - \} - - // Call continue to skip the current while(true) iteration and try reconnecting - // in the next one, allowing the worker to process future batches. - continue; - \} - - // Handle this depending on the subscription opening strategy - if (e is SubscriptionInUseException) - continue; - - // Call return to exit the while(true) loop, - // dispose the worker (via finally), and stop the subscription. - return; - \} - finally - \{ - subscriptionWorker.Dispose(); - \} -\} -`} - - - - - -## Worker with a specified batch size - -Here we create a worker and specify the maximum number of documents the server will send to the worker in each batch. - - - -{`var workerWBatch = store.Subscriptions.GetSubscriptionWorker( - new SubscriptionWorkerOptions(subscriptionName) - \{ - MaxDocsPerBatch = 20 - \}); - -_ = workerWBatch.Run(x => -\{ - // your custom logic -\}); -`} - - - - - -## Worker that operates with a session - -Here we create a subscription that sends _Order_ documents that do not have a shipping date. -The worker receiving these documents will update the `ShippedAt` field value and save the document back to the server via the session. - - -Note: -The session is opened with `batch.OpenSession` instead of with `Store.OpenSession`. - - - - -{`// Create the subscription task on the server: -// =========================================== - -var subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -\{ - Query = @"from Orders as o where o.ShippedAt = null" -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -var subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); -_ = subscriptionWorker.Run(batch => -\{ - // Open a session with 'batch.OpenSession' - using (var session = batch.OpenSession()) - \{ - foreach (var order in batch.Items.Select(x => x.Result)) - \{ - TransferOrderToShipmentCompany(order); // call your custom method - order.ShippedAt = DateTime.UtcNow; // update the document field - \} - - // Save the updated Order documents - session.SaveChanges(); - \} -\}); -`} - - - - - -## Worker that processes dynamic objects - -Here we define a subscription that projects the _Order_ documents into a dynamic format. -The worker processes the dynamic objects it receives. - - - -{`// Create the subscription task on the server: -// =========================================== - -var subscriptionName = "My dynamic subscription"; -store.Subscriptions.Create(new SubscriptionCreationOptions() -\{ - Name = subscriptionName, - Projection = order => - new \{ DynanamicField_1 = "Company: " + order.Company + " Employee: " + order.Employee \} -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -var subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); -_ = subscriptionWorker.Run(batch => -\{ - foreach (var item in batch.Items) - \{ - // Access the dynamic field in the document - dynamic field = item.Result.DynanamicField_1; - - // Call your custom method - ProcessItem(field); - \} -\}); -`} - - - - - -## Worker that processes a blittable object - -Create a worker that processes documents as low level blittable objects. -This can be useful in extreme high-performance scenarios, but may be dangerous due to the direct usage of unmanaged memory. - - - -{`// Create the subscription task on the server: -// =========================================== - -var subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions -\{ - Projection = x => new - \{ - x.Employee - \} -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -var subscriptionWorker = - // Specify \`BlittableJsonReaderObject\` as the generic type parameter - store.Subscriptions.GetSubscriptionWorker(subscriptionName); - -_ = subscriptionWorker.Run(batch => -\{ - foreach (var item in batch.Items) - \{ - // Access the Employee field within the blittable object - var employeeField = item.Result["Employee"].ToString(); - - ProcessItem(employeeField); // call your custom method - \} -\}); -`} - - - - - -## Subscription that ends when no documents are left - -Here we create a subscription client that runs until there are no more new documents to process. -This is useful for ad-hoc, single-use processing where the user needs to ensure that all documents are fully processed. - - - -{`// Create the subscription task on the server: -// =========================================== -var subscriptionName = store.Subscriptions.Create( - new SubscriptionCreationOptions - \{ - Filter = order => order.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 10000, - Projection = order => new OrderAndCompany - \{ - OrderId = order.Id, - Company = RavenQuery.Load(order.Company) - \} - \}); - -// Create the subscription worker that will consume the documents: -// =============================================================== -var highValueOrdersWorker = store.Subscriptions.GetSubscriptionWorker( - new SubscriptionWorkerOptions(subscriptionName) - \{ - // Here we set the worker to stop when there are no more documents left to send - // Will throw SubscriptionClosedException when it finishes it's job - CloseWhenNoDocsLeft = true - \}); - -try -\{ - await highValueOrdersWorker.Run(batch => - \{ - foreach (var item in batch.Items) - \{ - SendThankYouNoteToEmployee(item.Result); // call your custom method - \} - \}); -\} -catch (SubscriptionClosedException) -\{ - // That's expected, no more documents to process -\} -`} - - - - - -## Subscription that uses included documents - -Here we create a subscription that, in addition to sending all the _Order_ documents to the worker, -will include all the referenced _Product_ documents in the batch sent to the worker. - -When the worker accesses these _Product_ documents, no additional requests will be made to the server. - - - -{`// Create the subscription task on the server: -// =========================================== - -var subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -\{ - // Include the referenced Product documents for each Order document - Query = @"from Orders include Lines[].Product" -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -var subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); -_ = subscriptionWorker.Run(batch => -\{ - // Open a session via 'batch.OpenSession' - // in order to access the Product documents - using (var session = batch.OpenSession()) - \{ - foreach (var order in batch.Items.Select(x => x.Result)) - \{ - foreach (var orderLine in order.Lines) - \{ - // Calling Load will Not generate a request to the server, - // because orderLine.Product was included in the batch - var product = session.Load(orderLine.Product); - - ProcessOrderAndProduct(order, product); // call your custom method - \} - \} - \} -\}); -`} - - - - - -## Subscription workers with failover on other nodes - -In this configuration, any available node will create a worker. -If the worker fails, another available node will take over. - - - -{`var worker = store.Subscriptions.GetSubscriptionWorker( - new SubscriptionWorkerOptions(subscriptionName) -\{ - Strategy = SubscriptionOpeningStrategy.WaitForFree -\}); -`} - - - - - -## Primary and secondary workers - -Here we create two workers: - -* The primary worker, with a `TakeOver` strategy, will take over the other worker and establish the connection. -* The secondary worker, with a `WaitForFree` strategy, will wait for the first worker to fail (due to machine failure, etc.). - -The primary worker: - - -{`var primaryWorker = store.Subscriptions.GetSubscriptionWorker( - new SubscriptionWorkerOptions(subscriptionName) -\{ - Strategy = SubscriptionOpeningStrategy.TakeOver -\}); - -while (true) -\{ - try - \{ - await primaryWorker.Run(x => - \{ - // your logic - \}); - \} - catch (Exception) - \{ - // retry - \} -\} -`} - - - -The secondary worker: - - -{`var secondaryWorker = store.Subscriptions.GetSubscriptionWorker( - new SubscriptionWorkerOptions(subscriptionName) -\{ - Strategy = SubscriptionOpeningStrategy.WaitForFree -\}); - -while (true) -\{ - try - \{ - await secondaryWorker.Run(x => - \{ - // your logic - \}); - \} - catch (Exception) - \{ - // retry - \} -\} -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_examples-java.mdx b/docs/client-api/data-subscriptions/consumption/content/_examples-java.mdx deleted file mode 100644 index 8466cf34ac..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_examples-java.mdx +++ /dev/null @@ -1,294 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* this page: - * [Worker with a specified batch size](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-with-a-specified-batch-size) - * [Client with full exception handling and processing retries](../../../../client-api/data-subscriptions/consumption/examples.mdx#client-with-full-exception-handling-and-processing-retries) - * [Subscription that ends when no documents left](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-ends-when-no-documents-left) - * [Worker that processes raw objects](../../../../client-api/data-subscriptions/consumption/examples.mdx#sworker-that-processes-raw-objects) - * [Worker that operates with a session](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-operates-with-a-session) - * [Subscription that uses included documents](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-uses-included-documents) - * [Primary and secondary workers](../../../../client-api/data-subscriptions/consumption/examples.mdx#primary-and-secondary-workers) - - -## Worker with a specified batch size - -Here we create a worker and specify the maximum number of documents the server will send to the worker in each batch. - - - -{`SubscriptionWorkerOptions options = new SubscriptionWorkerOptions(subscriptionName); -options.setMaxDocsPerBatch(20); -SubscriptionWorker workerWBatch = store.subscriptions().getSubscriptionWorker(Order.class, options); -workerWBatch.run(x -> \{ /* custom logic */\}); -`} - - - - - -## Client with full exception handling and processing retries - -Here we implement a client that handles exceptions thrown by a worker. -If the exception is recoverable, the client retries creating the worker. - - - -{`while (true) \{ - SubscriptionWorkerOptions options = new SubscriptionWorkerOptions(subscriptionName); - // here we configure that we allow a down time of up to 2 hours, - // and will wait for 2 minutes for reconnecting - - options.setMaxErroneousPeriod(Duration.ofHours(2)); - options.setTimeToWaitBeforeConnectionRetry(Duration.ofMinutes(2)); - - subscriptionWorker = store.subscriptions().getSubscriptionWorker(Order.class, options); - - try \{ - // here we are able to be informed of any exception that happens during processing - subscriptionWorker.addOnSubscriptionConnectionRetry(exception -> \{ - logger.error("Error during subscription processing: " + subscriptionName, exception); - \}); - - subscriptionWorker.run(batch -> \{ - for (SubscriptionBatch.Item item : batch.getItems()) \{ - // we want to force close the subscription processing in that case - // and let the external code decide what to do with that - if ("Europe".equalsIgnoreCase(item.getResult().getShipVia())) \{ - throw new IllegalStateException("We cannot ship via Europe"); - \} - processOrder(item.getResult()); - \} - \}).get(); - - - // Run will complete normally if you have disposed the subscription - return; - \} catch (Exception e) \{ - logger.error("Failure in subscription: " + subscriptionName, e); - - e = ExceptionsUtils.unwrapException(e); - if (e instanceof DatabaseDoesNotExistException || - e instanceof SubscriptionDoesNotExistException || - e instanceof SubscriptionInvalidStateException || - e instanceof AuthorizationException) \{ - throw e; // not recoverable - \} - - if (e instanceof SubscriptionClosedException) \{ - // closed explicitly by admin, probably - return; - \} - - if (e instanceof SubscriberErrorException) \{ - SubscriberErrorException se = (SubscriberErrorException) e; - // for IllegalStateException type, we want to throw an exception, otherwise - // we continue processing - if (se.getCause() != null && se.getCause() instanceof IllegalStateException) \{ - throw e; - \} - - continue; - \} - - // handle this depending on subscription - // open strategy (discussed later) - if (e instanceof SubscriptionInUseException) \{ - continue; - \} - - return; - \} finally \{ - subscriptionWorker.close(); - \} -\} -`} - - - - - -## Subscription that ends when no documents left - -Here we create a subscription client that runs only up to the point there are no more new documents left to process. - -This is useful for ad-hoc, single-use processing where the user needs to ensure that all documents are fully processed. - - - -{`SubscriptionWorkerOptions options = new SubscriptionWorkerOptions(subsId); - -// Here we ask the worker to stop when there are no documents left to send. -// Will throw SubscriptionClosedException when it finishes it's job -options.setCloseWhenNoDocsLeft(true); -SubscriptionWorker highValueOrdersWorker = store - .subscriptions().getSubscriptionWorker(OrderAndCompany.class, options); - -try \{ - highValueOrdersWorker.run(batch -> \{ - for (SubscriptionBatch.Item item : batch.getItems()) \{ - sendThankYouNoteToEmployee(item.getResult()); - \} - \}); -\} catch (SubscriptionClosedException e) \{ - //that's expected -\} -`} - - - - - -## Worker that processes raw objects - -Here we create a worker that processes received data as ObjectNode objects. - - - -{`String subscriptionName = "My dynamic subscription"; - -SubscriptionCreationOptions subscriptionCreationOptions = new SubscriptionCreationOptions(); -subscriptionCreationOptions.setName("My dynamic subscription"); -subscriptionCreationOptions.setQuery("from Orders as o \\n" + - "select \{ \\n" + - " DynamicField_1: 'Company:' + o.Company + ' Employee: ' + o.Employee \\n" + - "\}"); - -SubscriptionWorker worker = store.subscriptions().getSubscriptionWorker(subscriptionName); -worker.run(x -> \{ - for (SubscriptionBatch.Item item : x.getItems()) \{ - ObjectNode result = item.getResult(); - raiseNotification(result.get("DynamicField_1")); - \} -\}); -`} - - - - - -## Worker that operates with a session - -Here we create a subscription that sends Order documents that do not have a shipping date. -The worker receiving these documents will update the `ShippedAt` field value and save the document back to the server via the session. - - - -{`SubscriptionCreationOptions subscriptionCreationOptions = new SubscriptionCreationOptions(); -subscriptionCreationOptions.setQuery("from Orders as o where o.ShippedAt = null"); -String subscriptionName = store.subscriptions().create(subscriptionCreationOptions); - -SubscriptionWorker subscriptionWorker = store.subscriptions().getSubscriptionWorker(Order.class, subscriptionName); - -subscriptionWorker.run(batch -> \{ - try (IDocumentSession session = batch.openSession()) \{ - for (SubscriptionBatch.Item orderItem : batch.getItems()) \{ - transferOrderToShipmentCompany(orderItem.getResult()); - orderItem.getResult().setShippedAt(new Date()); - \} - - // we know that we have at least one order to ship, - // because the subscription query above has that in it's WHERE clause - session.saveChanges(); - \} -\}); -`} - - - - - -## Subscription that uses included documents - -Here we create a subscription that, in addition to sending all the _Order_ documents to the worker, -will include all the referenced _Product_ documents in the batch sent to the worker. - -When the worker accesses these _Product_ documents, no additional requests will be made to the server. - - - -{`SubscriptionCreationOptions subscriptionCreationOptions = new SubscriptionCreationOptions(); -subscriptionCreationOptions.setQuery("from Orders include Lines[].Product"); - - -String subscriptionName = store.subscriptions().create(subscriptionCreationOptions); - -SubscriptionWorker subscriptionWorker = store.subscriptions().getSubscriptionWorker(Order.class, subscriptionName); - -subscriptionWorker.run(batch -> \{ - try (IDocumentSession session = batch.openSession()) \{ - for (SubscriptionBatch.Item orderItem : batch.getItems()) \{ - Order order = orderItem.getResult(); - for (OrderLine orderLine : order.getLines()) \{ - // this line won't generate a request, because orderLine.Product was included - Product product = session.load(Product.class, orderLine.getProduct()); - raiseProductNotification(order, product); - \} - \} - \} -\}); -`} - - - - - -## Primary and secondary workers - -Here we create two workers: - -* The primary worker, with a `TAKE_OVER` strategy, will take over the other worker and establish the connection. -* The secondary worker, with a `WAIT_FOR_FREE` strategy, will wait for the first worker to fail (due to machine failure, etc.). - -The primary worker: - - - -{`SubscriptionWorkerOptions options1 = new SubscriptionWorkerOptions(subscriptionName); -options1.setStrategy(SubscriptionOpeningStrategy.TAKE_OVER); -SubscriptionWorker worker1 = store.subscriptions().getSubscriptionWorker(Order.class, options1); - - -while (true) \{ - try \{ - worker1 - .run(x -> \{ - // your logic - \}); - \} catch (Exception e) \{ - // retry - \} -\} -`} - - - -The secondary worker: - - - -{`SubscriptionWorkerOptions options2 = new SubscriptionWorkerOptions(subscriptionName); -options2.setStrategy(SubscriptionOpeningStrategy.WAIT_FOR_FREE); -SubscriptionWorker worker2 = store.subscriptions().getSubscriptionWorker(Order.class, options2); - -while (true) \{ - try \{ - worker2 - .run(x -> \{ - // your logic - \}); - \} catch (Exception e) \{ - // retry - \} -\} -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_examples-nodejs.mdx b/docs/client-api/data-subscriptions/consumption/content/_examples-nodejs.mdx deleted file mode 100644 index e2c5222311..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_examples-nodejs.mdx +++ /dev/null @@ -1,456 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Client with full exception handling and processing retries](../../../../client-api/data-subscriptions/consumption/examples.mdx#client-with-full-exception-handling-and-processing-retries) - * [Worker with a specified batch size](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-with-a-specified-batch-size) - * [Worker that operates with a session](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-operates-with-a-session) - * [Worker that processes dynamic objects](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-processes-dynamic-objects) - * [Subscription that ends when no documents are left](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-ends-when-no-documents-are-left) - * [Subscription that uses included documents](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-uses-included-documents) - * [Primary and secondary workers](../../../../client-api/data-subscriptions/consumption/examples.mdx#primary-and-secondary-workers) - - -## Client with full exception handling and processing retries - -Here we implement a client that handles exceptions thrown by the worker. -If the exception is recoverable, the client retries creating the worker. - - - -{`// Create the subscription task on the server: -// =========================================== - -const subscriptionName = await documentStore.subscriptions.create(\{ - name: "ProcessOrdersWithLowFreight", - query: "from Orders where Freight < 0.5" -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -await setupReconnectingWorker(subscriptionName); - -async function setupReconnectingWorker(subscriptionName) \{ - let subscriptionWorker; - - await reconnect(); - - function closeWorker(worker) \{ - worker.dispose(); - \} - - async function reconnect() \{ - if (subscriptionWorker) \{ - closeWorker(subscriptionWorker); - \} - - // Configure the worker: - const subscriptionWorkerOptions = \{ - subscriptionName: subscriptionName, - // Allow a downtime of up to 2 hours - maxErroneousPeriod: 2 * 3600 * 1000, - // Wait 2 minutes before reconnecting - timeToWaitBeforeConnectionRetry: 2 * 60 * 1000 - \}; - - subscriptionWorker = - store.subscriptions.getSubscriptionWorker(subscriptionWorkerOptions); - - // Subscribe to connection retry events, - // and log any exceptions that occur during processing - subscriptionWorker.on("connectionRetry", error => \{ - console.error( - "Error during subscription processing: " + subscriptionName, error); - \}); - - // Run the worker: - // =============== - subscriptionWorker.on("batch", (batch, callback) => \{ - try \{ - for (const item of batch.items) \{ - const orderDocument = item.result; - - // Forcefully stop subscription processing if the ID is "companies/46-A" - // and throw an exception to let external logic handle the specific case - if (orderDocument.Company && orderDocument.Company === "companies/46-A") \{ - // 'The InvalidOperationException' thrown from here - // will be wrapped by \`SubscriberErrorException\` - callback(new InvalidOperationException( - "Company ID can't be 'companies/46-A', pleases fix")); - return; - \} - - // Process the order document - provide your own logic - processOrder(orderDocument); - \} - // Call 'callback' once you're done - // The worker will send an acknowledgement to the server, - // so that server can send next batch - callback(); - \} - catch(err) \{ - callback(err); - \} - \}); - - // Handle errors: - // ============== - subscriptionWorker.on("error", error => \{ - console.error("Failure in subscription: " + subscriptionName, error); - - // The following exceptions are Not recoverable - if (error.name === "DatabaseDoesNotExistException" || - error.name === "SubscriptionDoesNotExistException" || - error.name === "SubscriptionInvalidStateException" || - error.name === "AuthorizationException") \{ - throw error; - \} - - if (error.name === "SubscriptionClosedException") \{ - // Subscription probably closed explicitly by admin - return closeWorker(subscriptionWorker); - \} - - if (error.name === "SubscriberErrorException") \{ - // For the InvalidOperationException we want to throw an exception, - // otherwise, continue processing - if (error.cause && error.cause.name === "InvalidOperationException") \{ - throw error; - \} - - setTimeout(reconnect, 1000); - return; - \} - - // Handle this depending on the subscription opening strategy - if (error.name === "SubscriptionInUseException") \{ - setTimeout(reconnect, 1000); - return; - \} - - setTimeout(reconnect, 1000); - return; - \}); - - // Handle worker end event: - // ======================== - subscriptionWorker.on("end", () => \{ - closeWorker(subscriptionWorker); - \}); - \} -\} -`} - - - - - -## Worker with a specified batch size - -Here we create a worker and specify the maximum number of documents the server will send to the worker in each batch. - - - -{`// Create the subscription task on the server: -// =========================================== - -const subscriptionName = await documentStore.subscriptions.create(\{ - name: "ProcessOrders", - query: "from Orders" -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -const workerOptions = \{ - subscriptionName: subscriptionName, - maxDocsPerBatch: 20 // Set the maximum number of documents per batch -\}; - -const worker = documentStore.subscriptions.getSubscriptionWorker(workerOptions); - -worker.on("batch", (batch, callback) => \{ - try \{ - // Add your logic for processing the incoming batch items here... - - // Call 'callback' once you're done - // The worker will send an acknowledgement to the server, - // so that server can send next batch - callback(); - - \} catch(err) \{ - callback(err); - \} -\}); -`} - - - - - -## Worker that operates with a session - -Here we create a subscription that sends _Order_ documents that do not have a shipping date. -The worker receiving these documents will update the `ShippedAt` field value and save the document back to the server via the session. - - -Note: -The session is opened with `batch.openSession` instead of with `documentStore.openSession`. - - - - -{`// Create the subscription task on the server: -// =========================================== - -const subscriptionName = await documentStore.subscriptions.create(\{ - name: "ProcessOrdersThatWereNotShipped", - query: "from Orders as o where o.ShippedAt = null" -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -const workerOptions = \{ subscriptionName \}; -const worker = documentStore.subscriptions.getSubscriptionWorker(workerOptions); - -worker.on("batch", async (batch, callback) - try \{ - // Open a session with 'batch.openSession' - const session = batch.openSession(); - - for (const item of batch.items) \{ - orderDocument = item.result; - - transferOrderToShipmentCompany(orderDocument); // call your custom method - orderDocument.ShippedAt = new Date(); // update the document field - \} - - // Save the updated Order documents - await session.saveChanges(); - callback(); - - \} catch(err) \{ - callback(err); - \} -\}); -`} - - - - - -## Worker that processes dynamic objects - -Here we define a subscription that projects the _Order_ documents into a dynamic format. -The worker processes the dynamic objects it receives. - - - -{`// Create the subscription task on the server: -// =========================================== - -const subscriptionName = await documentStore.subscriptions.create(\{ - name: "ProcessDynamicFields", - query: \`From Orders as o - Select \{ - dynamicField: "Company: " + o.Company + " Employee: " + o.Employee, - \}\` -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -const workerOptions = \{ subscriptionName \}; -const worker = documentStore.subscriptions.getSubscriptionWorker(workerOptions); - -worker.on("batch", (batch, callback) => \{ - for (const item of batch.items) \{ - - // Access the dynamic field in the document - const field = item.result.dynamicField; - - // Call your custom method - processItem(field); - \} - - callback(); -\}); -`} - - - - - -## Subscription that ends when no documents are left - -Here we create a subscription client that runs until there are no more new documents to process. -This is useful for ad-hoc, single-use processing where the user needs to ensure that all documents are fully processed. - - - -{`// Create the subscription task on the server: -// =========================================== - -// Define the filtering criteria -const query = \` - declare function getOrderLinesSum(doc) \{ - var sum = 0; - for (var i in doc.Lines) \{ - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - \} - return sum; - \} - - from Orders as o - where getOrderLinesSum(o) > 10_000\`; - -// Create the subscription with the defined query -const subscriptionName = await documentStore.subscriptions.create(\{ query \}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -const workerOptions = \{ - subscriptionName: subscriptionName, - // Here we set the worker to stop when there are no more documents left to send - // Will throw SubscriptionClosedException when it finishes it's job - closeWhenNoDocsLeft: true -\}; - -const highValueOrdersWorker = - documentStore.subscriptions.getSubscriptionWorker(workerOptions); - -highValueOrdersWorker.on("batch", (batch, callback) => \{ - for (const item of batch.items) \{ - sendThankYouNoteToEmployee(item.result); // call your custom method - \} - - callback(); -\}); - -highValueOrdersWorker.on("error", err => \{ - if (err.name === "SubscriptionClosedException") \{ - // That's expected, no more documents to process - \} -\}); -`} - - - - - -## Subscription that uses included documents - -Here we create a subscription that, in addition to sending all the _Order_ documents to the worker, -will include all the referenced _Product_ documents in the batch sent to the worker. - -When the worker accesses these _Product_ documents, no additional requests will be made to the server. - - - -{`// Create the subscription task on the server: -// =========================================== - -const subscriptionName = await documentStore.subscriptions.create(\{ - name: "ProcessIncludedDocuments", - query: \`from Orders include Lines[].Product\` -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -const workerOptions = \{ subscriptionName \}; -const worker = documentStore.subscriptions.getSubscriptionWorker(workerOptions); - -worker.on("batch", async (batch, callback) => \{ - // Open a session via 'batch.openSession' - // in order to access the Product documents - const session = batch.openSession(); - - for (const item of batch.items) \{ - const orderDocument = item.result; - - for (const orderLine of orderDocument.Lines) - \{ - // Calling 'load' will Not generate a request to the server, - // because orderLine.Product was included in the batch - const product = await session.load(orderLine.Product); - const productName = product.Name; - - // Call your custom method - processOrderAndProduct(order, product); - \} - \} - - callback(); -\}); -`} - - - - - -## Primary and secondary workers - -Here we create two workers: - -* The primary worker, with a `TakeOver` strategy, will take over the other worker and establish the connection. -* The secondary worker, with a `WaitForFree` strategy, will wait for the first worker to fail (due to machine failure, etc.). - -The primary worker: - - - -{`const workerOptions1 = \{ - subscriptionName, - strategy: "TakeOver", - documentType: Order -\}; - -const worker1 = documentStore.subscriptions.getSubscriptionWorker(workerOptions1); - -worker1.on("batch", (batch, callback) => \{ - // your logic - callback(); -\}); - -worker1.on("error", err => \{ - // retry -\}); -`} - - - -The secondary worker: - - - -{`const workerOptions2 = \{ - subscriptionName, - strategy: "WaitForFree", - documentType: Order -\}; - -const worker2 = documentStore.subscriptions.getSubscriptionWorker(workerOptions2); - -worker2.on("batch", (batch, callback) => \{ - // your logic - callback(); -\}); - -worker2.on("error", err => \{ - // retry -\}); -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_examples-python.mdx b/docs/client-api/data-subscriptions/consumption/content/_examples-python.mdx deleted file mode 100644 index f2a6937706..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_examples-python.mdx +++ /dev/null @@ -1,314 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Client with full exception handling and processing retries](../../../../client-api/data-subscriptions/consumption/examples.mdx#client-with-full-exception-handling-and-processing-retries) - * [Worker with a specified batch size](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-with-a-specified-batch-size) - * [Worker that operates with a session](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-operates-with-a-session) - * [Worker that processes dynamic objects](../../../../client-api/data-subscriptions/consumption/examples.mdx#worker-that-processes-dynamic-objects) - * [Subscription that ends when no documents are left](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-ends-when-no-documents-are-left) - * [Subscription that uses included documents](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-uses-included-documents) - * [Subscription workers with failover on other nodes](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-workers-with-failover-on-other-nodes) - * [Primary and secondary workers](../../../../client-api/data-subscriptions/consumption/examples.mdx#primary-and-secondary-workers) - - -## Client with full exception handling and processing retries - -Here we implement a client that handles exceptions thrown by a worker. -If the exception is recoverable, the client retries creating the worker. - - - -{`while True: - options = SubscriptionWorkerOptions(subscription_name) - - # here we configure that we allow a down time of up to 2 hours, and will wait for 2 minutes for reconnecting - options.max_erroneous_period = timedelta(hours=2) - options.time_to_wait_before_connection_retry = timedelta(minutes=2) - - subscription_worker = store.subscriptions.get_subscription_worker(options, Order) - - try: - # here we are able to be informed of any exceptions that happens during processing - subscription_worker.add_on_subscription_connection_retry( - lambda exception: logger.error( - f"Error during subscription processing: \{subscription_name\}", exc_info=exception - ) - ) - - def _process_documents_callback(batch: SubscriptionBatch[Order]): - for item in batch.items: - # we want to force close the subscription processing in that case - # and let the external code decide what to do with that - if item.result.company == "companies/2-A": - raise UnsupportedCompanyException( - "Company Id can't be 'companies/2-A', you must fix this" - ) - process_order(item.result) - - # Run will complete normally if you have disposed the subscription - return - - # Pass the callback to worker.run() - subscription_worker.run(_process_documents_callback) - - except Exception as e: - logger.error(f"Failure in subscription: \{subscription_name\}", exc_info=e) - exception_type = type(e) - if ( - exception_type is DatabaseDoesNotExistException - or exception_type is SubscriptionDoesNotExistException - or exception_type is SubscriptionInvalidStateException - or exception_type is AuthorizationException - ): - raise # not recoverable - - if exception_type is SubscriptionClosedException: - # closed explicitely by admin, probably - return - - if exception_type is SubscriberErrorException: - # for UnsupportedCompanyException type, we want to throw an exception, otherwise - # we continue processing - if e.args[1] is not None and type(e.args[1]) is UnsupportedCompanyException: - raise - - continue - - # handle this depending on subscription - # open strategy (discussed later) - if e is SubscriptionInUseException: - continue - - return - finally: - subscription_worker.close(False) -`} - - - - - -## Worker with a specified batch size - -Here we create a worker and specify the maximum number of documents the server will send to the worker in each batch. - - - -{`worker_w_batch = store.subscriptions.get_subscription_worker( - SubscriptionWorkerOptions(subscription_name, max_docs_per_batch=20), Order -) - -_ = worker_w_batch.run( - process_documents=lambda batch: ... -) # Pass your method that takes SubscriptionBatch[_T] as an argument, with your logic in it -`} - - - - - -## Worker that operates with a session - -Here we create a subscription that sends _Order_ documents that do not have a shipping date. -The worker receiving these documents will update the `ShippedAt` field value and save the document back to the server via the session. - - - -{`subscription_name = store.subscriptions.create_for_options( - SubscriptionCreationOptions(query="from Orders as o where o.ShippedAt = null") -) - -subscription_worker = store.subscriptions.get_subscription_worker_by_name(subscription_name, Order) - -def _transfer_order_callback(batch: SubscriptionBatch[Order]): - with batch.open_session() as session: - for order in (item.result for item in batch.items): - transfer_order_to_shipment_company(order) - order.shipped_at = datetime.utcnow() - - # we know that we have at least one order to ship, - # because the subscription query above has that in it's WHERE clause - session.save_changes() - -_ = subscription_worker.run(_transfer_order_callback) -`} - - - - - -## Worker that processes dynamic objects - -Here we define a subscription that projects the _Order_ documents into a dynamic format. -The worker processes the dynamic objects it receives. - - - -{`subscription_name = "My dynamic subscription" -store.subscriptions.create_for_class( - Order, - SubscriptionCreationOptions( - subscription_name, - query=""" - From Orders as o - Select - \{ - dynamic_field_1: "Company: " + o.Company + " Employee: " + o.Employee, - \} - """, - ), -) - -subscription_worker = store.subscriptions.get_subscription_worker_by_name(subscription_name) - -def _raise_notification_callback(batch: SubscriptionBatch[Order]): - for item in batch.items: - raise_notification(item.result.dynamic_field_1) - -_ = subscription_worker.run(_raise_notification_callback) -`} - - - - - -## Subscription that ends when no documents are left - -Here we create a subscription client that runs only up to the point there are no more new documents left to process. - -This is useful for ad-hoc, single-use processing where the user needs to ensure that all documents are fully processed. - - - -{`high_value_orders_worker = store.subscriptions.get_subscription_worker( - SubscriptionWorkerOptions( - subs_id, - # Here we ask the worker to stop when there are no documents left to send. - # Will throw SubscriptionClosedException when it finishes its job - close_when_no_docs_left=True, - ), - OrderAndCompany, -) - -try: - - def _subscription_batch_callback(batch: SubscriptionBatch[OrderAndCompany]): - for item in batch.items: - send_thank_you_note_to_employee(item.result) - - high_value_orders_worker.run(_subscription_batch_callback) -except SubscriptionClosedException: - # that's expected - ... -`} - - - - - -## Subscription that uses included documents - -Here we create a subscription that, in addition to sending all the _Order_ documents to the worker, -will include all the referenced _Product_ documents in the batch sent to the worker. - -When the worker accesses these _Product_ documents, no additional requests will be made to the server. - - - -{`// Create the subscription task on the server: -// =========================================== - -var subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -\{ - // Include the referenced Product documents for each Order document - Query = @"from Orders include Lines[].Product" -\}); - -// Create the subscription worker that will consume the documents: -// =============================================================== - -var subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); -_ = subscriptionWorker.Run(batch => -\{ - // Open a session via 'batch.OpenSession' - // in order to access the Product documents - using (var session = batch.OpenSession()) - \{ - foreach (var order in batch.Items.Select(x => x.Result)) - \{ - foreach (var orderLine in order.Lines) - \{ - // Calling Load will Not generate a request to the server, - // because orderLine.Product was included in the batch - var product = session.Load(orderLine.Product); - - ProcessOrderAndProduct(order, product); // call your custom method - \} - \} - \} -\}); -`} - - - - - -## Subscription workers with failover on other nodes - -In this configuration, any available node will create a worker. -If the worker fails, another available node will take over. - - - -{`worker = store.subscriptions.get_subscription_worker( - SubscriptionWorkerOptions(subscription_name, strategy=SubscriptionOpeningStrategy.WAIT_FOR_FREE), Order -) -`} - - - - - -## Primary and secondary workers - -Here we create two workers: - -* The primary worker, with a `TAKE_OVER` strategy, will take over the other worker and establish the connection. -* The secondary worker, with a `WAIT_FOR_FREE` strategy, will wait for the first worker to fail (due to machine failure, etc.). - -The primary worker: - - -{`primary_worker = store.subscriptions.get_subscription_worker(SubscriptionWorkerOptions(subscription_name, strategy=SubscriptionOpeningStrategy.TAKE_OVER), Order) - -while True: - try: - run_future = primary_worker.run(lambda batch: ...) # your logic - except Exception: - ... # retry -`} - - - -The secondary worker: - - -{`secondary_worker = store.subscriptions.get_subscription_worker(SubscriptionWorkerOptions(subscription_name), strategy=SubscriptionOpeningStrategy.WAIT_FOR_FREE) - -while True: - try: - run_future = secondary_worker.run(lambda batch: ...) # your logic - except Exception: - ... # retry -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-csharp.mdx b/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-csharp.mdx deleted file mode 100644 index 7379c89553..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-csharp.mdx +++ /dev/null @@ -1,198 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* Batches of documents sent from a Subscription Task defined on the server are consumed and processed by a subscription worker client. - -* The `SubscriptionWorker` object, defined on the client, manages the communication between the server and the client and processes the document batches sent from the server. - -* There are several ways to create and configure the SubscriptionWorker - see [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions). - -* In this page: - * [SubscriptionWorker lifecycle](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#subscriptionworker-lifecycle) - * [Error handling](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#error-handling) - * [Worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies) - * [Determining which workers a subscription will serve](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#determining-which-workers-a-subscription-will-serve) - - -## SubscriptionWorker lifecycle - -A `SubscriptionWorker` object starts its life from being generated by the `DocumentsStore.Subscriptions`: - - -{`subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); -`} - - - -At this point, the worker has only got its configuration. No connection or processing happens at this moment. -To start processing, the `Run` method should be called. The Run method receives the batch processing logic that should be performed: - - -{`subscriptionRuntimeTask = subscriptionWorker.Run(batch => -\{ - // your logic here -\}); -`} - - - -From this point on, the subscription worker will start processing batches. -If processing is aborted for any reason, the returned task (`subscriptionRuntimeTask`) will complete with an exception. - - - -## Error handling - - - -Subscription worker connection failures may occur during the routine communication between the worker and the server. -When an unexpected error arises, the worker will attempt to **reconnect to the server**. - -However, there are several conditions under which the worker will stop its operation but will Not attempt to reconnect: - -* The subscription no longer exists or has been deleted. -* Another worker has taken control of the subscription (see [connection strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#available-worker-strategies)). -* The worker is unable to connect to any of the servers. -* The worker could not receive the node responsible for the task - (this can happen when there is no leader in the cluster). -* An authorization exception occurred. -* An exception occurred during the connection establishment phase. -* The database doesn't exist. - - - - - -An exception may occur while processing a batch of documents in the worker. -For example: - - - -{`_ = workerWBatch.Run(x => throw new Exception()); -`} - - - -When creating a worker, the worker can be configured to handle these exceptions in either of the following ways, -depending on the `IgnoreSubscriberErrors` property in [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions): - -* **Abort processing completely** - When `IgnoreSubscriberErrors = false` (default): - The current batch processing will be aborted, and in this case, the worker will wrap the thrown exception in a `SubscriberErrorException` and will rethrow it. - Processing of the subscription will be terminated without acknowledging progress to the server or retrying to connect. - As a result, the task returned by the `Run` function will complete in an erroneous state, throwing a _SubscriberErrorException_. - -* **Continue processing subsequent batches** - When `IgnoreSubscriberErrors = true`: - The current batch processing will be aborted; however, the erroneous batch will be acknowledged without retrying, - and processing will continue with the next batches. - - - - - -Two properties in the [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) -object control the behavior of a worker attempting to reconnect with the server: - -* `TimeToWaitBeforeConnectionRetry` - The time the worker will wait before attempting to reconnect. - Default: 5 seconds. -* `MaxErroneousPeriod` - The maximum amount of time the subscription connection can remain in an erroneous state. - Once this period is exceeded, the worker will stop trying to reconnect. - Default: 5 minutes. - - - - - -A worker will time out after losing its connectivity with the server for a given time period. - -* The timeout period can be set using the `ConnectionStreamTimeout` option. E.g.: - - -{`var options = new SubscriptionWorkerOptions(subscriptionName); - -// Set the worker's timeout period -options.ConnectionStreamTimeout = TimeSpan.FromSeconds(45); -`} - - -* Default timeout period: 30 second - - - - - -`OnUnexpectedSubscriptionError` is the event that is triggered when a connection failure occurs between the subscription worker and the server, -resulting in an unexpected exception. -When this happens, the worker will automatically attempt to reconnect. -This event is useful for logging these unexpected exceptions. - - - - - -## Worker strategies - -Subscription workers are configured with a **strategy** that determines whether multiple workers -can connect to the subscription concurrently or if only one worker can connect at a time. - -The _one-worker-at-a-time_ strategy also determines how the workers interact with each other -to resolve which will establish the subscription connection. -### One worker per subscription strategies - -The following three strategies allow only a **single worker to connect to the subscription at any given time**, -and determine what happens when one worker is connected and another tries to connect. - -* `SubscriptionOpeningStrategy.OpenIfFree` - The server will allow a worker to connect only if no other worker is currently connected. - If there is an existing connection, the incoming worker will throw a `SubscriptionInUseException`. -* `SubscriptionOpeningStrategy.WaitForFree` - If the worker cannot open the subscription because it is in use by another worker, it will wait for the currently connected worker to disconnect before establishing the connection. - This is useful in worker failover scenarios, where one worker is connected while another is awaiting its turn to take its place. -* `SubscriptionOpeningStrategy.TakeOver` - The server will allow an incoming connection to take over an existing one, - based on the connection strategy in use by the currently connected worker: - * If the existing connection **does not** have a `TakeOver` strategy: - The incoming connection will take over, causing the existing connection to throw a `SubscriptionInUseException`. - * If the existing connection **has** a `TakeOver` strategy: - The incoming connection will throw a `SubscriptionInUseException` exception. -### Multiple workers per subscription strategy - -* `SubscriptionOpeningStrategy.Concurrent` - The server allows multiple workers to connect to the same subscription **concurrently**. - Read more about concurrent subscriptions [here](../../../../client-api/data-subscriptions/concurrent-subscriptions.mdx). - - - -## Determining which workers a subscription will serve - - - -The **strategy used by the first worker connecting to a subscription** determines -which additional workers the subscription can serve until all worker connections are dropped. - - - -* A subscription that serves one or more [concurrent](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy) workers, - **can only serve other concurrent workers** until all connections are dropped. - If a worker with a [one worker per subscription](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#one-worker-per-subscription-strategies) - strategy attempts to connect - - * The connection attempt will be rejected. - * `SubscriptionInUseException` will be thrown. - -* A subscription that serves a worker with a [one worker per subscription](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#one-worker-per-subscription-strategies) strategy, - **cannot** serve [concurrent](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy) - workers until that worker's connection is dropped. - If a concurrent worker attempts to connect - - * The connection attempt will be rejected. - * `SubscriptionInUseException` will be thrown. - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-java.mdx b/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-java.mdx deleted file mode 100644 index ccc4a0db5e..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-java.mdx +++ /dev/null @@ -1,129 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -Subscriptions are consumed by processing batches of documents received from the server. -A `SubscriptionWorker` object manages the documents processing and the communication between the client and the server according to a set of configurations received upon it's creation. -We've introduced several ways to create and configure a SubscriptionWorker, starting from just giving a subscription name, and ending with a detailed configuration object - `SubscriptionWorkerOptions`. - -* In this page: - * [SubscriptionWorker lifecycle](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#subscriptionworker-lifecycle) - * [Error handling](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#error-handling) - * [Worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies) - - -## SubscriptionWorker lifecycle - -A `SubscriptionWorker` object starts its life from being generated by the `DocumentsStore.subscriptions`: - - - -{`subscriptionWorker = store.subscriptions().getSubscriptionWorker(Order.class, subscriptionName); -`} - - - -At this point, the worker has only got its configuration. No connection or processing happens at this moment. -In order to start processing, the `run` method should be called. The `run` method receives the batch processing logic that should be performed: - - - -{`subscriptionRuntimeTask = subscriptionWorker.run(batch -> \{ - // your logic here -\}); -`} - - - -From this point on, the subscription worker will start processing batches. If for any reason, the processing is aborted, the returned task (`subscriptionRuntimeTask`) will complete with an exception. - - - -## Error handling - - - -Subscription worker connection failures may occur during the routine communication between the worker and the server. -When an unexpected error arises, the worker will attempt to **reconnect to the server**. - -However, there are several conditions under which the worker will stop its operation but will Not attempt to reconnect: - -* The subscription no longer exists or has been deleted. -* Another worker has taken control of the subscription (see [connection strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#available-worker-strategies)). -* The worker is unable to connect to any of the servers. -* The worker could not receive the node responsible for the task - (this can happen when there is no leader in the cluster). -* An authorization exception occurred. -* An exception occurred during the connection establishment phase. -* The database doesn't exist. - - - - - -An exception may occur while processing a batch of documents in the worker. -For example: - - - -{`workerWBatch.run(x -> \{ - throw new RuntimeException(); -\}); -`} - - - -When creating a worker, the worker can be configured to handle these exceptions in either of the following ways, -depending on the `IgnoreSubscriberErrors` property in [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions): - -* **Abort processing completely** - When `IgnoreSubscriberErrors` is set to _false_ (default): - The current batch processing will be aborted, and in this case, the worker will wrap the thrown exception in a `SubscriberErrorException` and will rethrow it. - Processing of the subscription will be terminated without acknowledging progress to the server or retrying to connect. - As a result, the task returned by the `Run` function will complete in an erroneous state, throwing a _SubscriberErrorException_. - -* **Continue processing subsequent batches** - When `IgnoreSubscriberErrors` is set to _true_: - The current batch processing will be aborted; however, the erroneous batch will be acknowledged without retrying, - and processing will continue with the next batches. - - - - - -Two properties in the [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) -object control the behavior of a worker attempting to reconnect with the server: - -* `timeToWaitBeforeConnectionRetry` - The time the worker will wait before attempting to reconnect. - Default: 5 seconds. -* `maxErroneousPeriod` - The maximum amount of time the subscription connection can remain in an erroneous state. - Once this period is exceeded, the worker will stop trying to reconnect. - Default: 5 minutes. - - - - - -## Worker strategies - -There can only be one active subscription worker working on a subscription. -Nevertheless, there are scenarios where it is required to interact between an existing subscription worker and one that tries to connect. -This relationship and interoperation is configured by the `SubscriptionConnectionOptions` `Strategy` field. -The strategy field is an enum, having the following values: - -* `OPEN_IF_FREE` - the server will allow the worker to connect only if there isn't any other currently connected workers. - If there is a existing connection, the incoming worker will throw a SubscriptionInUseException. -* `WAIT_FOR_FREE` - If the client currently cannot open the subscription because it is used by another client, it will wait for the previous client to disconnect and only then will connect. - This is useful in client failover scenarios where there is one active client and another one already waiting to take its place. -* `TAKE_OVER` - the server will allow an incoming connection to overthrow an existing one. It will behave according to the existing connection strategy: - * The existing connection has a strategy that is not `TAKE_OVER`. In this case, the incoming connection will take over it causing the existing connection to throw a SubscriptionInUseException exception. - * The existing connection has a strategy that is `TAKE_OVER`. In this case, the incoming connection will throw a SubscriptionInUseException exception. - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-nodejs.mdx b/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-nodejs.mdx deleted file mode 100644 index 83eafbe3aa..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-nodejs.mdx +++ /dev/null @@ -1,204 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* Batches of documents sent from a Subscription Task defined on the server are consumed and processed by a subscription worker client. - -* The `SubscriptionWorker` object, defined on the client, manages the communication between the server and the client and processes the document batches sent from the server. - -* There are several ways to create and configure the SubscriptionWorker - see [subscription worker options](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions). - -* In this page: - * [SubscriptionWorker lifecycle](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#subscriptionworker-lifecycle) - * [Error handling](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#error-handling) - * [Worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies) - * [Determining which workers a subscription will serve](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#determining-which-workers-a-subscription-will-serve) - - -## SubscriptionWorker lifecycle - -Create a `SubscriptionWorker` object by calling `getSubscriptionWorker`: - - - -{`const worker = documentStore.subscriptions.getSubscriptionWorker(\{ - subscriptionName: "your subscription name" -\}); -`} - - - -At this stage, the worker is initialized, no connection to the server or document processing occurs yet. - -To start handling documents from the subscription, you need to define a listener for the `batch` event. -This event is triggered whenever a new batch of documents is received. - -Add an event handler using `on` method of the worker object to process incoming batches: - - - -{`worker.on("batch", (batch, callback) => \{ - try \{ - // Add your logic for processing the incoming batch items here... - - // Call 'callback' once you're done - // The worker will send an acknowledgement to the server, - // allowing the server to send the next batch - callback(); - - \} catch(err) \{ - // If processing fails for a particular batch then pass the error to the callback - callback(err); - \} -\}); -`} - - - -Once the event handler is defined, the worker will begin processing batches of documents sent by the server. -Each batch must be acknowledged by calling `callback()` once processing is complete. - - - -## Error handling - - - -Subscription worker connection failures may occur during the routine communication between the worker and the server. -When an unexpected error arises, the worker will attempt to **reconnect to the server**. - -However, there are several conditions under which the worker will stop its operation but will Not attempt to reconnect: - -* The subscription no longer exists or has been deleted. -* Another worker has taken control of the subscription (see [connection strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#available-worker-strategies)). -* The worker is unable to connect to any of the servers. -* The worker could not receive the node responsible for the task - (this can happen when there is no leader in the cluster). -* An authorization exception occurred. -* An exception occurred during the connection establishment phase. -* The database doesn't exist. - - - - - -An exception may occur while processing a batch of documents in the worker. -For example: - - - -{`worker.on("batch", (batch, callback) => \{ - try \{ - throw new Error("Exception occurred"); - \} catch (err) \{ - callback(err); // Pass the error to the callback to signal failure - \} -\}); -`} - - - -When creating a worker, the worker can be configured to handle these exceptions in either of the following ways, -depending on the `ignoreSubscriberErrors` property in the [subscription worker options](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions): - -* **Abort processing completely** - When `ignoreSubscriberErrors` is `false` (default): - The current batch processing will be aborted, and in this case, the worker will wrap the thrown exception in a `SubscriberErrorException` and will rethrow it. - Processing of the subscription will be terminated without acknowledging progress to the server or retrying to connect. - As a result, the worker task will complete in an erroneous state, throwing a _SubscriberErrorException_. - -* **Continue processing subsequent batches** - When `ignoreSubscriberErrors` is `true`: - The current batch processing will be aborted; however, the erroneous batch will be acknowledged without retrying, - and processing will continue with the next batches. - - - - - -Two properties in the [subscription worker options](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) -object control the behavior of a worker attempting to reconnect with the server: - -* `timeToWaitBeforeConnectionRetry` - The time the worker will wait before attempting to reconnect. - Default: 5 seconds. -* `maxErroneousPeriod` - The maximum amount of time the subscription connection can remain in an erroneous state. - Once this period is exceeded, the worker will stop trying to reconnect. - Default: 5 minutes. - - - - - -`unexpectedSubscriptionError` is the event that is triggered when a connection failure occurs between the subscription worker and the server, -resulting in an unexpected exception. -When this happens, the worker will automatically attempt to reconnect. -This event is useful for logging these unexpected exceptions. - - - - - -## Worker strategies - -Subscription workers are configured with a **strategy** that determines whether multiple workers -can connect to the subscription concurrently or if only one worker can connect at a time. - -The _one-worker-at-a-time_ strategy also determines how the workers interact with each other -to resolve which will establish the subscription connection. -### One worker per subscription strategies - -The following three strategies allow only a **single worker to connect to the subscription at any given time**, -and determine what happens when one worker is connected and another tries to connect. - -* `OpenIfFree` - The server will allow a worker to connect only if no other worker is currently connected. - If there is an existing connection, the incoming worker will throw a `SubscriptionInUseException`. -* `WaitForFree` - If the worker cannot open the subscription because it is in use by another worker, it will wait for the currently connected worker to disconnect before establishing the connection. - This is useful in worker failover scenarios, where one worker is connected while another is awaiting its turn to take its place. -* `TakeOver` - The server will allow an incoming connection to take over an existing one, - based on the connection strategy in use by the currently connected worker: - * If the existing connection **does not** have a `TakeOver` strategy: - The incoming connection will take over, causing the existing connection to throw a `SubscriptionInUseException`. - * If the existing connection **has** a `TakeOver` strategy: - The incoming connection will throw a `SubscriptionInUseException` exception. -### Multiple workers per subscription strategy - -* `Concurrent` - The server allows multiple workers to connect to the same subscription **concurrently**. - Read more about concurrent subscriptions [here](../../../../client-api/data-subscriptions/concurrent-subscriptions.mdx). - - - -## Determining which workers a subscription will serve - - - -The **strategy used by the first worker connecting to a subscription** determines -which additional workers the subscription can serve until all worker connections are dropped. - - - -* A subscription that serves one or more [concurrent](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy) workers, - **can only serve other concurrent workers** until all connections are dropped. - If a worker with a [one worker per subscription](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#one-worker-per-subscription-strategies) - strategy attempts to connect - - * The connection attempt will be rejected. - * `SubscriptionInUseException` will be thrown. - -* A subscription that serves a worker with a [one worker per subscription](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#one-worker-per-subscription-strategies) strategy, - **cannot** serve [concurrent](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy) - workers until that worker's connection is dropped. - If a concurrent worker attempts to connect - - * The connection attempt will be rejected. - * `SubscriptionInUseException` will be thrown. - - - - diff --git a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-python.mdx b/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-python.mdx deleted file mode 100644 index c9f14b40d8..0000000000 --- a/docs/client-api/data-subscriptions/consumption/content/_how-to-consume-data-subscription-python.mdx +++ /dev/null @@ -1,182 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* Batches of documents sent from a Subscription Task defined on the server are consumed and processed by a subscription worker client. - -* The `subscription_worker` object, defined on the client, manages the communication between the server and the client and processes the document batches sent from the server. - -* There are several ways to create and configure the SubscriptionWorker - see `SubscriptionWorkerOptions`. - -* In this page: - * [`subscription_worker` lifecycle](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#subscription_worker-lifecycle) - * [Error handling](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#error-handling) - * [Worker strategies](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies) - * [Determining which workers a subscription will serve](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#determining-which-workers-a-subscription-will-serve) - - -## `subscription_worker` lifecycle - -A `subscription_worker` object starts its life from being generated by the `store.subscriptions`: - - -{`subscription_worker = store.subscriptions.get_subscription_worker_by_name(subscription_name, Order) -`} - - - -At this point, the worker has only got its configuration. No connection or processing happens at this moment. -To start processing, the `run` method should be called. The Run method receives the batch processing logic that should be performed: - - -{`subscription_runtime_task = subscription_worker.run( - process_documents=lambda batch: ... -) # Pass your method that takes SubscriptionBatch[_T] as an argument, with your logic in it -`} - - - -From this point on, the subscription worker will start processing batches. -If processing is aborted for any reason, the returned task (`subscription_runtime_task`) will complete with an exception. - - - -## Error handling - - - -Subscription worker connection failures may occur during the routine communication between the worker and the server. -When an unexpected error arises, the worker will attempt to **reconnect to the server**. - -However, there are several conditions under which the worker will stop its operation but will Not attempt to reconnect: - -* The subscription no longer exists or has been deleted. -* Another worker has taken control of the subscription (see [connection strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#available-worker-strategies)). -* The worker is unable to connect to any of the servers. -* The worker could not receive the node responsible for the task - (this can happen when there is no leader in the cluster). -* An authorization exception occurred. -* An exception occurred during the connection establishment phase. -* The database doesn't exist. - - - - - -An exception may occur while processing a batch of documents in the worker. -For example: - - - -{`def _throw_exception(batch: SubscriptionBatch): - raise Exception() - -_ = worker_w_batch.run(_throw_exception) -`} - - - -When creating a worker, the worker can be configured to handle these exceptions in either of the following ways, -depending on the `ignore_subscriber_errors` property in [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions): - -* **Abort processing completely** - When `ignore_subscriber_errors` is set to _false_ (default): - The current batch processing will be aborted, and in this case, the worker will wrap the thrown exception in a `SubscriberErrorException` and will rethrow it. - Processing of the subscription will be terminated without acknowledging progress to the server or retrying to connect. - As a result, the task returned by the `Run` function will complete in an erroneous state, throwing a _SubscriberErrorException_. - -* **Continue processing subsequent batches** - When `ignore_subscriber_errors` is set to _true_: - The current batch processing will be aborted; however, the erroneous batch will be acknowledged without retrying, - and processing will continue with the next batches. - - - - - -Two properties in the [SubscriptionWorkerOptions](../../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) -object control the behavior of a worker attempting to reconnect with the server: - -* `time_to_wait_before_connection_retry` - The time the worker will wait before attempting to reconnect. - Default: 5 seconds. -* `max_erroneous_period` - The maximum amount of time the subscription connection can remain in an erroneous state. - Once this period is exceeded, the worker will stop trying to reconnect. - Default: 5 minutes. - - - - - -`on_unexpected_subscription_error` is the event that is triggered when a connection failure occurs between the subscription worker and the server, -resulting in an unexpected exception. -When this happens, the worker will automatically attempt to reconnect. -This event is useful for logging these unexpected exceptions. - - - - - -## Worker strategies - -Subscription workers are configured with a **strategy** that determines whether multiple workers -can connect to the subscription concurrently or if only one worker can connect at a time. - -The _one-worker-at-a-time_ strategy also determines how the workers interact with each other -to resolve which will establish the subscription connection. -### One worker per subscription strategies - -The following three strategies allow only a **single worker to connect to the subscription at any given time**, -and determine what happens when one worker is connected and another tries to connect. - -* `SubscriptionOpeningStrategy.OPEN_IF_FREE` - The server will allow a worker to connect only if no other worker is currently connected. - If there is an existing connection, the incoming worker will throw a `SubscriptionInUseException`. -* `SubscriptionOpeningStrategy.WAIT_FOR_FREE` - If the worker cannot open the subscription because it is in use by another worker, it will wait for the currently connected worker to disconnect before establishing the connection. - This is useful in worker failover scenarios, where one worker is connected while another is awaiting its turn to take its place. -* `SubscriptionOpeningStrategy.TAKE_OVER` - The server will allow an incoming connection to take over an existing one, - based on the connection strategy in use by the currently connected worker: - * If the existing connection **does not** have a `TAKE_OVER` strategy: - The incoming connection will take over, causing the existing connection to throw a `SubscriptionInUseException`. - * If the existing connection **has** a `TAKE_OVER` strategy: - The incoming connection will throw a `SubscriptionInUseException` exception. -### Multiple workers per subscription strategy - -* `SubscriptionOpeningStrategy.CONCURRENT` - The server allows multiple workers to connect to the same subscription **concurrently**. - Read more about concurrent subscriptions [here](../../../../client-api/data-subscriptions/concurrent-subscriptions.mdx). - - - -## Determining which workers a subscription will serve - - - -The **strategy used by the first worker connecting to a subscription** determines -which additional workers the subscription can serve until all worker connections are dropped. - - - -* A subscription that serves one or more [CONCURRENT](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy) workers, - **can only serve other concurrent workers** until all connections are dropped. - If a worker with a [one worker per subscription](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#one-worker-per-subscription-strategies) - strategy attempts to connect - - * The connection attempt will be rejected. - * `SubscriptionInUseException` will be thrown. - -* A subscription that serves a worker with a [one worker per subscription](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#one-worker-per-subscription-strategies) strategy, - **cannot** serve [CONCURRENT](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy) - workers until that worker's connection is dropped. - If a concurrent worker attempts to connect - - * The connection attempt will be rejected. - * `SubscriptionInUseException` will be thrown. - - - - diff --git a/docs/client-api/data-subscriptions/consumption/examples.mdx b/docs/client-api/data-subscriptions/consumption/examples.mdx deleted file mode 100644 index 0b0eb565ff..0000000000 --- a/docs/client-api/data-subscriptions/consumption/examples.mdx +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "Subscription Consumption Examples" -sidebar_label: Examples -description: "Practical examples of consuming RavenDB data subscriptions including worker creation, batch processing, error handling, and subscription strategies." -sidebar_position: 1 -supported_languages: ["csharp", "java", "python", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import ExamplesCsharp from './content/_examples-csharp.mdx'; -import ExamplesJava from './content/_examples-java.mdx'; -import ExamplesPython from './content/_examples-python.mdx'; -import ExamplesNodejs from './content/_examples-nodejs.mdx'; - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx b/docs/client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx deleted file mode 100644 index 92635f9660..0000000000 --- a/docs/client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "How to Consume a Data Subscription" -sidebar_label: How to Consume a Data Subscription -description: "Consume RavenDB data subscriptions by creating a worker that processes document batches with configurable strategies and error handling." -sidebar_position: 0 -supported_languages: ["csharp", "java", "python", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import HowToConsumeDataSubscriptionCsharp from './content/_how-to-consume-data-subscription-csharp.mdx'; -import HowToConsumeDataSubscriptionJava from './content/_how-to-consume-data-subscription-java.mdx'; -import HowToConsumeDataSubscriptionPython from './content/_how-to-consume-data-subscription-python.mdx'; -import HowToConsumeDataSubscriptionNodejs from './content/_how-to-consume-data-subscription-nodejs.mdx'; - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/content/_concurrent-subscriptions-csharp.mdx b/docs/client-api/data-subscriptions/content/_concurrent-subscriptions-csharp.mdx deleted file mode 100644 index 7a1ed39051..0000000000 --- a/docs/client-api/data-subscriptions/content/_concurrent-subscriptions-csharp.mdx +++ /dev/null @@ -1,119 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* With **Concurrent Subscriptions**, multiple data subscription workers can connect to the same subscription task simultaneously. - -* Each worker is assigned a different batch of documents to process. - -* By processing different batches in parallel, multiple workers can significantly accelerate the consumption of the subscription's contents. - -* Documents that were assigned to workers whose connection has ended unexpectedly, - can be reassigned by the server to available workers. - See [connection failure](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#connection-failure) below. - -* In this page: - * [Defining concurrent workers](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#defining-concurrent-workers) - * [Dropping a connection](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#dropping-a-connection) - * [Connection failure](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#connection-failure) - - -## Defining concurrent workers - -Concurrent workers are defined similarly to other workers, except their -[strategy](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies) -is set to [SubscriptionOpeningStrategy.Concurrent](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy). - -* To define a concurrent worker: - * Create the worker using [GetSubscriptionWorker](../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker). - * Pass it a [SubscriptionWorkerOptions](../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions) instance. - * Set the strategy to `SubscriptionOpeningStrategy.Concurrent` - -* Usage: - * Define two concurrent workers - - -{`// Define concurrent subscription workers -var subscriptionWorker1 = store.Subscriptions.GetSubscriptionWorker( - // Set the worker to connect to the "All Orders" subscription task - new SubscriptionWorkerOptions("All Orders") - \{ - // Set Concurrent strategy - Strategy = SubscriptionOpeningStrategy.Concurrent, - MaxDocsPerBatch = 20 - \}); - -var subscriptionWorker2 = store.Subscriptions.GetSubscriptionWorker( - new SubscriptionWorkerOptions("All Orders") - \{ - Strategy = SubscriptionOpeningStrategy.Concurrent, - MaxDocsPerBatch = 20 - \}); -`} - - - * Run both workers - - -{`// Start the concurrent worker. Workers will connect concurrently to the "All Orders" subscription task. -var subscriptionRuntimeTask1 = subscriptionWorker1.Run(batch => -\{ - // process batch - foreach (var item in batch.Items) - \{ - // process item - \} -\}); - -var subscriptionRuntimeTask2 = subscriptionWorker2.Run(batch => -\{ - // process batch - foreach (var item in batch.Items) - \{ - // process item - \} -\}); -`} - - - - - -## Dropping a connection - -* Use `Subscriptions.DropSubscriptionWorker` to **forcefully disconnect** - the specified worker from the subscription it is connected to. - - -{`public void DropSubscriptionWorker(SubscriptionWorker worker, string database = null) -`} - - - -* Usage: - - -{`//drop a concurrent subscription worker -store.Subscriptions.DropSubscriptionWorker(subscriptionWorker2); -`} - - - - - -## Connection failure - -* When a concurrent worker's connection ends unexpectedly, - the server may reassign the documents this worker has been processing to any other concurrent worker that is available. -* A worker that reconnects after a connection failure will be assigned a **new** batch of documents. - It is **not** guaranteed that the new batch will contain the same documents this worker was processing before the disconnection. -* As a result, documents may be processed more than once: - - first by a worker that disconnected unexpectedly without acknowledging the completion of its assigned documents, - - and later by other workers the documents are reassigned to. - - - - diff --git a/docs/client-api/data-subscriptions/content/_concurrent-subscriptions-nodejs.mdx b/docs/client-api/data-subscriptions/content/_concurrent-subscriptions-nodejs.mdx deleted file mode 100644 index 1e0836f58b..0000000000 --- a/docs/client-api/data-subscriptions/content/_concurrent-subscriptions-nodejs.mdx +++ /dev/null @@ -1,126 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* With **Concurrent Subscriptions**, multiple data subscription workers can connect to the same subscription task simultaneously. - -* Each worker is assigned a different batch of documents to process. - -* By processing different batches in parallel, multiple workers can significantly accelerate the consumption of the subscription's contents. - -* Documents that were assigned to workers whose connection has ended unexpectedly, - can be reassigned by the server to available workers. - See [connection failure](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#connection-failure) below. - -* In this page: - * [Defining concurrent workers](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#defining-concurrent-workers) - * [Dropping a connection](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#dropping-a-connection) - * [Connection failure](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx#connection-failure) - - -## Defining concurrent workers - -Concurrent workers are defined similarly to other workers, except their -[strategy](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-strategies) -is set to [Concurrent](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#multiple-workers-per-subscription-strategy). - -* To define a concurrent worker: - * Create the worker using [getSubscriptionWorker](../../../client-api/data-subscriptions/consumption/api-overview.mdx#create-the-subscription-worker). - * Pass it a [subscription worker options](../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscription-worker-options) object. - * Set the strategy to `Concurrent` - -* Usage: - * Define two concurrent workers - - -{`// Define 2 concurrent subscription workers -// ======================================== - -const options = \{ - // Set concurrent strategy - strategy: "Concurrent", - subscriptionName: "Get all orders", - maxDocsPerBatch: 20 -\}; - -const worker1 = documentStore.subscriptions.getSubscriptionWorker(options); -const worker2 = documentStore.subscriptions.getSubscriptionWorker(options); -`} - - - * Run both workers - - -{`worker1.on("batch", (batch, callback) => \{ - try \{ - for (const item of batch.items) \{ - // Process item - \} - callback(); - - \} catch(err) \{ - callback(err); - \} -\}); - -worker2.on("batch", (batch, callback) => \{ - try \{ - for (const item of batch.items) \{ - // Process item - \} - callback(); - - \} catch(err) \{ - callback(err); - \} -\}); -`} - - - - - -## Dropping a connection - -* Use `dropSubscriptionWorker` to **forcefully disconnect** - the specified worker from the subscription it is connected to. - -* Use `dropConnection` to disconnect ALL workers connected to the specified subscription. - - - -{`// Drop connection for worker2 -await documentStore.subscriptions.dropSubscriptionWorker(worker2); -`} - - - - - -{`// Available overloads: -dropConnection(options); -dropConnection(options, database); -dropSubscriptionWorker(worker); -dropSubscriptionWorker(worker, database); -`} - - - - - -## Connection failure - -* When a concurrent worker's connection ends unexpectedly, - the server may reassign the documents this worker has been processing to any other concurrent worker that is available. -* A worker that reconnects after a connection failure will be assigned a **new** batch of documents. - It is **not** guaranteed that the new batch will contain the same documents this worker was processing before the disconnection. -* As a result, documents may be processed more than once: - - first by a worker that disconnected unexpectedly without acknowledging the completion of its assigned documents, - - and later by other workers the documents are reassigned to. - - - - diff --git a/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-csharp.mdx b/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-csharp.mdx deleted file mode 100644 index 5948b0ed9a..0000000000 --- a/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-csharp.mdx +++ /dev/null @@ -1,160 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* Data subscriptions provide a reliable and handy way to perform document processing on the client side. -* The server sends batches of documents to the client. - The client then processes the batch and will receive the next one only after it acknowledges the batch was processed. - The server persists the processing progress, allowing you to pause and continue the processing. - -* In this page: - * [Data subscription consumption](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscription-consumption) - * [What defines a data subscription](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#what-defines-a-data-subscription) - * [Documents processing](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#documents-processing) - * [Progress Persistence](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#progress-persistence) - * [How the worker communicates with the server](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#how-the-worker-communicates-with-the-server) - * [Working with multiple clients](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#working-with-multiple-clients) - * [Data subscriptions usage example](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscriptions-usage-example) - - -## Data subscription consumption - -* Data subscriptions are consumed by clients, called **Subscription Workers**. -* You can determine whether workers would be able to connect a subscription - [concurrently, or only one at a time](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-interplay). -* A worker that connects to a data subscription receives a batch of documents, and gets to process it. - Depending on the code that the client provided the worker with, processing can take from seconds to hours. - When all documents are processed, the worker informs the server of its progress and the server can send it the next batch. - - - -## What defines a data subscription - -Data subscriptions are defined by the server-side definition and by the worker connecting to it: - -1. [Subscription Creation Options](../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptions): The documents that will be sent to the worker, it's filtering and projection. - -2. [Subscription Worker Options](../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions): Worker batch processing logic, batch size, interaction with other connections. - - - -## Documents processing - -Documents are sent in batches and progress will be registered only after the whole batch is processed and acknowledged. -Documents are always sent in Etag order which means that data that has already been processed and acknowledged won't be sent twice, except for the following scenarios: - -1. If the document was changed after it was already sent. - -2. If data was received but not acknowledged. - -3. In case of subscription failover (`Enterprise feature`), when there is a chance that documents will be processed again, because it's not always possible to find the same starting point on a different machine. - - -If the database has Revisions defined, the subscription can be configured to process pairs -of subsequent document revisions. -Read more here: [revisions support](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx) - - - - -## Progress Persistence - -* The processing progress is persisted on the server and therefore the subscription - task can be paused and resumed from the last point it was stopped. -* The persistence mechanism also ensures that no documents are missed even in the - presence of failure, whether it's client-side related, communication, or any other disaster. -* Subscriptions progress is stored in the cluster level, in the `Enterprise edition`. - In the case of a node failure, the processing can be automatically failed over to another node. -* The usage of **Change Vectors** allows us to continue from a point that is close to - the last point reached before failure rather than starting the process from scratch. - - -## How the worker communicates with the server - -A worker communicates with the data subscription using a custom protocol on top of a long-lived TCP connection. Each successful batch processing consists of these stages: - -1. The server sends documents in a batch. - -2. Worker sends acknowledgment message after it finishes processing the batch. - -3. The server returns the client a notification that the acknowledgment persistence is done and it is ready to send the next batch. - - -When the responsible node handling the subscription is down, the subscription task can be manually reassigned to another node in the cluster. -With the Enterprise license, the cluster will automatically reassign the work to another node. - - -* The status of the TCP connection is also used to determine the "state" of the worker process. - If the subscription and its workers implement a - [One Worker Per Subscription](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-interplay) - strategy, as long as the connection is alive the server will not allow - other clients to consume the subscription. -* The TCP connection is kept alive and monitored using "heartbeat" messages. - If the connection is found nonfunctional, the current batch progress will be restarted. - -See the sequence diagram below that summarizes the lifespan of a subscription connection. - -![Subscription document processing](../assets/SubscriptionsDocumentProcessing.png) - - - -## Working with multiple clients - -You can use a **Subscription Worker Strategy** to determine whether multiple -workers of the same subscription can connect to it one by one, or **concurrently**. - -* **One Worker Per Subscription Strategies** - The one-worker-per-subscription strategies allow workers of the same subscription - to connect to it **one worker at a time**, with different strategies to support various - inter-worker scenarios. - * One worker is allowed to take the place of another in the processing of a subscription. - Thanks to subscriptions persistence, the worker will be able to continue the work - starting at the point its predecessor got to. - * You can also configure a worker to wait for an existing connection to fail and take - its place, or to force an existing connection to close. - * Read more about these strategies [here](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#one-worker-per-subscription-strategies). - -* **Concurrent Subscription Strategy** - Using the concurrent subscription strategy, multiple workers of the same subscription can - connect to it simultaneously and divide the documents processing load between them to speed it up. - * Batch processing is divided between the multiple workers. - * Connection failure is handled by assigning batches of failing workers to - active available workers. - * Read more about this strategy [here](../../../client-api/data-subscriptions/concurrent-subscriptions.mdx). - - - -## Data subscriptions usage example - -Data subscriptions are accessible by a document store. -Here's an example of creating and using a data subscription: - - - -{`public async Task Worker(IDocumentStore store, CancellationToken cancellationToken) -\{ - // Create the ongoing subscription task on the server - string subscriptionName = await store.Subscriptions - .CreateAsync(x => x.Company == "companies/11"); - - // Create a worker on the client that will consume the subscription - SubscriptionWorker worker = store.Subscriptions - .GetSubscriptionWorker(subscriptionName); - - // Run the worker task and process data received from the subscription - Task workerTask = worker.Run(x => x.Items.ForEach(item => - Console.WriteLine($"Order #\{item.Result.Id\} will be shipped via: \{item.Result.ShipVia\}")), - cancellationToken); - - await workerTask; -\} -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-java.mdx b/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-java.mdx deleted file mode 100644 index 5199e144b2..0000000000 --- a/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-java.mdx +++ /dev/null @@ -1,134 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* Data subscriptions provide a reliable and handy way to perform document processing on the client side. -* The server sends batches of documents to the client. - The client then processes the batch and will receive the next one only after it acknowledges the batch was processed. - The server persists the processing progress, allowing you to pause and continue the processing. - -* In this page: - * [Data subscription consumption](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscription-consumption) - * [What defines a data subscription](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#what-defines-a-data-subscription) - * [Documents processing](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#documents-processing) - * [Progress Persistence](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#progress-persistence) - * [How the worker communicates with the server](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#how-the-worker-communicates-with-the-server) - * [Working with multiple clients](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#working-with-multiple-clients) - * [Data subscriptions usage example](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscriptions-usage-example) - - - -## Data subscription consumption - -Data subscriptions are consumed by clients, called subscription workers. In any given moment, only one worker can be connected to a data subscription. -A worker connected to a data subscription receives a batch of documents and gets to process it. -When it's done, depending on the code that the client gave the worker, it can take from seconds to hours. It informs the server about the progress, and the server is ready to send the next batch. - - - -## What defines a data subscription - -Data subscriptions are defined by the server side definition and by the worker connecting to it: - -1. [Subscription Creation Options](../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptions): The documents that will be received, it's filtering and projection. - -2. [Subscription Worker Options](../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions): Worker batch processing logic, batch size, interaction with other connections. - - - -## Documents processing - -Documents are sent in batches and progress will be registered only after the whole batch is processed and acknowledged. -Documents are always sent in Etag order which means that data that already been processed and acknowledged won't be sent twice, except for the following scenarios: - -1. If the document was changed after it was already sent. - -2. If data was received but not acknowledged. - -3. In case of subscription failover (`Enterprise feature`), when there is a chance that documents will be processed again, because it's not always possible to find the same starting point on a different machine. - - -If the database has Revisions defined, the subscription can be configured to process pairs -of subsequent document revisions. -Read more here: [revisions support](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx) - - - - -## Progress Persistence - -Processing progress is persisted and therefore it can be paused and resumed from the last point it was stopped. -The persistence mechanism also ensures that no documents are missed even in the presence of failure, whether it's client side related, communication, or any other disaster. -Subscriptions progress is stored in the cluster level, in the `Enterprise edition`. In the case of node failure, the processing can be automatically failed over to another node. -The usage of Change Vectors allows us to continue from a point that is close to the last point reached before failure rather than starting the process from scratch. - - -## How the worker communicates with the server - -A worker communicates with the data subscription using a custom protocol on top of a long-lived TCP connection. Each successful batch processing consists of these stages: - -1. The server sends documents a batch. - -2. Worker sends acknowledgment message after it finishes processing the batch. - -3. The server returns the client a notification that the acknowledgment persistence is done and it is ready to send the next batch. - - -When the responsible node handling the subscription is down, the subscription task can be manually reassigned to another node in the cluster. -With the Enterprise license the cluster will automatically reassign the work to another node. - - -The TCP connection is also used as the "state" of the worker process and as long as it's alive, the server will not allow other clients to consume the subscription. -The TCP connection is kept alive and monitored using "heartbeat" messages. If it's found nonfunctional, the current batch progress will be restarted. - -See the sequence diagram below that summarizes the lifetime of a subscription connection. - -![Subscription document processing](../assets/SubscriptionsDocumentProcessing.png) - - - -## Working with multiple clients - -In order to support various inter-worker scenarios, one worker is allowed to take the place of another in the processing of a subscription. -Thanks to subscriptions persistence, the worker will be able to continue the work from the point it's predecessor stopped. - -It's possible to configure that a worker will wait for an existing connection to fail, and take it's place, or we can configure it to force close an existing connection etc. See more in [Workers interplay](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#workers-interplay). - - - -## Data subscriptions usage example - -Data subscriptions are accessible by a document store. Here's an example of an ad-hoc creation and usage of data subscriptions: - - - -{`public void worker(IDocumentStore store) \{ - - // Create the ongoing subscription task on the server - SubscriptionCreationOptions options = new SubscriptionCreationOptions(); - options.setQuery("from Orders where Company = 'companies/11'"); - String subscriptionName = store.subscriptions().create(Order.class, options); - - // Create a worker on the client that will consume the subscription - SubscriptionWorker worker = store - .subscriptions().getSubscriptionWorker(Order.class, subscriptionName); - - // Run the worker task and process data received from the subscription - worker.run(x -> \{ - for (SubscriptionBatch.Item item : x.getItems()) \{ - System.out.println("Order #" - + item.getResult().getId() - + " will be shipped via: " + item.getResult().getShipVia()); - \} - \}); -\} -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-nodejs.mdx b/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-nodejs.mdx deleted file mode 100644 index d0c082823b..0000000000 --- a/docs/client-api/data-subscriptions/content/_what-are-data-subscriptions-nodejs.mdx +++ /dev/null @@ -1,133 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* Data subscriptions provide a reliable and handy way to perform document processing on the client side. -* The server sends batches of documents to the client. - The client then processes the batch and will receive the next one only after it acknowledges the batch was processed. - The server persists the processing progress, allowing you to pause and continue the processing. - -* In this page: - * [Data subscription consumption](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscription-consumption) - * [What defines a data subscription](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#what-defines-a-data-subscription) - * [Documents processing](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#documents-processing) - * [Progress Persistence](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#progress-persistence) - * [How the worker communicates with the server](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#how-the-worker-communicates-with-the-server) - * [Working with multiple clients](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#working-with-multiple-clients) - * [Data subscriptions usage example](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscriptions-usage-example) - - - -## Data subscription consumption - -Data subscriptions are consumed by clients, called subscription workers. In any given moment, only one worker can be connected to a data subscription. -A worker connected to a data subscription receives a batch of documents and gets to process it. -When it's done, depending on the code that the client gave the worker, it can take from seconds to hours. It informs the server about the progress, and the server is ready to send the next batch. - - - -## What defines a data subscription - -Data subscriptions are defined by the server side definition and by the worker connecting to it: - -1. [Subscription Creation Options](../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptions): The documents that will be received, it's filtering and projection. - -2. [Subscription Worker Options](../../../client-api/data-subscriptions/consumption/api-overview.mdx#subscriptionworkeroptions): Worker batch processing logic, batch size, interaction with other connections. - - - -## Documents processing - -Documents are sent in batches and progress will be registered only after the whole batch is processed and acknowledged. -Documents are always sent in Etag order which means that data that already been processed and acknowledged won't be sent twice, except for the following scenarios: - -1. If the document was changed after it was already sent. - -2. If data was received but not acknowledged. - -3. In case of subscription failover (`Enterprise feature`), when there is a chance that documents will be processed again, because it's not always possible to find the same starting point on a different machine. - - -If the database has Revisions defined, the subscription can be configured to process pairs -of subsequent document revisions. -Read more here: [revisions support](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx) - - - - -## Progress Persistence - -Processing progress is persisted and therefore it can be paused and resumed from the last point it was stopped. -The persistence mechanism also ensures that no documents are missed even in the presence of failure, whether it's client side related, communication, or any other disaster. -Subscriptions progress is stored in the cluster level, in the `Enterprise edition`. In the case of node failure, the processing can be automatically failed over to another node. -The usage of Change Vectors allows us to continue from a point that is close to the last point reached before failure rather than starting the process from scratch. - - -## How the worker communicates with the server - -A worker communicates with the data subscription using a custom protocol on top of a long-lived TCP connection. Each successful batch processing consists of these stages: - -1. The server sends documents a batch. - -2. Worker sends acknowledgment message after it finishes processing the batch. - -3. The server returns the client a notification that the acknowledgment persistence is done and it is ready to send the next batch. - - -When the responsible node handling the subscription is down, the subscription task can be manually reassigned to another node in the cluster. -With the Enterprise license the cluster will automatically reassign the work to another node. - - -The TCP connection is also used as the "state" of the worker process and as long as it's alive, the server will not allow other clients to consume the subscription. -The TCP connection is kept alive and monitored using "heartbeat" messages. If it's found nonfunctional, the current batch progress will be restarted. - -See the sequence diagram below that summarizes the lifetime of a subscription connection. - -![Subscription document processing](../assets/SubscriptionsDocumentProcessing.png) - - - -## Working with multiple clients - -In order to support various inter-worker scenarios, one worker is allowed to take the place of another in the processing of a subscription. -Thanks to subscriptions persistence, the worker will be able to continue the work from the point it's predecessor stopped. - -It's possible to configure that a worker will wait for an existing connection to fail, and take it's place, or we can configure it to force close an existing connection etc. See more in [Workers interplay](../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#workers-interplay). - - - -## Data subscriptions usage example - -Data subscriptions are accessible by a document store. Here's an example of an ad-hoc creation and usage of data subscriptions: - - - -{`async function worker() \{ - - // Create the ongoing subscription task on the server - const subscriptionName = await store.subscriptions.create(\{ - query: "from Orders where Company = 'companies/11'" - \}); - - // Create a worker on the client that will consume the subscription - const worker = store.subscriptions.getSubscriptionWorker(subscriptionName); - - // Listen for and process data received in batches from the subscription - worker.on("batch", (batch, callback) => \{ - for (const item of batch.items) \{ - console.log(\`Order #$\{item.result.Id\} will be shipped via: $\{item.result.ShipVia\}\`); - \} - - callback(); - \}); -\} -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/creation/_category_.json b/docs/client-api/data-subscriptions/creation/_category_.json deleted file mode 100644 index 696f998ee4..0000000000 --- a/docs/client-api/data-subscriptions/creation/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "position": 1, - "label": Creation, -} \ No newline at end of file diff --git a/docs/client-api/data-subscriptions/creation/api-overview.mdx b/docs/client-api/data-subscriptions/creation/api-overview.mdx deleted file mode 100644 index 04e5026a6d..0000000000 --- a/docs/client-api/data-subscriptions/creation/api-overview.mdx +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: "Create and Update Subscription API" -sidebar_label: API Overview -description: "API reference for creating RavenDB data subscriptions, covering creation options, filtering, projections, and subscription configuration." -sidebar_position: 2 -supported_languages: ["csharp", "python", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" - - title: "JavaScript Engine" - link: "server/kb/javascript-engine" - source: "docs" - path: "Server > Knowledge Base" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import ApiOverviewCsharp from './content/_api-overview-csharp.mdx'; -import ApiOverviewPython from './content/_api-overview-python.mdx'; -import ApiOverviewNodejs from './content/_api-overview-nodejs.mdx'; - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_api-overview-csharp.mdx b/docs/client-api/data-subscriptions/creation/content/_api-overview-csharp.mdx deleted file mode 100644 index dc05ade4dd..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_api-overview-csharp.mdx +++ /dev/null @@ -1,277 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Create subscription](../../../../client-api/data-subscriptions/creation/api-overview.mdx#create-subscription) - * [Subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options) - * [Update subscription](../../../../client-api/data-subscriptions/creation/api-overview.mdx#update-subscription) - * [Subscription update options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-update-options) - * [Subscription query](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-query) - - -## Create subscription - -Subscriptions can be created using the following `Create` methods available through the `Subscriptions` property of the `DocumentStore`. - - - -{`string Create(SubscriptionCreationOptions options, - string database = null); - -string Create(SubscriptionCreationOptions options = null, - string database = null); - -string Create(SubscriptionCreationOptions options, - string database = null); - -string Create(Expression> predicate = null, - PredicateSubscriptionCreationOptions options = null, - string database = null); - -Task CreateAsync(SubscriptionCreationOptions options, - string database = null, - CancellationToken token = default); - -public Task CreateAsync(SubscriptionCreationOptions options = null, - string database = null, - CancellationToken token = default); - -Task CreateAsync(SubscriptionCreationOptions options, - string database = null, - CancellationToken token = default); - -Task CreateAsync(Expression> predicate = null, - PredicateSubscriptionCreationOptions options = null, - string database = null, - CancellationToken token = default); -`} - - - -| Parameter | Type | Description | -|----------------|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **predicate** | `Expression>` | An optional lambda expression that returns a boolean.
This predicate defines the filter criteria for the subscription documents. | -| **options** | `SubscriptionCreationOptions` | Contains subscription creation options.
See [Subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options) | -| **options** | `SubscriptionCreationOptions` | Contains subscription creation options
(non-generic version).
See [Subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options) | -| **options** | `PredicateSubscriptionCreationOptions ` | Contains subscription creation options
(when passing a predicate).
See [Subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options) | -| **database** | `string` | The name of the database where the subscription task will be created. If `null`, the default database configured in the DocumentStore will be used. | -| **token** | `CancellationToken` | Cancellation token used in to halt the subscription creation process. | - -| Return value | Description | -|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `string` | The name of the created data subscription.
If the name was provided in `SubscriptionCreationOptions`, it will be returned.
Otherwise, a unique name will be generated by server. | - - - -## Subscription creation options - - - -Options for the **generic** version of the subscription creation options object: - - -{`public class SubscriptionCreationOptions -\{ - public string Name \{ get; set; \} - public Expression> Filter \{ get; set; \} - public Expression> Projection \{ get; set; \} - public Action> Includes \{ get; set; \} - public string ChangeVector \{ get; set; \} - public bool Disabled \{ get; set; \} - public string MentorNode \{ get; set; \} - public bool PinToMentorNode \{ get; set; \} - public ArchivedDataProcessingBehavior? ArchivedDataProcessingBehavior \{ get; set; \} -\} -`} - - - - - - -Options for the **non-generic** version of the subscription creation options object: - - -{`public class SubscriptionCreationOptions -\{ - public string Name \{ get; set; \} - public string Query \{ get; set; \} - public string ChangeVector \{ get; set; \} - public virtual bool Disabled \{ get; set; \} - public string MentorNode \{ get; set; \} - public virtual bool PinToMentorNode \{ get; set; \} - public ArchivedDataProcessingBehavior? ArchivedDataProcessingBehavior \{ get; set; \} -\} -`} - - - - - - -Options for the **non-generic** version of the subscription creation options object when passing a **predicate**: - - -{`public sealed class PredicateSubscriptionCreationOptions -\{ - public string Name \{ get; set; \} - public string ChangeVector \{ get; set; \} - public bool Disabled \{ get; set; \} - public string MentorNode \{ get; set; \} - public bool PinToMentorNode \{ get; set; \} - public ArchivedDataProcessingBehavior? ArchivedDataProcessingBehavior \{ get; set; \} -\} -`} - - - - - -| Member | Type | Description | -|------------------------------------|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **<T>** | `T` | Type of object from which the collection of documents managed by the subscription will be derived. | -| **Name** | `string` | User defined name for the subscription.
The name must be unique in the database. | -| **Query** | `string` | RQL query that defines the subscription. This RQL comes with additional support to JavaScript functions inside the `where` clause and special semantics for subscriptions on documents revisions. | -| **Filter** | `Expression>` | Lambda expression defining the filter logic for the subscription. Will be translated to a JavaScript function. | -| **Projection** | `Expression>` | Lambda expression defining the projection that will be sent by the subscription for each matching document. Will be translated to a JavaScript function. | -| **Includes** | `Action>` | An action that defines include clauses for the subscription. [Included documents](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-documents) and/or [included counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters) will be part of the batch sent by the subscription. Include methods can be chained. | -| **ChangeVector** | `string` | Allows to define a change vector from which the subscription will start processing.
Learn more [below](../../../../client-api/data-subscriptions/creation/api-overview.mdx#the--property). | -| **Disabled** | `bool` | `true` - task will be disabled.
`false` - task will be enabled. | -| **MentorNode** | `string` | Allows to define a node in the cluster that will be responsible to handle the subscription. Useful when you prefer a specific server due to its stronger hardware, closer geographic proximity to clients, or other reasons. | -| **PinToMentorNode** | `bool` | `true` - the selected responsible node will be pinned to handle the task.
`false` - Another node will execute the task if the responsible node is down. | -| **ArchivedDataProcessingBehavior** | `ArchivedDataProcessingBehavior?` | Define whether [archived documents](../../../../data-archival/archived-documents-and-other-features.mdx#archived-documents-and-subscriptions) will be included in the subscription. | - - - -###### The `ChangeVector` property: - -* The _ChangeVector_ property allows you to define a starting point from which the subscription will begin processing changes. -* This is useful for ad-hoc processes that need to process only recent changes. In such cases, you can: - * Set the field to _"LastDocument"_ to start processing from the latest document in the collection. - * Or, provide an actual Change Vector to begin processing from a specific point. -* By default, the subscription will send all documents matching the RQL query, regardless of their creation time. - - - - - -## Update subscription - -Existing subscriptions can be modified using the following `Update` methods available through the `Subscriptions` property of the `DocumentStore`. - - - -{`string Update(SubscriptionUpdateOptions options, string database = null); - -Task UpdateAsync(SubscriptionUpdateOptions options, string database = null, - CancellationToken token = default); -`} - - - -| Parameter | Type | Description | -|--------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **options** | `SubscriptionUpdateOptions` | The subscription update options object.
See [SubscriptionUpdateOptions](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptionupdateoptions) | -| **database** | `string` | The name of the database where the subscription task resides.
If `null`, the default database configured in the DocumentStore will be used. | -| **token** | `CancellationToken` | Cancellation token used to halt the update process. | - -| Return value | Description | -|---------------|--------------------------------------------| -| `string` | The name of the updated data subscription. | - - - -## Subscription update options - -`SubscriptionUpdateOptions` inherits from [SubscriptionCreationOptions](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptions) -and adds two additional fields: - - - -{`public class SubscriptionUpdateOptions : SubscriptionCreationOptions -\{ - public long? Id \{ get; set; \} - public bool CreateNew \{ get; set; \} -\} -`} - - - -| Parameter | Type | Description | -|---------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Id** | `long?` | The unique ID that was assigned to the subscription by the server at creation time.
You can retrieve it by [getting the subscription status](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#getting-subscription-status).
When updating, the `Id` can be used instead of the `Name` field, and takes precedence over it. This allows you to modify the subscription's name: provide the Id and submit a new name in the Name field. | -| **CreateNew** | `bool` | Determines the behavior when the subscription you wish to update does Not exist.
`true` - a new subscription is created with the provided option parameters.
`false` - an exception will be thrown.
Default: `false` | - - - -## Subscription query - -All subscriptions are eventually translated to an RQL-like statement. These statements have the following parts: - -* Functions definition part, like in ordinary RQL. Those functions can contain any JavaScript code, - and also supports `load` and `include` operations. - -* From statement, defining the documents source, ex: `from Orders`. The from statement can only address collections, therefore, indexes are not supported. - -* Where statement describing the criteria according to which it will be decided to either - send the documents to the worker or not. Those statements support either RQL like `equality` operations (`=`, `==`) , - plain JavaScript expressions or declared function calls, allowing to perform complex filtering logic. - The subscriptions RQL does not support any of the known RQL searching keywords. - -* Select statement, that defines the projection to be performed. - The select statements can contain function calls, allowing complex transformations. - -* Include statement allowing to define include path in document. - - -Although subscription's query syntax has an RQL-like structure, it supports only the `declare`, `select` and `where` keywords, usage of all other RQL keywords is not supported. -Usage of JavaScript ES5 syntax is supported. - - - -Paths in subscriptions RQL statements are treated as JavaScript indirections and not like regular RQL paths. -It means that a query that in RQL would look like: - -``` -from Orders as o -where o.Lines[].Product = "products/1-A" -``` - -Will look like that in subscriptions RQL: - -``` -declare function filterLines(doc, productId) -{ - if (!!doc.Lines){ - return doc.Lines.filter(x=>x.Product == productId).length >0; - } - return false; -} - -from Orders as o -where filterLines(o, "products/1-A") -``` - - - - -To define a data subscription that sends document revisions to the client, -you must first [configure revisions](../../../../document-extensions/revisions/overview.mdx#defining-a-revisions-configuration) -for the specific collection managed by the subscription. - -The subscription should be defined in a special way: - -* In case of the generic API, the `SubscriptionCreationOptions<>` generic parameter should be of the generic type `Revision<>`, - while it's generic parameter correlates to the collection to be processed. Ex: `new SubscriptionCreationOptions>()` -* For RQL syntax, concatenate the `(Revisions = true)` clause to the collection being queried. - For example: `From Orders(Revisions = true) as o` - - - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_api-overview-nodejs.mdx b/docs/client-api/data-subscriptions/creation/content/_api-overview-nodejs.mdx deleted file mode 100644 index 154528cb19..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_api-overview-nodejs.mdx +++ /dev/null @@ -1,285 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Create subscription](../../../../client-api/data-subscriptions/creation/api-overview.mdx#create-subscription) - * [Subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options) - * [Include methods](../../../../client-api/data-subscriptions/creation/api-overview.mdx#include-methods) - * [Update subscription](../../../../client-api/data-subscriptions/creation/api-overview.mdx#update-subscription) - * [Subscription update options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-update-options) - * [Subscribe to revisions](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscribe-to-revisions) - * [Subscription RQL](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-rql) - - -## Create subscription - -Subscriptions can be created using the following `create` methods available through the `subscriptions` property of the `DocumentStore`. - - - -{`// Available overloads: -// ==================== - -create(options); - -create(options, database); - -create(documentType); -`} - - - -| Parameter | Type | Description | -|------------------|-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| **options** | `object` | The [subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options). | -| **database** | `string` | The name of the database where the subscription task will be created.
If `null`, the default database configured in the DocumentStore will be used. | -| **documentType** | `object` | The class type from which the collection of documents managed by the subscription will be derived. | - -| Return value | Description | -|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `Promise` | A Promise that resolves to the **name** of the created data subscription (a `string`).
If the name was provided in the [subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options), it will be returned.
Otherwise, a unique name will be generated by server. | - -Examples for creating subscriptions are available [here](../../../../client-api/data-subscriptions/creation/examples.mdx). - - - -## Subscription creation options - - - -{`// The SubscriptionCreationOptions object: -// ======================================= -\{ - name; - query; - includes; - changeVector; - mentorNode; - pinToMentorNode; - disabled; - documentType; -\} -`} - - - -| Member | Type | Description | -|---------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **name** | `string` | User defined name for the subscription.
The name must be unique in the database. | -| **query** | `string` | RQL query that defines the subscription. This RQL comes with additional support to JavaScript functions inside the `where` clause and special semantics for subscriptions on documents revisions.
Learn more in [subscription RQL](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-rql). | -| **includes** | `(builder) => void` | A function that accepts a builder object, which allows you to include related documents, counters, and time series in the batch that is sent to the client.
See [Include methods](../../../../client-api/data-subscriptions/creation/api-overview.mdx#include-methods). | -| **changeVector** | `string` | Allows to define a change vector from which the subscription will start processing. Useful for ad-hoc processes that need to process only recent changes. In such cases, you can set the field to _"LastDocument"_ to start processing from the latest document in the collection. | -| **mentorNode** | `string` | Allows to define a specific node in the cluster to handle the subscription. Useful when you prefer a specific server due to its stronger hardware, closer geographic proximity to clients, or other reasons. | -| **pinToMentorNode** | `boolean` | `true` - task will only be handled by the specified mentor node.
`false` - When the specified mentor node is down, the cluster selects another node from the Database Group to handle the task.
Learn more in [pinning a task](../../../../server/clustering/distribution/highly-available-tasks.mdx#pinning-a-task). | -| **disabled** | `boolean` | `true` - the created subscription will be in a disabled state.
`false` (default) - the created subscription will be enabled. | -| **documentType** | `object` | The class type from which the collection of documents managed by the subscription will be derived. | - - - - -## Include methods - -**Including documents**: - - - -{`includeDocuments(path); -`} - - - -| Parameter | Type | Description | -|-----------|------------|----------------------------------------------------------------| -| **path** | `string` | Path to the property which contains ID of document to include. | - -An example of including documents is available [here](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-documents). -**Including counters**: - - - -{`// Include a single counter -includeCounter(name); - -// Include multiple counters -includeCounters(names); - -// Include ALL counters from ALL documents that match the subscription criteria -includeAllCounters(); -`} - - - -| Parameter | Type | Description | -|------------|------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| **name** | `string` | The name of a counter. The subscription will include all counters with this name that are contained in the documents the subscription retrieves. | -| **names** | `string[]` | Array of counter names. | - -An example of including counters is available [here](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). -**Including time series**: - - - -{`includeTimeSeries(name, type, time); -includeTimeSeries(name, type, count); - -includeTimeSeries(names, type, time); -includeTimeSeries(names, type, count); - -includeAllTimeSeries(type, time); -includeAllTimeSeries(type, count); -`} - - - -| Parameter | Type | Description | -|-------------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **name** | `string` | The name of the time series to include. | -| **names** | `string[]` | The names of the time series to include. | -| **type** | `string` | Indicates how to retrieve the time series entries.
Range type can be: `"None"` or `"Last"`.
When set to _Last_, retrieve the last X entries, where X is determined by _count_. | -| **time** | `TimeValue` | The time range to consider when retrieving time series entries.
E.g.: `TimeValue.ofDays(7)` | -| **count** | `number` | The maximum number of entries to take when retrieving time series entries. | - - - -## Update subscription - -Existing subscriptions can be modified using the following `update` methods available through the `subscriptions` property of the `DocumentStore`. - - - -{`// Available overloads: -// ==================== - -update(options); - -update(options, database); -`} - - - -| Parameter | Type | Description | -|--------------|---------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| **options** | `object` | The [subscription update options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-update-options). | -| **database** | `string` | The name of the database where the subscription task resides.
If `null`, the default database configured in the DocumentStore will be used. | - -| Return value | Description | -|---------------|----------------------------------------------------------------------------------------| -| `Promise` | A Promise that resolves to the **name** of the updated data subscription (a `string`). | - -Examples for updating an existing subscription are available [here](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription). - - - -## Subscription update options - -The subscription update options object extends the [creation options object](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options) -and adds two additional fields: - - - -{`// The SubscriptionUpdateOptions object: -// ===================================== -\{ - id; - createNew; -\} -`} - - - -| Parameter | Type | Description | -|---------------|-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **id** | `number` | The unique ID that was assigned to the subscription by the server at creation time.
You can retrieve it by [getting the subscription status](../../../../client-api/data-subscriptions/advanced-topics/maintenance-operations.mdx#getting-subscription-status).
When updating, the `id` can be used instead of the `name` field, and takes precedence over it. This allows you to modify the subscription's name: provide the id and submit a new name in the name field. | -| **createNew** | `boolean` | Determines the behavior when the subscription you wish to update does Not exist.
`true` - a new subscription is created with the provided option parameters.
`false` - an exception will be thrown.
Default: `false` | - - - - -## Subscribe to revisions - -To define a data subscription that sends document revisions to the client, -you must first [configure revisions](../../../../document-extensions/revisions/overview.mdx#defining-a-revisions-configuration) -for the specific collection managed by the subscription. - -Create a subscription that sends document revisions using the following `createForRevisions` methods: - - - -{`// Available overloads: -// ==================== - -createForRevisions(options); - -createForRevisions(options, database); -`} - - - -| Parameter | Type | Description | -|--------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| **options** | `object` | The [subscription creation options](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation-options). | -| **database** | `string` | The name of the database where the subscription task will be created.
If `null`, the default database configured in the DocumentStore will be used. | - -When providing raw RQL to the `query` param in the options object, -concatenate the `(Revisions = true)` clause to the collection being queried. -For example: `From Orders(Revisions = true) as o` - -Learn more about subscribing to revisions in [revisions support](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx). - - - -## Subscription RQL - -All subscriptions are eventually translated to an RQL-like statement. These statements have the following parts: - -* Functions definition part, like in ordinary RQL. Those functions can contain any JavaScript code, - and also supports `load` and `include` operations. - -* From statement, defining the documents source, ex: `from Orders`. The from statement can only address collections, therefore, indexes are not supported. - -* Where statement describing the criteria according to which it will be decided to either - send the documents to the worker or not. Those statements support either RQL like `equality` operations (`=`, `==`) , - plain JavaScript expressions or declared function calls, allowing to perform complex filtering logic. - The subscriptions RQL does not support any of the known RQL searching keywords. - -* Select statement, that defines the projection to be performed. - The select statements can contain function calls, allowing complex transformations. - -* Include statement allowing to define include path in document. - - -Although subscription's query syntax has an RQL-like structure, it supports only the `declare`, `select` and `where` keywords, usage of all other RQL keywords is not supported. -Usage of JavaScript ES5 syntax is supported. - - - -Paths in subscriptions RQL statements are treated as JavaScript indirections and not like regular RQL paths. -It means that a query that in RQL would look like: - -``` -from Orders as o -where o.Lines[].Product = "products/1-A" -``` - -Will look like that in subscriptions RQL: - -``` -declare function filterLines(doc, productId) -{ - if (!!doc.Lines){ - return doc.Lines.filter(x=>x.Product == productId).length >0; - } - return false; -} - -from Orders as o -where filterLines(o, "products/1-A") -``` - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_api-overview-python.mdx b/docs/client-api/data-subscriptions/creation/content/_api-overview-python.mdx deleted file mode 100644 index 42a4912648..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_api-overview-python.mdx +++ /dev/null @@ -1,179 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* In this page: - * [Create subscription](../../../../client-api/data-subscriptions/creation/api-overview.mdx#create-subscription) - * [SubscriptionCreationOptions](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptions) - * [Update subscription](../../../../client-api/data-subscriptions/creation/api-overview.mdx#update-subscription) - * [SubscriptionUpdateOptions](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptionupdateoptions) - * [Subscription query](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-query) - - -## Create subscription - -Subscriptions can be created using the `create_for_options` and `create_for_class` methods. - - -{`def create_for_options(self, options: SubscriptionCreationOptions, database: Optional[str] = None) -> str: ... - -def create_for_class( - self, - object_type: Type[_T], - options: Optional[SubscriptionCreationOptions] = None, - database: Optional[str] = None, -) -> str: ... -`} - - - -| Parameter | Type | Description | -| ------------- | ------------- | ----- | -| **options** | `SubscriptionCreationOptions` | Contains subscription creation options | -| **database** (Optional) | `[str]` | The name of the database where the subscription task will be created. If `None`, default database configured in DocumentStore will be used. | -| **object_type** | `Type[_T]` | Predicate describing the subscription documents filter | - -| Return value | Description | -| ------------- | ----- | -| `str` | Created data subscription name. If the name was provided in `SubscriptionCreationOptions`, it will be returned. Otherwise, a unique name will be generated by the server. | - - - -## SubscriptionCreationOptions - -An RQL statement will be built based on the fields. - - -{`class SubscriptionCreationOptions: - def __init__( - self, - name: Optional[str] = None, - query: Optional[str] = None, - includes: Optional[Callable[[SubscriptionIncludeBuilder], None]] = None, - change_vector: Optional[str] = None, - mentor_node: Optional[str] = None, - ): - self.name = name - self.query = query - self.includes = includes - self.change_vector = change_vector - self.mentor_node = mentor_node -`} - - - -| Member | Type | Description | -|------------------------------|:-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **name** (Optional) | `str` | User-defined name of the subscription: allows to have a human readable identification of a subscription. The name must be unique in the database. | -| **query** (Optional) | `str` | RQL query that describes the subscription. This RQL comes with additional support to JavaScript functions inside the `where` clause and special semantics for subscriptions on documents revisions. | -| **change_vector** (Optional) | `str` | Allows to define a change vector from which the subscription will start processing. Useful for ad-hoc processes that need to process only recent changes. In such cases, you can set the field to _"LastDocument"_ to start processing from the latest document in the collection. | -| **mentor_node** (Optional) | `str` | Allows to define a specific node in the cluster to handle the subscription. Useful when you prefer a specific server due to its stronger hardware, closer geographic proximity to clients, or other reasons. | -| **includes** (Optional) | `[Callable[[SubscriptionIncludeBuilder]` | Action with a [SubscriptionIncludeBuilder](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-documents) parameter that allows you to define an include clause for the subscription. Methods can be chained to include documents as well as [counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). | - - - -## Update subscription - -Modifies an existing data subscription. These methods are accessible at `DocumentStore.Subscriptions`. - - - -{`def update(self, options: SubscriptionUpdateOptions, database: Optional[str] = None) -> str: ... -`} - - - -| Parameter | Type | Description | -| - | - | - | -| **options** | `SubscriptionUpdateOptions` | A subscription update options object | -| **database** (Optional) | `str` | The name of the database where the subscription task will be created. If `None`, default database configured in DocumentStore will be used. | - -| Return value | Description | -| ------------- | ----- | -| `str` | The updated data subscription's name. | - - - -## SubscriptionUpdateOptions - -Inherits from `SubscriptionCreationOptions` and has all the same fields (see [above](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptions)) plus the two additional fields described below: - - - -{`class SubscriptionUpdateOptions(SubscriptionCreationOptions): - def __init__( - self, - name: Optional[str] = None, - query: Optional[str] = None, - includes: Optional[Callable[[SubscriptionIncludeBuilder], None]] = None, - change_vector: Optional[str] = None, - mentor_node: Optional[str] = None, - key: Optional[int] = None, - create_new: Optional[bool] = None, - ): ... -`} - - - -| Parameter | Type | Description | -| - | - | - | -| **key** (Optional) | `int` | Unique server-side ID of the data subscription. `key` can be used instead of the subscription update options `name` field, and takes precedence over it. This allows you to change the subscription's name: submit a subscription's ID, and submit a different name in the `name` field. | -| **create_new** (Optional) | `bool` | Determines the behavior when the subscription you wish to update does Not exist.
`true` - a new subscription is created with the provided option parameters.
`false` - an exception will be thrown.
Default: `false` | - - - -## Subscription query - -All subscriptions are eventually translated to an RQL-like statement. These statements have the following parts: - -* Functions definition part, like in ordinary RQL. Those functions can contain any JavaScript code, - and also supports `load` and `include` operations. - -* From statement, defining the documents source, ex: `from Orders`. The from statement can only address collections, therefore, indexes are not supported. - -* Where statement describing the criteria according to which it will be decided to either - send the documents to the worker or not. Those statements support either RQL like `equality` operations (`=`, `==`) , - plain JavaScript expressions or declared function calls, allowing to perform complex filtering logic. - The subscriptions RQL does not support any of the known RQL searching keywords. - -* Select statement, that defines the projection to be performed. - The select statements can contain function calls, allowing complex transformations. - -* Include statement allowing to define include path in document. - - -Although subscription's query syntax has an RQL-like structure, it supports only the `declare`, `select` and `where` keywords, usage of all other RQL keywords is not supported. -Usage of JavaScript ES5 syntax is supported. - - - -Paths in subscriptions RQL statements are treated as JavaScript indirections and not like regular RQL paths. -It means that a query that in RQL would look like: - -``` -from Orders as o -where o.Lines[].Product = "products/1-A" -``` - -Will look like that in subscriptions RQL: - -``` -declare function filterLines(doc, productId) -{ - if (!!doc.Lines){ - return doc.Lines.filter(x=>x.Product == productId).length >0; - } - return false; -} - -from Orders as o -where filterLines(o, "products/1-A") -``` - - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_examples-csharp.mdx b/docs/client-api/data-subscriptions/creation/content/_examples-csharp.mdx deleted file mode 100644 index 4c71d9684a..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_examples-csharp.mdx +++ /dev/null @@ -1,530 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* This page contains examples of **creating a subscription**. - To learn how to consume and process documents sent by the subscription, see these [examples](../../../../client-api/data-subscriptions/consumption/examples.mdx). - -* For a detailed syntax of the available subscription methods and objects, see this [API overview](../../../../client-api/data-subscriptions/creation/api-overview.mdx). - -* In this page: - * [Create subscription - for all documents in a collection](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---for-all-documents-in-a-collection) - * [Create subscription - filter documents](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-documents) - * [Create subscription - filter documents using regex](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-documents-using-regex) - * [Create subscription - filter and project fields](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-and-project-fields) - * [Create subscription - project data from a related document](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---project-data-from-a-related-document) - * [Create subscription - include documents](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-documents) - * [Create subscription - include counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters) - * [Create subscription - subscribe to revisions](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---subscribe-to-revisions) - * [Create subscription - via update](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---via-update) - * [Update existing subscription](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription) - - -## Create subscription - for all documents in a collection - -Here we create a plain subscription on the _Orders_ collection without any constraints or transformations. -The server will send ALL documents from the _Orders_ collection to a client that connects to this subscription. - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions -{ - // Set a custom name for the subscription - Name = "OrdersProcessingSubscription" -}); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = "From Orders", - Name = "OrdersProcessingSubscription" -}); -`} - - - - - - -## Create subscription - filter documents - -Here we create a subscription for documents from the _Orders_ collection where the total order revenue is greater than 100. -Only documents that match this condition will be sent from the server to a client connected to this subscription. - - - - -{`subscriptionName = store.Subscriptions.Create(x => - // Only documents matching this criteria will be sent - x.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 100); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"declare function getOrderLinesSum(doc) { - var sum = 0; - for (var i in doc.Lines) { - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - } - return sum; - } - - From Orders as o - Where getOrderLinesSum(o) > 100" -}); -`} - - - - - - -## Create subscription - filter documents using regex - -Here we create a subscription for documents from the _Orders_ collection where the `ShipTo.City` field matches a regular expression pattern. -You can use `Regex.IsMatch` within the subscription's LINQ filter expression to perform server-side pattern matching. - - - - -{`// Filter using Regex.IsMatch in the subscription's LINQ filter -subscriptionName = store.Subscriptions.Create(x => - Regex.IsMatch(x.ShipTo.City, "^New")); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"From Orders - Where regex(ShipTo.City, '^New')" -}); -`} - - - - -You can pass supported `RegexOptions` as a third argument: - - -{`// Filter with case-insensitive regex matching -subscriptionName = store.Subscriptions.Create(x => - Regex.IsMatch(x.ShipTo.City, "^new", RegexOptions.IgnoreCase)); -`} - - - - -The following `RegexOptions` are supported: -* `RegexOptions.IgnoreCase` -* `RegexOptions.Multiline` -* `RegexOptions.Singleline` - -Other `RegexOptions` values (e.g. `Compiled`, `ExplicitCapture`) and the timeout overload are **not supported** and will throw a `NotSupportedException`. - -The regex pattern must be a **constant expression** — it cannot be a variable computed at runtime from document fields. - - - - - -## Create subscription - filter and project fields - -Here, again, we create a subscription for documents from the _Orders_ collection where the total order revenue is greater than 100. -However, this time we only project the document ID and the Total Revenue properties in each object sent to the client. - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - // The subscription criteria: - Filter = x => x.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 100, - - // The object properties that will be sent for each matching document: - Projection = x => new - { - Id = x.Id, - Total = x.Lines.Sum(line => line.PricePerUnit * line.Quantity) - } -}); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"declare function getOrderLinesSum(doc) { - var sum = 0; - for (var i in doc.Lines) { - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - } - return sum; - } - - declare function projectOrder(doc) { - return { - Id: doc.Id, - Total: getOrderLinesSum(doc) - }; - } - - From Orders as o - Where getOrderLinesSum(o) > 100 - Select projectOrder(o)" -}); -`} - - - - - - -## Create subscription - project data from a related document - -In this subscription, in addition to projecting the document fields, -we also project data from a [related document](../../../../indexes/indexing-related-documents.mdx#what-are-related-documents) that is loaded using the `Load` method. - - - - -{`subscriptionName = store.Subscriptions.Create( - new SubscriptionCreationOptions() - { - // The subscription criteria: - Filter = x => x.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 100, - - // The object properties that will be sent for each matching document: - Projection = x => new - { - Id = x.Id, - Total = x.Lines.Sum(line => line.PricePerUnit * line.Quantity), - ShipTo = x.ShipTo, - - // 'Load' the related Employee document and use its data in the projection - EmployeeName = RavenQuery.Load(x.Employee).FirstName + " " + - RavenQuery.Load(x.Employee).LastName - } - }); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"declare function getOrderLinesSum(doc) { - var sum = 0; - for (var i in doc.Lines) { - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - } - return sum; - } - - declare function projectOrder(doc) { - var employee = load(doc.Employee); - return { - Id: doc.Id, - Total: getOrderLinesSum(doc), - ShipTo: doc.ShipTo, - EmployeeName: employee.FirstName + ' ' + employee.LastName - }; - } - - From Orders as o - Where getOrderLinesSum(o) > 100 - Select projectOrder(o)" -}); -`} - - - - - - -## Create subscription - include documents - -Here we create a subscription on the _Orders_ collection, which will send all the _Order_ documents. - -In addition, the related _Product_ documents associated with each Order are **included** in the batch sent to the client. -This way, when the subscription worker that processes the batch in the client accesses a _Product_ document, no additional call to the server will be made. - -See how to consume this type of subscription [here](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-uses-included-documents). - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Includes = builder => builder - // The documents whose IDs are specified in the 'Product' property - // will be included in the batch - .IncludeDocuments(x => x.Lines.Select(y => y.Product)) -}); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"from Orders include Lines[].Product" -}); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"declare function includeProducts(doc) { - let includedFields = 0; - let linesCount = doc.Lines.length; - - for (let i = 0; i < linesCount; i++) { - includedFields++; - include(doc.Lines[i].Product); - } - - return doc; - } - - from Orders as o select includeProducts(o)" -}); -`} - - - - - - -**Include using builder**: - -Include statements can be added to the subscription with `ISubscriptionIncludeBuilder`. -This builder is assigned to the `Includes` property in [SubscriptionCreationOptions<T>](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptionst). -It supports methods for including documents as well as [counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). -These methods can be chained. - -To include related documents, use method `IncludeDocuments`. -(See the _Builder-syntax_ tab in the example above). - - - - -**Include using RQL**: - -The include statements can be written in two ways: - -1. Use the `include` keyword at the end of the query, followed by the paths to the fields containing the IDs of the documents to include. - It is recommended to prefer this approach whenever possible, both for the clarity of the query and for slightly better performance. - (See the _RQL-path-syntax_ tab in the example above). - -2. Define the `include` within a JavaScript function that is called from the `select` clause. - (See the _RQL-javascript-syntax_ tab in the example above). - - - - - -If you include documents when making a [projection](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-and-project-fields), -the include will search for the specified paths in the projected fields rather than in the original document. - - - - -## Create subscription - include counters - -Here we create a subscription on the _Orders_ collection, which will send all the _Order_ documents. -In addition, values for the specified counters will be **included** in the batch. - -Note: -Modifying an existing counter's value after the document has been sent to the client does Not trigger re-sending. -However, adding a new counter to the document or removing an existing one will trigger re-sending the document. - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Includes = builder => builder - // Values for the specified counters will be included in the batch - .IncludeCounters(new[] { "Pros", "Cons" }) -}); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - Query = @"from Orders include counters('Pros'), counters('Cons')" -}); -`} - - - - -`ISubscriptionIncludeBuilder` has three methods for including counters: - - - -{`// Include a single counter -ISubscriptionIncludeBuilder IncludeCounter(string name); - -// Include multiple counters -ISubscriptionIncludeBuilder IncludeCounters(string[] names); - -// Include ALL counters from ALL documents that match the subscription criteria -ISubscriptionIncludeBuilder IncludeAllCounters(); -`} - - - -| Parameter | Type | Description | -|------------|------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| **name** | `string` | The name of a counter. The subscription will include all counters with this name that are contained in the documents the subscription retrieves. | -| **names** | `string[]` | Array of counter names. | - -**All include methods can be chained**: -For example, the following subscription includes multiple counters and documents: - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -\{ - Includes = builder => builder - .IncludeCounter("Likes") - .IncludeCounters(new[] \{ "Pros", "Cons" \}) - .IncludeDocuments("Employee") -\}); -`} - - - - - -## Create subscription - subscribe to revisions - -Here we create a simple revisions subscription on the _Orders_ collection that will send pairs of subsequent document revisions to the client. - - - - -{`subscriptionName = store.Subscriptions.Create( - // Use > as the type for the processed items - // e.g. > - new SubscriptionCreationOptions>()); -`} - - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() -{ - // Add (Revisions = true) to your subscription RQL - Query = @"From Orders (Revisions = true)" -}); -`} - - - - -Learn more about subscribing to document revisions in [subscriptions: revisions support](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx). - - - -## Create subscription - via update - -When attempting to update a subscription that does Not exist, -you can request a new subscription to be created by setting `CreateNew` to `true`. -In such a case, a new subscription will be created with the provided query. - - - -{`subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() -\{ - Name = "my subscription", - Query = "from Products where PricePerUnit > 20", - - // Set to true so that a new subscription will be created - // if a subscription with name "mySubscription" does Not exist - CreateNew = true -\}); -`} - - - - - -## Update existing subscription - -**Update subscription by name**: -The subscription definition can be updated after it has been created. -In this example we update the filtering **query** of an existing subscription named "my subscription". - - - -{`subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() -\{ - // Specify the subscription you wish to modify - Name = "my subscription", - - // Provide a new query - Query = "from Products where PricePerUnit > 50" -\}); -`} - - -**Update subscription by id**: -In addition to the subscription name, each subscription is assigned a subscription ID when it is created by the server. -This ID can be used instead of the name when updating the subscription. - - - -{`// Get the subscription's ID -SubscriptionState mySubscription = store.Subscriptions.GetSubscriptionState("my subscription"); -long subscriptionId = mySubscription.SubscriptionId; - -// Update the subscription -subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() -\{ - Id = subscriptionId, - Query = "from Products where PricePerUnit > 50" -\}); -`} - - - -Using the subscription ID allows you to modify the subscription name: - - - -{`// Get the subscription's ID -mySubscription = store.Subscriptions.GetSubscriptionState("my subscription"); -subscriptionId = mySubscription.SubscriptionId; - -// Update the subscription name -subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() -\{ - Id = subscriptionId, - Name = "New name" -\}); -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_examples-nodejs.mdx b/docs/client-api/data-subscriptions/creation/content/_examples-nodejs.mdx deleted file mode 100644 index ae8a4ac21d..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_examples-nodejs.mdx +++ /dev/null @@ -1,434 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* This page contains examples of **creating a subscription**. - To learn how to consume and process documents sent by the subscription, see these [examples](../../../../client-api/data-subscriptions/consumption/examples.mdx). - -* For a detailed syntax of the available subscription methods and objects, see this [API overview](../../../../client-api/data-subscriptions/creation/api-overview.mdx). - -* In this page: - * [Create subscription - for all documents in a collection](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---for-all-documents-in-a-collection) - * [Create subscription - filter documents](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-documents) - * [Create subscription - filter documents using regex](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-documents-using-regex) - * [Create subscription - filter and project fields](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-and-project-fields) - * [Create subscription - project data from a related document](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---project-data-from-a-related-document) - * [Create subscription - include documents](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-documents) - * [Create subscription - include counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters) - * [Create subscription - subscribe to revisions](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---subscribe-to-revisions) - * [Create subscription - via update](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---via-update) - * [Update existing subscription](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription) - - -## Create subscription - for all documents in a collection - -Here we create a plain subscription on the _Orders_ collection without any constraints or transformations. -The server will send ALL documents from the _Orders_ collection to a client that connects to this subscription. - - - -{`const subscriptionName = await documentStore.subscriptions.create(\{ - // Optionally, provide a custom name for the subscription - name: "OrdersProcessingSubscription", - - // You can provide the collection name in the RQL string in the 'query' param - query: "from Orders" -\}); -`} - - - - -{`const subscriptionName = await documentStore.subscriptions.create(\{ - name: "OrdersProcessingSubscription", - - // Or, you can provide the document type for the collection in the 'documentType' param - documentType: Order -\}); -`} - - - - -{`// Or, you can use the folllowing overload, -// pass the document class type to the 'create' method -const subscriptionName = await documentStore.subscriptions.create(Order); -`} - - - - - -## Create subscription - filter documents - -Here we create a subscription for documents from the _Orders_ collection where the total order revenue is greater than 100. -Only documents that match this condition will be sent from the server to a client connected to this subscription. - - - -{`// Define the filtering criteria -const query = \` - declare function getOrderLinesSum(doc) \{ - var sum = 0; - for (var i in doc.Lines) \{ - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - \} - return sum; - \} - - from Orders as o - where getOrderLinesSum(o) > 100\`; - -// Create the subscription with the defined query -const subscriptionName = await documentStore.subscriptions.create(\{ query \}); - -// In this case, the server will create a default name for the subscription -// since no specific name was provided when creating the subscription. -`} - - - - - -## Create subscription - filter documents using regex - -Here we create a subscription for documents from the _Orders_ collection where the `ShipTo.City` field matches a regular expression pattern. -Use the RQL `regex` function to perform server-side pattern matching. - - - -{`const query = \`from Orders where regex(ShipTo.City, '^New')\`; - -const subscriptionName = await documentStore.subscriptions.create(\{ query \}); -`} - - - - - -## Create subscription - filter and project fields - -Here, again, we create a subscription for documents from the _Orders_ collection where the total order revenue is greater than 100. -However, this time we only project the document ID and the Total Revenue properties in each object sent to the client. - - - -{`const query = \` - declare function getOrderLinesSum(doc) \{ - var sum = 0; - for (var i in doc.Lines) \{ - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - \} - return sum; - \} - - declare function projectOrder(doc) \{ - return \{ - Id: doc.Id, - Total: getOrderLinesSum(doc) - \} - \} - - from order as o - where getOrderLinesSum(o) > 100 - select projectOrder(o)\`; - -const subscriptionName = await documentStore.subscriptions.create(\{ query \}); -`} - - - - - -## Create subscription - project data from a related document - -In this subscription, in addition to projecting the document fields, -we also project data from a [related document](../../../../indexes/indexing-related-documents.mdx#what-are-related-documents) that is loaded using the `load` method. - - - -{`const query = \` - declare function getOrderLinesSum(doc) \{ - var sum = 0; - for (var i in doc.Lines) \{ - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - \} - return sum; - \} - - declare function projectOrder(doc) \{ - var employee = load(doc.Employee); - return \{ - Id: doc.Id, - Total: getOrderLinesSum(doc), - ShipTo: doc.ShipTo, - EmployeeName: employee.FirstName + ' ' + employee.LastName - \} - \} - - from order as o - where getOrderLinesSum(o) > 100 - select projectOrder(o)\`; - -const subscriptionName = await documentStore.subscriptions.create(\{ query \}); -`} - - - - - -## Create subscription - include documents - -Here we create a subscription on the _Orders_ collection, which will send all the _Order_ documents. - -In addition, the related _Product_ documents associated with each Order are **included** in the batch sent to the client. -This way, when the subscription worker that processes the batch in the client accesses a _Product_ document, no additional call to the server will be made. - -See how to consume this type of subscription [here](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-uses-included-documents). - - - - -{`const options = { - // The documents whose IDs are specified in the 'Product' property - // will be included in the batch - includes: builder => builder.includeDocuments("Lines[].Product"), - documentType: Order -}; - -const subscriptionName = await documentStore.subscriptions.create(options); -`} - - - - -{`const query = \`from Orders include Lines[].Product\`; -const subscriptionName = await documentStore.subscriptions.create({ query }); -`} - - - - -{`const query = \` - declare function includeProducts(doc) { - let includedFields = 0; - let linesCount = doc.Lines.length; - - for (let i = 0; i < linesCount; i++) { - includedFields++; - include(doc.Lines[i].Product); - } - - return doc; - } - - from Orders as o select includeProducts(o)\`; - -const subscriptionName = await documentStore.subscriptions.create({ query }); -`} - - - - - - -**Include using builder**: - -Include statements can be added to the subscription with a _builder_ object. -This builder is assigned to the `includes` property in the _options_ object. -It supports methods for including documents as well as [counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). -These methods can be chained. - -See this [API overview](../../../../client-api/data-subscriptions/creation/api-overview.mdx#include-methods) for all available include methods. - -To include related documents, use method `includeDocuments`. -(See the _Builder-syntax_ tab in the example above). - - - - -**Include using RQL**: - -The include statements can be written in two ways: - -1. Use the `include` keyword at the end of the query, followed by the paths to the fields containing the IDs of the documents to include. - It is recommended to prefer this approach whenever possible, both for the clarity of the query and for slightly better performance. - (See the _RQL-path-syntax_ tab in the example above). - -2. Define the `include` within a JavaScript function that is called from the `select` clause. - (See the _RQL-javascript-syntax_ tab in the example above). - - - - - -If you include documents when making a [projection](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-and-project-fields), -the include will search for the specified paths in the projected fields rather than in the original document. - - - - -## Create subscription - include counters - -Here we create a subscription on the _Orders_ collection, which will send all the _Order_ documents. -In addition, values for the specified counters will be **included** in the batch. - -Note: -Modifying an existing counter's value after the document has been sent to the client does Not trigger re-sending. -However, adding a new counter to the document or removing an existing one will trigger re-sending the document. - - - - -{`const options = { - includes: builder => builder - // Values for the specified counters will be included in the batch - .includeCounters(["Pros", "Cons"]), - documentType: Order -}; - -const subscriptionName = await documentStore.subscriptions.create(options); -`} - - - - -{`const options = { - query: "from Orders include counters('Pros'), counters('Cons')" -}; - -const subscriptionName = await documentStore.subscriptions.create(options); -`} - - - - -**All include methods can be chained**: -For example, the following subscription includes multiple counters and documents: - - - -{`const options = \{ - includes: builder => builder - .includeCounter("Likes") - .includeCounters(["Pros", "Cons"]) - .includeDocuments("Employee"), - documentType: Order -\}; - -const subscriptionName = await documentStore.subscriptions.create(options); -`} - - - - - -## Create subscription - subscribe to revisions - -Here we create a simple revisions subscription on the _Orders_ collection that will send pairs of subsequent document revisions to the client. - - - - -{`const subscriptionName = await documentStore.subscriptions.createForRevisions({ - documentType: Order -}); -`} - - - - -{`const subscriptionName = await documentStore.subscriptions.createForRevisions({ - query: "from Orders (Revisions = true)" -}); -`} - - - - -Learn more about subscribing to document revisions in [subscriptions: revisions support](../../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx). - - - -## Create subscription - via update - -When attempting to update a subscription that does Not exist, -you can request a new subscription to be created by setting `createNew` to `true`. -In such a case, a new subscription will be created with the provided query. - - - -{`const subscriptionName = await documentStore.subscriptions.update(\{ - name: "my subscription", - query: "from Products where PricePerUnit > 20", - - // Set to true so that a new subscription will be created - // if a subscription with name "my subscription" does Not exist - createNew: true -\}); -`} - - - - - -## Update existing subscription - -**Update subscription by name**: -The subscription definition can be updated after it has been created. -In this example we update the filtering **query** of an existing subscription named "my subscription". - - - -{`const subscriptionName = await documentStore.subscriptions.update(\{ - // Specify the subscription you wish to modify - name: "my subscription", - - // Provide a new query - query: "from Products where PricePerUnit > 50" -\}); -`} - - -**Update subscription by id**: -In addition to the subscription name, each subscription is assigned a subscription ID when it is created by the server. -This ID can be used instead of the name when updating the subscription. - - - -{`// Get the subscription's ID -const mySubscription = await documentStore.subscriptions.getSubscriptionState("my subscription"); -const subscriptionId = mySubscription.subscriptionId; - -// Update the subscription -const subscriptionName = await documentStore.subscriptions.update(\{ - id: subscriptionId, - query: "from Products where PricePerUnit > 50" -\}); -`} - - - -Using the subscription ID allows you to modify the subscription name: - - - -{`// Get the subscription's ID -const mySubscription = await documentStore.subscriptions.getSubscriptionState("my subscription"); -const subscriptionId = mySubscription.subscriptionId; - -// Update the subscription's name -const subscriptionName = await documentStore.subscriptions.update(\{ - id: subscriptionId, - name: "new name" -\}); -`} - - - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_examples-python.mdx b/docs/client-api/data-subscriptions/creation/content/_examples-python.mdx deleted file mode 100644 index 637bddc08d..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_examples-python.mdx +++ /dev/null @@ -1,341 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* This page contains examples of **creating a subscription**. - To learn how to consume and process documents sent by the subscription, see these [examples](../../../../client-api/data-subscriptions/consumption/examples.mdx). - -* For a detailed syntax of the available subscription methods and objects, see this [API overview](../../../../client-api/data-subscriptions/creation/api-overview.mdx). - -* In this page: - * [Create subscription - for all documents in a collection](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---for-all-documents-in-a-collection) - * [Create subscription - filter documents](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-documents) - * [Create subscription - filter documents using regex](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-documents-using-regex) - * [Create subscription - filter and project fields](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-and-project-fields) - * [Create subscription - project data from a related document](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---project-data-from-a-related-document) - * [Create subscription - include documents](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-documents) - * [Create subscription - include counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters) - * [Update existing subscription](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription) - - -## Create subscription - for all documents in a collection - -Here we create a plain subscription on the _Orders_ collection without any constraints or transformations. -The server will send ALL documents from the _Orders_ collection to a client that connects to this subscription. - - - - -{`name = store.subscriptions.create_for_class( - Order, SubscriptionCreationOptions(name="OrdersProcessingSubscription") -) -`} - - - - -{`name = store.subscriptions.create_for_options(SubscriptionCreationOptions(query="From Orders")) -`} - - - - - - -## Create subscription - filter documents - -Here we create a subscription for documents from the _Orders_ collection where the total order revenue is greater than 100. -Only documents that match this condition will be sent from the server to a client connected to this subscription. - - - -{`name = store.subscriptions.create_for_options( - SubscriptionCreationOptions( - query=( - "declare function getOrderLinesSum(doc) \{" - " var sum = 0;" - " for (var i in doc.Lines) \{" - " sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity;" - " \}" - " return sum;" - "\}" - "From Orders as o " - "Where getOrderLinesSum(o) > 100 " - ) - ), -) -`} - - - - - -## Create subscription - filter documents using regex - -Here we create a subscription for documents from the _Orders_ collection where the `ShipTo.City` field matches a regular expression pattern. -Use the RQL `regex` function to perform server-side pattern matching. - - - -{`name = store.subscriptions.create_for_options( - SubscriptionCreationOptions( - query="From Orders Where regex(ShipTo.City, '^New')" - ), -) -`} - - - - - -## Create subscription - filter and project fields - -Here, again, we create a subscription for documents from the _Orders_ collection where the total order revenue is greater than 100. -However, this time we only project the document ID and the Total Revenue properties in each object sent to the client. - - - -{`name = store.subscriptions.create_for_options( - SubscriptionCreationOptions( - query=""" - declare function getOrderLinesSum(doc) \{ - var sum = 0; - for (var i in doc.Lines) \{ - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - \} - return sum; - \} - - declare function projectOrder(doc) \{ - return \{ - Id: doc.Id, - Total: getOrderLinesSum(doc) - \}; - \} - - From Orders as o - Where getOrderLinesSum(o) > 100 - Select projectOrder(o) - """ - ) -) -`} - - - - - -## Create subscription - project data from a related document - -In this subscription, in addition to projecting the document fields, -we also project data from a [related document](../../../../indexes/indexing-related-documents.mdx#what-are-related-documents) that is loaded using the `load` method. - - - -{`name = store.subscriptions.create_for_options( - SubscriptionCreationOptions( - query=""" - declare function getOrderLinesSum(doc) \{ - var sum = 0; - for (var i in doc.Lines) \{ - sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; - \} - return sum; - \} - - declare function projectOrder(doc) \{ - var employee = load(doc.Employee); - return \{ - Id: doc.Id, - Total: getOrderLinesSum(doc), - ShipTo: doc.ShipTo, - EmployeeName: employee.FirstName + ' ' + employee.LastName - \}; - \} - - From Orders as o - Where getOrderLinesSum(o) > 100 - Select projectOrder(o) - """ - ) -) -`} - - - - - -## Create subscription - include documents - -Here we create a subscription on the _Orders_ collection, which will send all the _Order_ documents. - -In addition, the related _Product_ documents associated with each Order are **included** in the batch sent to the client. -This way, when the subscription worker that processes the batch in the client accesses a _Product_ document, no additional call to the server will be made. - -See how to consume this type of subscription [here](../../../../client-api/data-subscriptions/consumption/examples.mdx#subscription-that-uses-included-documents). - - - - -{`store.subscriptions.create_for_class( - Order, - SubscriptionCreationOptions(includes=lambda builder: builder.include_documents("Lines[].Product")), -) -`} - - - - -{`store.subscriptions.create_for_options( - SubscriptionCreationOptions(query="from Orders include Lines[].Product") -) -`} - - - - -{`store.subscriptions.create_for_options( - SubscriptionCreationOptions( - query=""" - declare function includeProducts(doc) { - let includedFields = 0; - let linesCount = doc.Lines.length; - - for (let i = 0; i < linesCount; i++) { - includedFields++; - include(doc.Lines[i].Product); - } - - return doc; - } - - from Orders as o select includeProducts(o) - """ - ) -) -`} - - - - - - -**Include using builder**: - -Include statements can be added to the subscription with `SubscriptionIncludeBuilder`. -This builder is assigned to the `includes` property in [SubscriptionCreationOptions](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptionst). -It supports methods for including documents as well as [counters](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). -These methods can be chained. - -To include related documents, use method `include_documents`. -(See the _Builder-syntax_ tab in the example above). - - - - -**Include using RQL**: - -The include statements can be written in two ways: - -1. Use the `include` keyword at the end of the query, followed by the paths to the fields containing the IDs of the documents to include. - It is recommended to prefer this approach whenever possible, both for the clarity of the query and for slightly better performance. - (See the _RQL-path-syntax_ tab in the example above). - -2. Define the `include` within a JavaScript function that is called from the `select` clause. - (See the _RQL-javascript-syntax_ tab in the example above). - - - - - -If you include documents when making a [projection](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---filter-and-project-fields), -the include will search for the specified paths in the projected fields rather than in the original document. - - - - - -## Create subscription - include counters - -`SubscriptionIncludeBuilder` has three methods for including counters: - - - -{`def include_counter(self, name: str) -> SubscriptionIncludeBuilder: ... - -def include_counters(self, *names: str) -> SubscriptionIncludeBuilder: ... - -def include_all_counters(self) -> SubscriptionIncludeBuilder: ... -`} - - - -`include_counter` is used to specify a single counter. -`include_counters` is used to specify multiple counters. -`include_all_counters` retrieves all counters from all subscribed documents. - -| Parameter | Type | Description | -|-------------|-------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| **name** | `str` | The name of a counter. The subscription will include all counters with this name that are contained in the documents the subscription retrieves. | -| **\*names** | `str` | Array of counter names. | - -The following subscription, which includes multiple counters in the batch sent to the client, -demonstrates how the methods can be chained. - - - -{`store.subscriptions.create_for_class( - Order, - SubscriptionCreationOptions( - includes=lambda builder: builder - .include_counter("Likes") - .include_counters("Pros", "Cons") - ), -) -`} - - - - - -## Update existing subscription - -The subscription definition can be updated after it has been created. -In this example we update the filtering query of an existing subscription named "my subscription". - - - -{`store.subscriptions.update(SubscriptionUpdateOptions( - name="My subscription", query="from Products where PricePerUnit > 50")) -`} - - - - -**Modifying the subscription's name**: - -In addition to the subscription name, each subscription is assigned a **subscription ID** when it is created by the server. -This ID can be used to identify the subscription, instead of the name, when updating the subscription. - -This allows users to change an existing subscription's **name** by specifying the subscription's ID -and submitting a new string in the `name` field of `SubscriptionUpdateOptions`. - - - -{`my_subscription = store.subscriptions.get_subscription_state("my subscription") - -subscription_id = my_subscription.subscription_id - -store.subscriptions.update(SubscriptionUpdateOptions(key=subscription_id, name="new name")) -`} - - - - - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-csharp.mdx b/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-csharp.mdx deleted file mode 100644 index 1ea774503c..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-csharp.mdx +++ /dev/null @@ -1,105 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* A subscription task can be created in two ways: - * **From the client API**: - The client can create a subscription task on the server using this [creation API](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation). - * **From the Studio**: - See [creating subscription task](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx) to learn how to create a subscription task on the server via the Studio. - -* Once created, its definition and progress will be stored on the cluster, and not in a single server. - -* Upon subscription creation, the cluster will choose a preferred node that will run the subscription - (unless the client has stated a responsible node). - -* From that point and on, clients that will connect to a server in order to consume the subscription will be redirected to the node mentioned above. - -* In this page: - * [Subscription creation](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-creation) - * [Subscription name](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-name) - * [Responsible node](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#responsible-node) - - -## Subscription creation - -Data subscription is a batch processing mechanism that sends documents that meet specific criteria to connected clients. - -In order to create a data subscription, we first need to define the criteria. -The basic requirement is to specify the collection from which the subscription will retrieve documents. -However, the criteria can be a complex RQL-like expression defining JavaScript functions that filter documents and project their content. - -* The following is a simple subscription definition: - - - -{`// With the following subscription definition, the server will send ALL documents -// from the 'Orders' collection to a client that connects to this subscription. -subscriptionName = store.Subscriptions.Create(); -`} - - - -* For more complex subscription creation scenarios, see the these [examples](../../../../client-api/data-subscriptions/creation/examples.mdx). - -* A subscription also can be modified after it has been created, see [update existing subscription](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription). - - - - -## Subscription name - -In order to consume a data subscription, a subscription name is required to identify it. -If you don't specify a name when creating the subscription, the server will automatically generate a default name. -However, you have the option to provide a custom name for the subscription. - -A dedicated name can be useful for use cases like dedicated, long-running batch processing mechanisms, -where it'll be more comfortable to use a human-readable name in the code and even use the same name between different environments -(as long as subscription creation is taken care of upfront). - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions -\{ - // Set a custom name for the subscription - Name = "OrdersProcessingSubscription" -\}); -`} - - - - -Note that the subscription name is unique and it will not be possible to create two subscriptions with the same name in the same database. - - - - -## Responsible node - -As stated above, upon creation, the cluster will choose a node that will be responsible for managing the subscription task on the server-side. -Once chosen, that node will be the only node to manage the subscription. - -There is an enterprise license level feature that supports subscription (and any other ongoing task) failover between nodes, -but eventually, as long as the originally assigned node is online, it will be the one to manage the data subscription task. - -Nevertheless, there is an option to manually decide which node will be responsible for managing the subscription task. -Provide the tag of the node you wish to be responsible in the `MentorNode` property as follows: - - - -{`subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions -\{ - MentorNode = "D" -\}); -`} - - - -Manually setting the node can help choose a more suitable server based on factors such as resources, client proximity, or other considerations. - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-java.mdx b/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-java.mdx deleted file mode 100644 index e7e5952443..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-java.mdx +++ /dev/null @@ -1,99 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* A subscription task can be created in two ways: - * **From the client API**: - The client can create a subscription task on the server using this [creation API](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation). - * **From the Studio**: - See [creating subscription task](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx) to learn how to create a subscription task on the server via the Studio. - -* Once created, it's definition and progress will be stored on the cluster, and not in a single server. - -* Upon subscription creation, the cluster will choose a preferred node that will run the subscription - (unless client has stated a mentor node). - -* From that point and on, clients that will connect to a server in order to consume the subscription will be redirected to the node mentioned above. - -* In this page: - * [Subscription creation](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-creation) - * [Subscription name](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-name) - * [Responsible node](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#responsible-node) - - -## Subscription creation - -Data subscription is a batch processing mechanism that sends documents that meet specific criteria to connected clients. - -In order to create a data subscription, we first need to define the criteria. -The basic requirement is to specify the collection from which the subscription will retrieve documents. -However, the criteria can be a complex RQL-like expression defining JavaScript functions that filter documents and project their content. - -* The following is a simple subscription definition: - - - -{`// With the following subscription definition, the server will send ALL documents -// from the 'Orders' collection to a client that connects to this subscription. -name = store.subscriptions().create(Order.class); -`} - - - -* For more complex subscription definitions, see these [examples](../../../../client-api/data-subscriptions/creation/examples.mdx). - - - -## Subscription name - -In order to consume a data subscription, a subscription name is required to identify it. -If you don't specify a name when creating the subscription, the server will automatically generate a default name. -However, you have the option to provide a custom name for the subscription. - -A dedicated name can be useful for use cases like dedicated, long-running batch processing mechanisms, -where it'll be more comfortable to use a human-readable name in the code and even use the same name between different environments -(as long as subscription creation is taken care of upfront). - - - -{`SubscriptionCreationOptions options = new SubscriptionCreationOptions(); -options.setName("OrdersProcessingSubscription"); -name = store.subscriptions().create(Order.class, options); -`} - - - - -Note that subscription name is unique and it will not be possible to create two subscriptions with the same name in the same database. - - - - -## Responsible node - -As stated above, upon creation, the cluster will choose a node that will be responsible for managing the subscription task on the server-side. -Once chosen, that node will be the only node to manage the subscription. - -There is an enterprise license level feature that supports subscription (and any other ongoing task) failover between nodes, -but eventually, as long as the originally assigned node is online, it will be the one to manage the data subscription task. - -Nevertheless, there is an option to manually decide which node will be responsible for managing the subscription task. -Provide the tag of the node you wish to be responsible in the `MentorNode` property as follows: - - - -{`SubscriptionCreationOptions options = new SubscriptionCreationOptions(); -options.setMentorNode("D"); -name = store.subscriptions().create(Order.class, options); -`} - - - -Manually setting the node can help choose a more suitable server based on factors such as resources, client proximity, or other considerations. - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-nodejs.mdx b/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-nodejs.mdx deleted file mode 100644 index 5aed7ed6c4..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-nodejs.mdx +++ /dev/null @@ -1,107 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* A subscription task can be created in two ways: - * **From the client API**: - The client can create a subscription task on the server using this [creation API](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation). - * **From the Studio**: - See [creating subscription task](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx) to learn how to create a subscription task on the server via the Studio. - -* Once created, it's definition and progress will be stored on the cluster, and not in a single server. - -* Upon subscription creation, the cluster will choose a preferred node that will run the subscription - (unless client has stated a mentor node). - -* From that point and on, clients that will connect to a server in order to consume the subscription will be redirected to the node mentioned above. - -* In this page: - * [Subscription creation](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-creation) - * [Subscription name](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-name) - * [Responsible node](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#responsible-node) - - -## Subscription creation - -Data subscription is a batch processing mechanism that sends documents that meet specific criteria to connected clients. - -In order to create a data subscription, we first need to define the criteria. -The basic requirement is to specify the collection from which the subscription will retrieve documents. -However, the criteria can be a complex RQL-like expression defining JavaScript functions that filter documents and project their content. - -* The following is a simple subscription definition: - - - -{`// With the following subscription definition, the server will send ALL documents -// from the 'Orders' collection to a client that connects to this subscription. -const subscriptionName = await documentStore.subscriptions.create(\{ - query: "from Orders" -\}); -`} - - - -* For more complex subscription creation scenarios, see the these [examples](../../../../client-api/data-subscriptions/creation/examples.mdx). - -* A subscription also can be modified after it has been created, see [update existing subscription](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription). - - - -## Subscription name - -In order to consume a data subscription, a subscription name is required to identify it. -If you don't specify a name when creating the subscription, the server will automatically generate a default name. -However, you have the option to provide a custom name for the subscription. - -A dedicated name can be useful for use cases like dedicated, long-running batch processing mechanisms, -where it'll be more comfortable to use a human-readable name in the code and even use the same name between different environments -(as long as subscription creation is taken care of upfront). - - - -{`const name = await store.subscriptions.create(\{ - query: "from Orders", - // Set a custom name for the subscription - name: "OrdersProcessingSubscription" -\}); -`} - - - - -Note that subscription name is unique and it will not be possible to create two subscriptions with the same name in the same database. - - - - -## Responsible node - -As stated above, upon creation, the cluster will choose a node that will be responsible for managing the subscription task on the server-side. -Once chosen, that node will be the only node to manage the subscription. - -There is an enterprise license level feature that supports subscription (and any other ongoing task) failover between nodes, -but eventually, as long as the originally assigned node is online, it will be the one to manage the data subscription task. - -Nevertheless, there is an option to manually decide which node will be responsible for managing the subscription task. -Provide the tag of the node you wish to be responsible in the `mentorNode` property as follows: - - - -{`const name = await store.subscriptions.create(\{ - query: "from Orders", - // Set a responsible node for the subscritpion task - mentorNode: "D" -\}); -`} - - - -Manually setting the node can help choose a more suitable server based on factors such as resources, client proximity, or other considerations. - - - - diff --git a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-python.mdx b/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-python.mdx deleted file mode 100644 index 7136152ec7..0000000000 --- a/docs/client-api/data-subscriptions/creation/content/_how-to-create-data-subscription-python.mdx +++ /dev/null @@ -1,99 +0,0 @@ -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - - -* A subscription task can be created in two ways: - * **From the client API**: - The client can create a subscription task on the server using this [creation API](../../../../client-api/data-subscriptions/creation/api-overview.mdx#subscription-creation). - * **From the Studio**: - See [creating subscription task](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx) to learn how to create a subscription task on the server via the Studio. - -* Once created, its definition and progress will be stored on the cluster, and not in a single server. - -* Upon subscription creation, the cluster will choose a preferred node that will run the subscription - (unless the client has stated a responsible node). - -* From that point and on, clients that will connect to a server in order to consume the subscription will be redirected to the node mentioned above. - -* In this page: - * [Subscription creation](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-creation) - * [Subscription name](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#subscription-name) - * [Responsible node](../../../../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx#responsible-node) - - -## Subscription creation - -Data subscription is a batch processing mechanism that sends documents that meet specific criteria to connected clients. - -In order to create a data subscription, we first need to define the criteria. -The basic requirement is to specify the collection from which the subscription will retrieve documents. -However, the criteria can be a complex RQL-like expression defining JavaScript functions that filter documents and project their content. - -* The following is a simple subscription definition: - - - -{`# With the following subscription definition, the server will send ALL documents -# from the 'Orders' collection to a client that connects to this subscription. -name = store.subscriptions.create_for_class(Order) -`} - - - -* For more complex subscription definitions, see these [examples](../../../../client-api/data-subscriptions/creation/examples.mdx). - -* A subscription also can be modified after it has been created, see [update existing subscription](../../../../client-api/data-subscriptions/creation/examples.mdx#update-existing-subscription). - - - -## Subscription name - -In order to consume a data subscription, a subscription name is required to identify it. -If you don't specify a name when creating the subscription, the server will automatically generate a default name. -However, you have the option to provide a custom name for the subscription. - -A dedicated name can be useful for use cases like dedicated, long-running batch processing mechanisms, -where it'll be more comfortable to use a human-readable name in the code and even use the same name between different environments -(as long as subscription creation is taken care of upfront). - - - -{`name = store.subscriptions.create_for_class( - Order, SubscriptionCreationOptions(name="OrdersProcessingSubscription") -) -`} - - - - -Note that the subscription name is unique and it will not be possible to create two subscriptions with the same name in the same database. - - - - -## Responsible node - -As stated above, upon creation, the cluster will choose a node that will be responsible for managing the subscription task on the server-side. -Once chosen, that node will be the only node to manage the subscription. - -There is an enterprise license level feature that supports subscription (and any other ongoing task) failover between nodes, -but eventually, as long as the originally assigned node is online, it will be the one to manage the data subscription task. - -Nevertheless, there is an option to manually decide which node will be responsible for managing the subscription task. -Provide the tag of the node you wish to be responsible in the `mentor_node` property as follows: - - - -{`name = store.subscriptions.create_for_class(Order, SubscriptionCreationOptions(mentor_node="D")) -`} - - - -Manually setting the node can help choose a more suitable server based on factors such as resources, client proximity, or other considerations. - - - - diff --git a/docs/client-api/data-subscriptions/creation/examples.mdx b/docs/client-api/data-subscriptions/creation/examples.mdx deleted file mode 100644 index 4982da9c18..0000000000 --- a/docs/client-api/data-subscriptions/creation/examples.mdx +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: "Data Subscription Creation Examples" -sidebar_label: Examples -description: "Examples of creating RavenDB data subscriptions with filtering, projections, revisions support, and custom configuration options." -sidebar_position: 1 -supported_languages: ["csharp", "python", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" - - title: "JavaScript Engine" - link: "server/kb/javascript-engine" - source: "docs" - path: "Server > Knowledge Base" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import ExamplesCsharp from './content/_examples-csharp.mdx'; -import ExamplesPython from './content/_examples-python.mdx'; -import ExamplesNodejs from './content/_examples-nodejs.mdx'; - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx b/docs/client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx deleted file mode 100644 index 46be35bc9b..0000000000 --- a/docs/client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "How to Create a Data Subscription" -sidebar_label: How to Create a Data Subscription -description: "Create RavenDB data subscriptions to receive continuous document feeds filtered by collection, query, or JavaScript projection." -sidebar_position: 0 -supported_languages: ["csharp", "java", "python", "nodejs"] -see_also: - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" - source: "docs" - path: "Client API > Data Subscriptions" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" - - title: "JavaScript Engine" - link: "server/kb/javascript-engine" - source: "docs" - path: "Server > Knowledge Base" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import HowToCreateDataSubscriptionCsharp from './content/_how-to-create-data-subscription-csharp.mdx'; -import HowToCreateDataSubscriptionJava from './content/_how-to-create-data-subscription-java.mdx'; -import HowToCreateDataSubscriptionPython from './content/_how-to-create-data-subscription-python.mdx'; -import HowToCreateDataSubscriptionNodejs from './content/_how-to-create-data-subscription-nodejs.mdx'; - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/data-subscriptions/what-are-data-subscriptions.mdx b/docs/client-api/data-subscriptions/what-are-data-subscriptions.mdx deleted file mode 100644 index d5d6c62627..0000000000 --- a/docs/client-api/data-subscriptions/what-are-data-subscriptions.mdx +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: "Data Subscriptions" -sidebar_label: What are Data Subscriptions -description: "RavenDB data subscriptions push matching documents to client workers in reliable batches with automatic failover and change tracking." -sidebar_position: 0 -supported_languages: ["csharp", "java", "nodejs"] -see_also: - - title: "How to Create a Data Subscription" - link: "client-api/data-subscriptions/creation/how-to-create-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Creation" - - title: "How to Consume a Data Subscription" - link: "client-api/data-subscriptions/consumption/how-to-consume-data-subscription" - source: "docs" - path: "Client API > Data Subscriptions > Consumption" - - title: "Maintenance Operations" - link: "client-api/data-subscriptions/advanced-topics/maintenance-operations" - source: "docs" - path: "Client API > Data Subscriptions > Advanced topics" - - title: "Sharding Overview" - link: "sharding/overview" - source: "docs" - path: "Sharding" - - title: "Sharding: ETL" - link: "sharding/etl" - source: "docs" - path: "Sharding" ---- - -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -import WhatAreDataSubscriptionsCsharp from './content/_what-are-data-subscriptions-csharp.mdx'; -import WhatAreDataSubscriptionsJava from './content/_what-are-data-subscriptions-java.mdx'; -import WhatAreDataSubscriptionsNodejs from './content/_what-are-data-subscriptions-nodejs.mdx'; - - - - - - - - - - - - - - - - - - diff --git a/docs/client-api/session/querying/content/_how-to-project-query-results-csharp.mdx b/docs/client-api/session/querying/content/_how-to-project-query-results-csharp.mdx index 84feb767d1..b423f939c1 100644 --- a/docs/client-api/session/querying/content/_how-to-project-query-results-csharp.mdx +++ b/docs/client-api/session/querying/content/_how-to-project-query-results-csharp.mdx @@ -37,7 +37,7 @@ import CodeBlock from '@theme/CodeBlock'; * Projections allow you to tailor the query results specifically to your needs. Getting specific details to display can be useful when presenting data to users or populating user interfaces. - Projection queries are also useful with [subscriptions](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + Projection queries are also useful with [subscriptions](../../../../data-subscriptions/overview.mdx) since all transformation work is done on the server side without having to send a lot of data over the wire. * Returning partial document data from the server reduces network traffic, diff --git a/docs/client-api/session/querying/content/_how-to-project-query-results-nodejs.mdx b/docs/client-api/session/querying/content/_how-to-project-query-results-nodejs.mdx index 920e6ee484..710312ae19 100644 --- a/docs/client-api/session/querying/content/_how-to-project-query-results-nodejs.mdx +++ b/docs/client-api/session/querying/content/_how-to-project-query-results-nodejs.mdx @@ -36,7 +36,7 @@ import CodeBlock from '@theme/CodeBlock'; * Projections allow you to tailor the query results specifically to your needs. Getting specific details to display can be useful when presenting data to users or populating user interfaces. - Projection queries are also useful with [subscriptions](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + Projection queries are also useful with [subscriptions](../../../../data-subscriptions/overview.mdx) since all transformation work is done on the server side without having to send a lot of data over the wire. * Returning partial document data from the server reduces network traffic, diff --git a/docs/client-api/session/querying/content/_how-to-project-query-results-php.mdx b/docs/client-api/session/querying/content/_how-to-project-query-results-php.mdx index 433242d4a9..2ca35aa1df 100644 --- a/docs/client-api/session/querying/content/_how-to-project-query-results-php.mdx +++ b/docs/client-api/session/querying/content/_how-to-project-query-results-php.mdx @@ -43,7 +43,7 @@ import CodeBlock from '@theme/CodeBlock'; * Projections allow you to tailor the query results specifically to your needs. Getting specific details to display can be useful when presenting data to users or populating user interfaces. - Projection queries are also useful with [subscriptions](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + Projection queries are also useful with [subscriptions](../../../../data-subscriptions/overview.mdx) since all transformation work is done on the server side without having to send a lot of data over the wire. * Returning partial document data from the server reduces network traffic, diff --git a/docs/client-api/session/querying/content/_how-to-project-query-results-python.mdx b/docs/client-api/session/querying/content/_how-to-project-query-results-python.mdx index 6cf008121f..1b5804071e 100644 --- a/docs/client-api/session/querying/content/_how-to-project-query-results-python.mdx +++ b/docs/client-api/session/querying/content/_how-to-project-query-results-python.mdx @@ -44,7 +44,7 @@ import CodeBlock from '@theme/CodeBlock'; * Projections allow you to tailor the query results specifically to your needs. Getting specific details to display can be useful when presenting data to users or populating user interfaces. - Projection queries are also useful with [subscriptions](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + Projection queries are also useful with [subscriptions](../../../../data-subscriptions/overview.mdx) since all transformation work is done on the server side without having to send a lot of data over the wire. * Returning partial document data from the server reduces network traffic, diff --git a/docs/data-archival/configuration.mdx b/docs/data-archival/configuration.mdx index fc2f1820ff..e5927a897c 100644 --- a/docs/data-archival/configuration.mdx +++ b/docs/data-archival/configuration.mdx @@ -9,7 +9,7 @@ see_also: source: "docs" path: "Server > Configuration" - title: "Subscription Configuration" - link: "server/configuration/subscription-configuration" + link: "data-subscriptions/configuration" source: "docs" path: "Server > Configuration" --- diff --git a/docs/data-subscriptions/assets/snagit/subscriptions_ongoing-tasks_task-bar_collapsed.snagx b/docs/data-subscriptions/assets/snagit/subscriptions_ongoing-tasks_task-bar_collapsed.snagx new file mode 100644 index 0000000000..756f2ef1a9 Binary files /dev/null and b/docs/data-subscriptions/assets/snagit/subscriptions_ongoing-tasks_task-bar_collapsed.snagx differ diff --git a/docs/data-subscriptions/assets/snagit/subscriptions_ongoing-tasks_task-bar_expanded.snagx b/docs/data-subscriptions/assets/snagit/subscriptions_ongoing-tasks_task-bar_expanded.snagx new file mode 100644 index 0000000000..6707b8b8cc Binary files /dev/null and b/docs/data-subscriptions/assets/snagit/subscriptions_ongoing-tasks_task-bar_expanded.snagx differ diff --git a/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_collapsed.snagx b/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_collapsed.snagx new file mode 100644 index 0000000000..1a62669146 Binary files /dev/null and b/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_collapsed.snagx differ diff --git a/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_expanded.snagx b/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_expanded.snagx new file mode 100644 index 0000000000..08717f2fdc Binary files /dev/null and b/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_expanded.snagx differ diff --git a/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_indicators-pending.snagx b/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_indicators-pending.snagx new file mode 100644 index 0000000000..bf9fe10cae Binary files /dev/null and b/docs/data-subscriptions/assets/snagit/subscriptions_task-stats_indicators-pending.snagx differ diff --git a/docs/data-subscriptions/assets/subscriptions_document-processing.png b/docs/data-subscriptions/assets/subscriptions_document-processing.png new file mode 100644 index 0000000000..d501b462e3 Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_document-processing.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_ongoing-tasks_task-bar_collapsed.png b/docs/data-subscriptions/assets/subscriptions_ongoing-tasks_task-bar_collapsed.png new file mode 100644 index 0000000000..d6abf4eb05 Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_ongoing-tasks_task-bar_collapsed.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_ongoing-tasks_task-bar_expanded.png b/docs/data-subscriptions/assets/subscriptions_ongoing-tasks_task-bar_expanded.png new file mode 100644 index 0000000000..1476374e13 Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_ongoing-tasks_task-bar_expanded.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_task-stats_collapsed.png b/docs/data-subscriptions/assets/subscriptions_task-stats_collapsed.png new file mode 100644 index 0000000000..19ea39c47b Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_task-stats_collapsed.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_task-stats_expanded.png b/docs/data-subscriptions/assets/subscriptions_task-stats_expanded.png new file mode 100644 index 0000000000..3a267fd497 Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_task-stats_expanded.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-aborted.png b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-aborted.png new file mode 100644 index 0000000000..13fa2ce950 Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-aborted.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-initiated.png b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-initiated.png new file mode 100644 index 0000000000..7dc0b97ae8 Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-initiated.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-pending.png b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-pending.png new file mode 100644 index 0000000000..66bb6fe142 Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-pending.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-rejected.png b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-rejected.png new file mode 100644 index 0000000000..2bef45040b Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_task-stats_indicator-rejected.png differ diff --git a/docs/data-subscriptions/assets/subscriptions_task-stats_indicators-pending.png b/docs/data-subscriptions/assets/subscriptions_task-stats_indicators-pending.png new file mode 100644 index 0000000000..fe1e6cf0fe Binary files /dev/null and b/docs/data-subscriptions/assets/subscriptions_task-stats_indicators-pending.png differ diff --git a/docs/data-subscriptions/assets/svg/subscriptions_document-processing.svg b/docs/data-subscriptions/assets/svg/subscriptions_document-processing.svg new file mode 100644 index 0000000000..481aa25b20 --- /dev/null +++ b/docs/data-subscriptions/assets/svg/subscriptions_document-processing.svg @@ -0,0 +1,143 @@ + + + + + + + + + + + + + + + + + + + + + Subscription worker (client) + + Subscription connection (server) + + Cluster + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Connection request + Connection accepted + Documents sent + Acknowledgment + Confirmation + + + + + + + Worker starts with your + batch-processing logic + + + Waits for documents + + + Documents processed + by your logic + + + Acknowledges the batch + + + Acknowledgment confirmed + + + + Subscription connection + request received + + + Checks the subscription exists + and the worker may connect, + per its connection strategy + + + Subscription connection + approved + + + Queries the database for + the next matching batch + + + Sends documents to the worker + + + Sends the acknowledgment + to the cluster + + + Ready to send the next batch + + + + Subscription progress + found + + + Subscription progress + stored + + + + + + + Database + (responsible node) + + + diff --git a/docs/data-subscriptions/concurrent-subscriptions.mdx b/docs/data-subscriptions/concurrent-subscriptions.mdx new file mode 100644 index 0000000000..de0365b706 --- /dev/null +++ b/docs/data-subscriptions/concurrent-subscriptions.mdx @@ -0,0 +1,24 @@ +--- +title: "Concurrent subscriptions" +sidebar_label: "Concurrent subscriptions" +description: "Run several workers on one RavenDB data subscription so they consume its documents in parallel, and manage their connections." +sidebar_position: 4 +--- + +import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; +import LanguageContent from "@site/src/components/LanguageContent"; + +import ConcurrentSubscriptionsCsharp from './content/_concurrent-subscriptions-csharp.mdx'; +import ConcurrentSubscriptionsNodejs from './content/_concurrent-subscriptions-nodejs.mdx'; + +export const supportedLanguages = ["csharp", "nodejs"]; + + + + + + + + + + diff --git a/docs/data-subscriptions/configuration.mdx b/docs/data-subscriptions/configuration.mdx new file mode 100644 index 0000000000..1f13b1c7dd --- /dev/null +++ b/docs/data-subscriptions/configuration.mdx @@ -0,0 +1,57 @@ +--- +title: "Data subscriptions configuration options" +sidebar_label: "Configuration" +description: "Configure data subscription behavior on a RavenDB server: the default processing of archived documents and the maximum number of concurrent connections per subscription." +sidebar_position: 8 +--- + +import Admonition from '@theme/Admonition'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + +# Data subscriptions configuration options + + + +* Use the configuration keys below to set data subscription options. +* Learn how to apply configuration keys in the [Configuration overview](../server/configuration/configuration-options.mdx). + +* On this page: + * [Subscriptions.ArchivedDataProcessingBehavior](../data-subscriptions/configuration.mdx#subscriptionsarchiveddataprocessingbehavior) + * [Subscriptions.MaxNumberOfConcurrentConnections](../data-subscriptions/configuration.mdx#subscriptionsmaxnumberofconcurrentconnections) + + + + + +The following keys control data subscription behavior. Each can be set server-wide or per database. + +--- + + + +## Subscriptions.ArchivedDataProcessingBehavior + +The default processing behavior for archived documents in a subscription query. + +- **Type**: `enum ArchivedDataProcessingBehavior`: + * `ExcludeArchived`: only non-archived documents are processed by the subscription query. + * `IncludeArchived`: both archived and non-archived documents are processed by the subscription query. + * `ArchivedOnly`: only archived documents are processed by the subscription query. +- **Default**: `ExcludeArchived` +- **Scope**: Server-wide or per database + + + + +## Subscriptions.MaxNumberOfConcurrentConnections + +The maximum number of concurrent subscription connections allowed per database. + +- **Type**: `int` +- **Default**: `1000` +- **Scope**: Server-wide or per database + + + + diff --git a/docs/data-subscriptions/consuming-subscription.mdx b/docs/data-subscriptions/consuming-subscription.mdx new file mode 100644 index 0000000000..005b127243 --- /dev/null +++ b/docs/data-subscriptions/consuming-subscription.mdx @@ -0,0 +1,34 @@ +--- +title: "Consuming a data subscription" +sidebar_label: "Consuming a data subscription" +description: "Consume a RavenDB data subscription with a client worker: create and run the worker, process document batches, set its strategy, and handle errors." +sidebar_position: 3 +--- + +import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; +import LanguageContent from "@site/src/components/LanguageContent"; + +import ConsumingSubscriptionCsharp from './content/_consuming-subscription-csharp.mdx'; +import ConsumingSubscriptionJava from './content/_consuming-subscription-java.mdx'; +import ConsumingSubscriptionNodejs from './content/_consuming-subscription-nodejs.mdx'; +import ConsumingSubscriptionPython from './content/_consuming-subscription-python.mdx'; + +export const supportedLanguages = ["csharp", "java", "nodejs", "python"]; + + + + + + + + + + + + + + + + + + diff --git a/docs/data-subscriptions/content/_concurrent-subscriptions-csharp.mdx b/docs/data-subscriptions/content/_concurrent-subscriptions-csharp.mdx new file mode 100644 index 0000000000..29c267698b --- /dev/null +++ b/docs/data-subscriptions/content/_concurrent-subscriptions-csharp.mdx @@ -0,0 +1,200 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* With concurrent subscriptions, multiple workers can consume the same subscription simultaneously. + +* The server assigns each worker a different batch of documents to process. + +* By processing different batches in parallel, multiple workers can significantly accelerate the + consumption of the subscription's contents. + +* To run workers concurrently, set their [strategy](../../data-subscriptions/consuming-subscription.mdx#worker-strategies) + to `SubscriptionOpeningStrategy.Concurrent`. + Other strategies (`OpenIfFree`, `TakeOver`, `WaitForFree`) let only one worker consume a subscription at a time. + +* If a concurrent worker disconnects, the server may reassign the documents this worker was + processing to other available workers consuming the same subscription. + A document can reach more than one worker this way. + See [Connection failure and reassignment](../../data-subscriptions/concurrent-subscriptions.mdx#connection-failure-and-reassignment) below. + +* In this article: + * [Defining concurrent workers](../../data-subscriptions/concurrent-subscriptions.mdx#defining-concurrent-workers) + * [Dropping a worker](../../data-subscriptions/concurrent-subscriptions.mdx#dropping-a-worker) + * [Connection failure and reassignment](../../data-subscriptions/concurrent-subscriptions.mdx#connection-failure-and-reassignment) + * [Syntax](../../data-subscriptions/concurrent-subscriptions.mdx#syntax) + + + + + +To run workers concurrently, set each worker's strategy to `Concurrent`. +Create the worker using [GetSubscriptionWorker](../../data-subscriptions/consuming-subscription.mdx#methods), +pass it a [SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) +whose `Strategy` is `SubscriptionOpeningStrategy.Concurrent`, and +[run it](../../data-subscriptions/consuming-subscription.mdx#running-the-worker) as you would any +other worker. + +#### Example: Running two concurrent workers + +```csharp +// Two workers, both using the Concurrent strategy, +// so they connect to the "All Orders" subscription at the same time +var worker1 = store.Subscriptions.GetSubscriptionWorker( + new SubscriptionWorkerOptions("All Orders") + { + Strategy = SubscriptionOpeningStrategy.Concurrent, + MaxDocsPerBatch = 20 + }); + +var worker2 = store.Subscriptions.GetSubscriptionWorker( + new SubscriptionWorkerOptions("All Orders") + { + Strategy = SubscriptionOpeningStrategy.Concurrent, + MaxDocsPerBatch = 20 + }); + +// Run both workers. The server assigns each one a different batch of documents. +var subscriptionRuntimeTask1 = worker1.Run(batch => +{ + foreach (var item in batch.Items) + { + ProcessOrder(item.Result); // your processing logic + } +}); + +var subscriptionRuntimeTask2 = worker2.Run(batch => +{ + foreach (var item in batch.Items) + { + ProcessOrder(item.Result); // your processing logic + } +}); +``` + + + + + +To forcefully disconnect a worker from its subscription, pass the worker object to +`Subscriptions.DropSubscriptionWorker`. +The documents the worker was processing then become available for reassignment to the remaining workers. + +#### Example: Dropping one worker + +```csharp +// Disconnect worker2; its documents become available to the other workers +store.Subscriptions.DropSubscriptionWorker(worker2); +``` + +
+ +To disconnect every worker on a subscription at once, by subscription name rather than by a +specific worker, use `DropConnection`. +See [Maintenance operations](../../data-subscriptions/maintenance-operations.mdx#dropping-a-connection). + + + + + +When a concurrent worker's connection ends unexpectedly, the server may reassign the documents the +worker was processing to the subscription's other available workers. +A worker that reconnects to the subscription after a failure receives a new batch, which is not +guaranteed to hold the documents it was processing before. + +Because a worker can disconnect before acknowledging the documents it already handled, the same +document can be processed more than once: first by the worker that dropped, and again by the worker +the document is reassigned to. + + +Plan for [at-least-once delivery](../../data-subscriptions/consuming-subscription.mdx#at-least-once-delivery). +Make each worker's processing idempotent, so reprocessing a document the server reassigns has no +unintended effect. + + + + + + + + +### Methods + + + + +Forcefully disconnect a specific worker from the subscription it is connected to. + +```csharp +void DropSubscriptionWorker(SubscriptionWorker worker, + string database = null) where T : class; + +Task DropSubscriptionWorkerAsync(SubscriptionWorker worker, + string database = null, CancellationToken token = default) where T : class; +``` + +
+ +Usage: + +```csharp +store.Subscriptions.DropSubscriptionWorker(worker2); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **worker** | `SubscriptionWorker` | The worker to disconnect from the subscription. | +| **database** | `string` | The database where the subscription resides. If `null`, the document store's default database is used. | +| **token** | `CancellationToken` | A token used to cancel the asynchronous operation. | + +| Return value | | +|-----------|-------------| +| `void` / `Task` | `DropSubscriptionWorker` returns once the worker is disconnected; `DropSubscriptionWorkerAsync` is its asynchronous version. | + + + + + + + + +### Classes + + + + +The strategy set on a worker's [SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) +`Strategy` property. `Concurrent` is the value that lets multiple workers consume the subscription +at once; the other values allow a single worker at a time, covered in +[Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies). + +```csharp +public enum SubscriptionOpeningStrategy +{ + // Connect only if no other worker is connected + OpenIfFree, + + // Take over the connection from the current worker + TakeOver, + + // Wait for the current worker to disconnect, then take over + WaitForFree, + + // Connect concurrently with the other workers + Concurrent +} +``` + + + + + + + diff --git a/docs/data-subscriptions/content/_concurrent-subscriptions-nodejs.mdx b/docs/data-subscriptions/content/_concurrent-subscriptions-nodejs.mdx new file mode 100644 index 0000000000..eb61782874 --- /dev/null +++ b/docs/data-subscriptions/content/_concurrent-subscriptions-nodejs.mdx @@ -0,0 +1,186 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* With concurrent subscriptions, multiple workers can consume the same subscription simultaneously. + +* The server assigns each worker a different batch of documents to process. + +* By processing different batches in parallel, multiple workers can significantly accelerate the + consumption of the subscription's contents. + +* To run workers concurrently, set their [strategy](../../data-subscriptions/consuming-subscription.mdx#worker-strategies) + to `"Concurrent"`. + Other strategies (`"OpenIfFree"`, `"TakeOver"`, `"WaitForFree"`) let only one worker consume a subscription at a time. + +* If a concurrent worker disconnects, the server may reassign the documents this worker was + processing to other available workers consuming the same subscription. + A document can reach more than one worker this way. + See [Connection failure and reassignment](../../data-subscriptions/concurrent-subscriptions.mdx#connection-failure-and-reassignment) below. + +* In this article: + * [Defining concurrent workers](../../data-subscriptions/concurrent-subscriptions.mdx#defining-concurrent-workers) + * [Dropping a worker](../../data-subscriptions/concurrent-subscriptions.mdx#dropping-a-worker) + * [Connection failure and reassignment](../../data-subscriptions/concurrent-subscriptions.mdx#connection-failure-and-reassignment) + * [Syntax](../../data-subscriptions/concurrent-subscriptions.mdx#syntax) + + + + + +To run workers concurrently, set each worker's strategy to `"Concurrent"`. +Create the worker using [getSubscriptionWorker](../../data-subscriptions/consuming-subscription.mdx#methods), +pass it a [subscription worker options](../../data-subscriptions/consuming-subscription.mdx#classes) +object whose `strategy` is `"Concurrent"`, and run it as you would any other worker by listening for +its `batch` event. + +#### Example: Running two concurrent workers + +```js +// Two workers, both using the Concurrent strategy, +// so they connect to the "All Orders" subscription at the same time +const options = { + subscriptionName: "All Orders", + strategy: "Concurrent", + maxDocsPerBatch: 20 +}; + +const worker1 = documentStore.subscriptions.getSubscriptionWorker(options); +const worker2 = documentStore.subscriptions.getSubscriptionWorker(options); + +// Run both workers. The server assigns each one a different batch of documents. +worker1.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + processOrder(item.result); // your processing logic + } + callback(); + } catch (err) { + callback(err); + } +}); + +worker2.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + processOrder(item.result); // your processing logic + } + callback(); + } catch (err) { + callback(err); + } +}); +``` + + + + + +To forcefully disconnect a worker from its subscription, pass the worker object to +`subscriptions.dropSubscriptionWorker`. +The documents the worker was processing then become available for reassignment to the remaining workers. + +#### Example: Dropping one worker + +```js +// Disconnect worker2; its documents become available to the other workers +await documentStore.subscriptions.dropSubscriptionWorker(worker2); +``` + +
+ +To disconnect every worker on a subscription at once, by subscription name rather than by a +specific worker, use `dropConnection`. +See [Maintenance operations](../../data-subscriptions/maintenance-operations.mdx#dropping-a-connection). + + + + + +When a concurrent worker's connection ends unexpectedly, the server may reassign the documents the +worker was processing to the subscription's other available workers. +A worker that reconnects to the subscription after a failure receives a new batch, which is not +guaranteed to hold the documents it was processing before. + +Because a worker can disconnect before acknowledging the documents it already handled, the same +document can be processed more than once: first by the worker that dropped, and again by the worker +the document is reassigned to. + + +Plan for [at-least-once delivery](../../data-subscriptions/consuming-subscription.mdx#at-least-once-delivery). +Make each worker's processing idempotent, so reprocessing a document the server reassigns has no +unintended effect. + + + + + + + + +### Methods + + + + +Forcefully disconnect a specific worker from the subscription it is connected to. + +```js +dropSubscriptionWorker(worker); +dropSubscriptionWorker(worker, database); +``` + +
+ +Usage: + +```js +await documentStore.subscriptions.dropSubscriptionWorker(worker2); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **worker** | `SubscriptionWorker` | The worker to disconnect from the subscription. | +| **database** | `string` | The database where the subscription resides. If not specified, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `Promise` | Resolves once the worker is disconnected. | + + + + + + + + +### Classes + + + + +The strategy set on the subscription worker options `strategy` property. `"Concurrent"` is the value +that lets multiple workers consume the subscription at once; the other values allow a single worker +at a time, covered in [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies). + +```js +// SubscriptionOpeningStrategy values: +"OpenIfFree" // Connect only if no other worker is connected +"TakeOver" // Take over the connection from the current worker +"WaitForFree" // Wait for the current worker to disconnect, then take over +"Concurrent" // Connect concurrently with the other workers +``` + + + + + + + diff --git a/docs/data-subscriptions/content/_consuming-subscription-csharp.mdx b/docs/data-subscriptions/content/_consuming-subscription-csharp.mdx new file mode 100644 index 0000000000..fe02bd783e --- /dev/null +++ b/docs/data-subscriptions/content/_consuming-subscription-csharp.mdx @@ -0,0 +1,852 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* A **subscription worker** is the client-side component that consumes a subscription. + +* The worker connects to the server, receives a batch of documents, hands the batch to your processing code, and + when done, acknowledges its progress to the server so the server can keep track and provide the next batch. + +* This page explains how to create and run a subscription worker using the client API. + +* In this article: + * [Creating a subscription worker](../../data-subscriptions/consuming-subscription.mdx#creating-a-subscription-worker) + * [Running the worker](../../data-subscriptions/consuming-subscription.mdx#running-the-worker) + * [At-least-once delivery](../../data-subscriptions/consuming-subscription.mdx#at-least-once-delivery) + * [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies) + * [Error handling](../../data-subscriptions/consuming-subscription.mdx#error-handling) + * [Examples](../../data-subscriptions/consuming-subscription.mdx#examples) + * [Full exception handling with retries](../../data-subscriptions/consuming-subscription.mdx#example-full-exception-handling-with-retries) + * [Setting the batch size](../../data-subscriptions/consuming-subscription.mdx#example-setting-the-batch-size) + * [Processing documents in a session](../../data-subscriptions/consuming-subscription.mdx#example-processing-documents-in-a-session) + * [Processing dynamic objects](../../data-subscriptions/consuming-subscription.mdx#example-processing-dynamic-objects) + * [Processing a blittable object](../../data-subscriptions/consuming-subscription.mdx#example-processing-a-blittable-object) + * [End when no documents are left](../../data-subscriptions/consuming-subscription.mdx#example-end-when-no-documents-are-left) + * [Including documents in the batch](../../data-subscriptions/consuming-subscription.mdx#example-including-documents-in-the-batch) + * [Failing over to another node](../../data-subscriptions/consuming-subscription.mdx#example-failing-over-to-another-node) + * [Primary and secondary workers](../../data-subscriptions/consuming-subscription.mdx#example-primary-and-secondary-workers) + * [Syntax](../../data-subscriptions/consuming-subscription.mdx#syntax) + + + + + +Create a worker using the store's `Subscriptions.GetSubscriptionWorker` method, passing either the +subscription name or a [SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) object. + +```csharp +subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); +``` + +This constructs the worker but does not run it yet. + + + + + +To start processing the documents the server sends, call the worker's `Run` method, passing the +logic that handles each batch: + +```csharp +subscriptionRuntimeTask = subscriptionWorker.Run(batch => +{ + // your batch-processing logic here +}); +``` + +
+ +From this point on, the worker receives and processes document batches, acknowledging each batch +so the server can keep track of the subscription's progress and provide the next batch if available. + +`Run` returns a task that stays alive while the worker is processing. If processing is aborted, +this task completes with an exception. + + + + + +The server advances a subscription's progress only when the worker acknowledges a batch. +If a worker stops (or its connection fails) before acknowledging a batch, the server will re-send the +same batch the next time a worker connects to the subscription. + +Delivery is therefore at-least-once: documents may be processed more than once, but none are skipped. +Make your processing idempotent to make sure that reprocessing a document has no unintended effect. + +[Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx#connection-failure-and-reassignment) +add another way a document can be reprocessed, by reassigning a dropped worker's documents to the +other workers. + + + + + +A worker is configured with a strategy, set in the +[SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) +`Strategy` property, that decides whether the worker may connect to the subscription +and how it behaves toward a worker that is already connected. + +The following strategies allow only one worker connected at a time: + +* **`OpenIfFree`** + The server lets the worker connect only if no other worker is connected. + If another worker is already connected, the incoming worker throws a `SubscriptionInUseException`. +* **`WaitForFree`** + If the subscription is in use, the worker waits for the connected worker to disconnect, + then takes its place. + This suits failover, where a standby worker waits to take over from the active one. +* **`TakeOver`** + The incoming worker tries to take over the existing connection. + It succeeds if the connected worker does not itself use `TakeOver`. + - If the takeover succeeds, the worker that has been taken over throws a `SubscriptionInUseException`. + - If the takeover fails, the incoming worker throws the exception instead. + +To let several workers consume the same subscription at the same time, use the `Concurrent` strategy. +See [Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx). + + + +### First worker to connect sets the subscription mode + +The first worker to connect sets whether the subscription runs in single-worker or concurrent mode, +and this mode holds until all workers disconnect: + +* A subscription running concurrent workers admits only other concurrent workers. A worker that uses + a single-worker strategy is rejected with `SubscriptionInUseException`. +* A subscription running a single-worker strategy admits no concurrent worker until the connected + worker disconnects. A concurrent worker is rejected with `SubscriptionInUseException`. + + + + + + + + + +### Connection failures + +When an unexpected error occurs during communication with the server, the worker tries to +reconnect. +It stops without reconnecting in these cases: + +* The subscription no longer exists. +* Another worker took over the subscription (see [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies)). +* The worker cannot connect to any of the servers. +* The worker could not learn which node is responsible for the task (e.g., when the cluster + has no leader). +* An authorization error occurred. +* An error occurred while establishing the connection. +* The database does not exist. + +--- + +When an unexpected error interrupts the worker's connection to the server, the +`OnUnexpectedSubscriptionError` event fires and then the worker automatically attempts to +reconnect. +Subscribe to this event to log worker connection errors: + +```csharp +// Subscribe to the worker's connection-error event +subscriptionWorker.OnUnexpectedSubscriptionError += exception => +{ + // Log the error + Logger.Error("Subscription connection error: " + subscriptionName, exception); +}; +``` + + + + + +### Errors while processing a batch + +When your batch-processing code throws, the `IgnoreSubscriberErrors` option in +`SubscriptionWorkerOptions` determines what happens: + +* **`false`** (default) + Processing stops. The worker wraps the exception in a `SubscriberErrorException` and rethrows + it without acknowledging progress, so the task returned by `Run` completes in error. +* **`true`** + The failed batch is acknowledged without a retry, and the worker continues with the next + batches. + + + + + +### Reconnection and timeouts + +These `SubscriptionWorkerOptions` properties govern reconnection and connection timeouts: + +* **`TimeToWaitBeforeConnectionRetry`** + How long the worker waits before each reconnect attempt. + Default: 5 seconds. +* **`MaxErroneousPeriod`** + How long the connection may stay in an erroneous state before the worker stops retrying. + Default: 5 minutes. +* **`ConnectionStreamTimeout`** + How long the worker waits after losing connectivity before timing out. + Default: 30 seconds. Cannot be set below 15 seconds. + +```csharp +var options = new SubscriptionWorkerOptions(subscriptionName); + +// Set the worker's timeout period +options.ConnectionStreamTimeout = TimeSpan.FromSeconds(45); +``` + + + + + + + + + +### Example: Full exception handling with retries + +In this example, the client handles the exceptions the worker may throw, +recreating the worker to resume processing when the error is recoverable. + +```csharp +while (true) +{ + // Create the worker: + var options = new SubscriptionWorkerOptions(subscriptionName); + + // Configure the worker: + // Allow a downtime of up to 2 hours, + // and wait 2 minutes before reconnecting + options.MaxErroneousPeriod = TimeSpan.FromHours(2); + options.TimeToWaitBeforeConnectionRetry = TimeSpan.FromMinutes(2); + + subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(options); + + try + { + // Subscribe to connection retry events + // and log any exceptions that occur during processing + subscriptionWorker.OnSubscriptionConnectionRetry += exception => + { + Logger.Error("Error during subscription processing: " + subscriptionName, + exception); + }; + + // Run the worker: + await subscriptionWorker.Run(batch => + { + foreach (var item in batch.Items) + { + // Forcefully stop subscription processing if the order's Company is "companies/2-A" + // and throw an exception to let external logic handle the specific case + if (item.Result.Company == "companies/2-A") + { + // The custom exception thrown from here + // will be wrapped by `SubscriberErrorException` + throw new UnsupportedCompanyException( + "Company ID cannot be 'companies/2-A', please fix"); + } + + // Process the order document - provide your own logic + ProcessOrder(item.Result); + } + }, cancellationToken); + + // The Run method will stop if the subscription worker is disposed, + // exiting the while loop + return; + } + catch (Exception e) + { + Logger.Error("Failure in subscription: " + subscriptionName, e); + + // The following exceptions are not recoverable + if (e is DatabaseDoesNotExistException || + e is SubscriptionDoesNotExistException || + e is SubscriptionInvalidStateException || + e is AuthorizationException) + throw; + + if (e is SubscriptionClosedException) + // Subscription probably closed explicitly by admin + return; + + if (e is SubscriberErrorException se) + { + // For UnsupportedCompanyException we want to throw an exception, + // otherwise, continue processing + if (se.InnerException != null && se.InnerException is UnsupportedCompanyException) + { + throw; + } + + // Call continue to skip the current while(true) iteration and try reconnecting + // in the next one, allowing the worker to process future batches. + continue; + } + + // Handle this depending on the subscription opening strategy + if (e is SubscriptionInUseException) + continue; + + // Call return to exit the while(true) loop, + // dispose of the worker (via finally), and stop the subscription. + return; + } + finally + { + subscriptionWorker.Dispose(); + } +} +``` + + + + + +### Example: Setting the batch size + +In this example, the worker sets the maximum number of documents the server sends in each batch. + +```csharp +var workerWBatch = store.Subscriptions.GetSubscriptionWorker( + new SubscriptionWorkerOptions(subscriptionName) + { + MaxDocsPerBatch = 20 + }); + +var subscriptionRuntimeTask = workerWBatch.Run(x => +{ + // your custom logic +}); +``` + + + + + +### Example: Processing documents in a session + +In this example, the subscription sends to the worker `Orders` that have no shipping date. +The worker sets each document's `ShippedAt` field and saves it back through a session. + + +Open the session using `batch.OpenSession`, not `store.OpenSession`. + + +```csharp +// Create the subscription task on the server: +var subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"from Orders as o where o.ShippedAt = null" +}); + +// Create the subscription worker that will consume the documents: +var subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); +var subscriptionRuntimeTask = subscriptionWorker.Run(batch => +{ + // Open a session using 'batch.OpenSession' + using (var session = batch.OpenSession()) + { + foreach (var order in batch.Items.Select(x => x.Result)) + { + TransferOrderToShipmentCompany(order); // call your custom method + order.ShippedAt = DateTime.UtcNow; // update the document field + } + + // Save the updated Order documents + session.SaveChanges(); + } +}); +``` + + + + + +### Example: Processing dynamic objects + +In this example, the subscription projects each `Order` into a dynamic shape, and the worker processes the +dynamic objects it receives. + +```csharp +// Create the subscription task on the server: +var subscriptionName = "My dynamic subscription"; +store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Name = subscriptionName, + Projection = order => + new { DynamicField_1 = "Company: " + order.Company + " Employee: " + order.Employee } +}); + +// Create the subscription worker that will consume the documents: +var subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); +var subscriptionRuntimeTask = subscriptionWorker.Run(batch => +{ + foreach (var item in batch.Items) + { + // Access the dynamic field in the document + dynamic field = item.Result.DynamicField_1; + + // Call your custom method + ProcessItem(field); + } +}); +``` + + + + + +### Example: Processing a blittable object + +In this example, the worker processes documents as low-level blittable objects. + + +Working with blittable objects directly can help in extreme high-performance scenarios, but is +risky because it uses unmanaged memory directly. + + +```csharp +// Create the subscription task on the server: +var subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions +{ + Projection = x => new + { + x.Employee + } +}); + +// Create the subscription worker that will consume the documents: +var subscriptionWorker = + // Specify `BlittableJsonReaderObject` as the generic type parameter + store.Subscriptions.GetSubscriptionWorker(subscriptionName); + +var subscriptionRuntimeTask = subscriptionWorker.Run(batch => +{ + foreach (var item in batch.Items) + { + // Access the Employee field within the blittable object + var employeeField = item.Result["Employee"].ToString(); + + ProcessItem(employeeField); // call your custom method + } +}); +``` + + + + + +### Example: End when no documents are left + +In this example, the worker runs until no new documents remain, which suits ad-hoc, single-use processing where +you need every matching document processed once. +Set `CloseWhenNoDocsLeft` to `true`; the worker then throws a `SubscriptionClosedException` when +it finishes. + +```csharp +// Create the subscription task on the server: +var subscriptionName = store.Subscriptions.Create( + new SubscriptionCreationOptions + { + Filter = order => order.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 10000, + Projection = order => new OrderAndCompany + { + OrderId = order.Id, + Company = RavenQuery.Load(order.Company) + } + }); + +// Create the subscription worker that will consume the documents: +var highValueOrdersWorker = store.Subscriptions.GetSubscriptionWorker( + new SubscriptionWorkerOptions(subscriptionName) + { + // Here we set the worker to stop when there are no more documents left to send + // Will throw SubscriptionClosedException when it finishes its job + CloseWhenNoDocsLeft = true + }); + +try +{ + await highValueOrdersWorker.Run(batch => + { + foreach (var item in batch.Items) + { + SendThankYouNoteToCompany(item.Result); // call your custom method + } + }); +} +catch (SubscriptionClosedException) +{ + // That's expected, no more documents to process +} +``` + + + + + +### Example: Including documents in the batch + +In this example, the subscription sends to the worker every `Order` and includes in the batch +`Product` documents referenced in the order. +When the worker accesses an included `Product`, the product's data has already arrived in the batch, +so the worker doesn't need to fetch it from the server. + +```csharp +// Create the subscription task on the server: +var subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + // Include the referenced Product documents for each Order document + Query = @"from Orders include Lines[].Product" +}); + +// Create the subscription worker that will consume the documents: +var subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(subscriptionName); +var subscriptionRuntimeTask = subscriptionWorker.Run(batch => +{ + // Open a session using 'batch.OpenSession' to access the Product documents + using (var session = batch.OpenSession()) + { + foreach (var order in batch.Items.Select(x => x.Result)) + { + foreach (var orderLine in order.Lines) + { + // Calling Load will not generate a request to the server, + // since orderLine.Product was included in the batch + var product = session.Load(orderLine.Product); + + ProcessOrderAndProduct(order, product); // call your custom method + } + } + } +}); +``` + + + + + +### Example: Failing over to another node + +With the `WaitForFree` strategy, any available node can create a worker, and another node takes +over the connection if the worker fails. + +```csharp +var worker = store.Subscriptions.GetSubscriptionWorker( + new SubscriptionWorkerOptions(subscriptionName) + { + // WaitForFree: wait and take over if the connected worker fails + Strategy = SubscriptionOpeningStrategy.WaitForFree + }); +``` + + + + + +### Example: Primary and secondary workers + +In this example, the two workers form an active-standby pair: the primary, using `TakeOver`, establishes the +connection; the secondary, using `WaitForFree`, waits to take over if the primary fails. + +The primary worker: + +```csharp +var primaryWorker = store.Subscriptions.GetSubscriptionWorker( + new SubscriptionWorkerOptions(subscriptionName) + { + Strategy = SubscriptionOpeningStrategy.TakeOver + }); + +while (true) +{ + try + { + await primaryWorker.Run(x => + { + // your logic + }); + } + catch (Exception) + { + // retry + } +} +``` + +
+ +The secondary worker: + +```csharp +var secondaryWorker = store.Subscriptions.GetSubscriptionWorker( + new SubscriptionWorkerOptions(subscriptionName) + { + Strategy = SubscriptionOpeningStrategy.WaitForFree + }); + +while (true) +{ + try + { + await secondaryWorker.Run(x => + { + // your logic + }); + } + catch (Exception) + { + // retry + } +} +``` + + + + + + + + + +### Methods + + + + +Create a subscription worker through `store.Subscriptions`. + +```csharp +SubscriptionWorker GetSubscriptionWorker( + string subscriptionName, string database = null); + +SubscriptionWorker GetSubscriptionWorker( + SubscriptionWorkerOptions options, string database = null); + +SubscriptionWorker GetSubscriptionWorker( + string subscriptionName, string database = null) where T : class; + +SubscriptionWorker GetSubscriptionWorker( + SubscriptionWorkerOptions options, string database = null) where T : class; +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **subscriptionName** | `string` | The name of the subscription the worker connects to. | +| **options** | `SubscriptionWorkerOptions` | Options that affect how the worker interacts with the subscription. They do not change the subscription's own definition. | +| **database** | `string` | The database where the subscription task resides. If `null`, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `SubscriptionWorker` | The created worker. It is idle until you call `Run`. | + + + + +Start the worker's batch processing. Pass the delegate that processes each received batch. + +```csharp +Task Run(Action> processDocuments, + CancellationToken ct = default(CancellationToken)); + +Task Run(Func, Task> processDocuments, + CancellationToken ct = default(CancellationToken)); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **processDocuments** | `Action>` | Delegate for synchronous batch processing. | +| **processDocuments** | `Func, Task>` | Delegate for asynchronous batch processing. | +| **ct** | `CancellationToken` | A token used to halt the worker. | + +| Return value | | +|-----------|-------------| +| `Task` | A task that stays alive while the worker is processing or trying to. If processing is aborted, the task completes with an exception. | + + + + + + + + +### Classes + + + + +When you create a worker using `SubscriptionWorkerOptions`, the only required property is +`SubscriptionName`; the rest fall back to their defaults. + +```csharp +public class SubscriptionWorkerOptions +{ + public string SubscriptionName { get; } + public int MaxDocsPerBatch { get; set; } + public int SendBufferSizeInBytes { get; set; } + public int ReceiveBufferSizeInBytes { get; set; } + public bool IgnoreSubscriberErrors { get; set; } + public bool CloseWhenNoDocsLeft { get; set; } + public TimeSpan TimeToWaitBeforeConnectionRetry { get; set; } + public TimeSpan ConnectionStreamTimeout { get; set; } + public TimeSpan MaxErroneousPeriod { get; set; } + public SubscriptionOpeningStrategy Strategy { get; set; } +} +``` + +
+ +| Property | Type | Description | +|----------|------|-------------| +| **SubscriptionName** | `string` | The name of the subscription the worker connects to. | +| **MaxDocsPerBatch** | `int` | The maximum number of documents the server tries to send in a batch. If it finds fewer, it sends what it has without waiting. Default: 4096. | +| **SendBufferSizeInBytes** | `int` | The size of the TCP socket buffer used for sending data. Default: 32,768 bytes (32 KiB). | +| **ReceiveBufferSizeInBytes** | `int` | The size of the TCP socket buffer used for receiving data. Default: 32,768 bytes (32 KiB). | +| **IgnoreSubscriberErrors** | `bool` | Whether processing continues when your batch-handling code throws. `true` continues; `false` (default) aborts. | +| **CloseWhenNoDocsLeft** | `bool` | Whether the connection closes when no new documents remain. `true` processes all available documents and then throws a `SubscriptionClosedException` (useful for one-time processing); `false` (default) keeps the worker waiting for new documents. | +| **TimeToWaitBeforeConnectionRetry** | `TimeSpan` | The wait before each reconnect attempt after a non-aborting failure. Default: 5 seconds. | +| **ConnectionStreamTimeout** | `TimeSpan` | How long the worker waits after losing connectivity before timing out. Default: 30 seconds. | +| **MaxErroneousPeriod** | `TimeSpan` | The maximum time the connection may stay in an erroneous state before it is terminated. Default: 5 minutes. | +| **Strategy** | `SubscriptionOpeningStrategy` | How the server handles this worker's connection attempt. Default: `OpenIfFree`. | + + + + +The strategy that decides whether a worker may connect to a subscription. See +[Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies). + +```csharp +public enum SubscriptionOpeningStrategy +{ + // Connect if no other worker is connected + OpenIfFree, + + // Take over the connection + TakeOver, + + // Wait for the currently connected worker to disconnect + WaitForFree, + + // Connect concurrently + Concurrent +} +``` + + + + + + + +The batch passed to your `Run` delegate. + +| Property | Type | Description | +|----------|------|-------------| +| **Items** | `List.Item>` | The items in the batch. | +| **NumberOfItemsInBatch** | `int` | The number of items in the batch. | + +| Method | Return value | Description | +|--------|--------------|-------------| +| **OpenSession()** | `IDocumentSession` | Opens a session that tracks all items and their included items in the current batch. | +| **OpenAsyncSession()** | `IAsyncDocumentSession` | The asynchronous version of `OpenSession()`. | + + +As long as no exception occurs, the worker keeps addressing the server it received the first +batch from. +If it cannot reach that node, it fails over to another node from the session's topology, see +[load balancing](../../client-api/configuration/load-balance/overview.mdx). The node it reaches +then tells it which node currently handles data subscriptions. + + + + + +A single item in a subscription batch. + +```csharp +public struct Item +{ + public T Result { get; internal set; } + public string ExceptionMessage { get; internal set; } + public string Id { get; internal set; } + public string ChangeVector { get; internal set; } + public bool Projection { get; internal set; } + public bool Revision { get; internal set; } + public BlittableJsonReaderObject RawResult { get; internal set; } + public BlittableJsonReaderObject RawMetadata { get; internal set; } + public IMetadataDictionary Metadata { get; internal set; } +} +``` + +
+ +| Property | Type | Description | +|----------|------|-------------| +| **Result** | `T` | The batch item. If `T` is `BlittableJsonReaderObject`, no deserialization takes place. | +| **ExceptionMessage** | `string` | The exception message thrown while processing the document on the server side. | +| **Id** | `string` | The document ID of the item's underlying document. | +| **ChangeVector** | `string` | The change vector of the item's underlying document. | +| **Projection** | `bool` | `true` when the item is a projection produced by the subscription query rather than a full document. | +| **Revision** | `bool` | `true` when the item is a document revision, for a subscription that streams revisions. | +| **RawResult** | `BlittableJsonReaderObject` | The item before serialization to `T`. | +| **RawMetadata** | `BlittableJsonReaderObject` | The metadata of the item's underlying document. | +| **Metadata** | `IMetadataDictionary` | The item's underlying metadata values. | + + +Use this struct only within the subscription's `Run` delegate. Using it outside that scope may +cause unexpected behavior. + + + + + + + + +The worker returned by `GetSubscriptionWorker`. + +**Methods** + +| Method | Return type | Description | +|--------|-------------|-------------| +| **Dispose()** | `void` | Aborts the worker, waiting for the task returned by `Run` to finish. | +| **DisposeAsync()** | `ValueTask` | The asynchronous version of `Dispose()`. | +| **Dispose(bool waitForSubscriptionTask)** | `void` | Aborts the worker, letting you choose whether to wait for the `Run` task. | +| **DisposeAsync(bool waitForSubscriptionTask)** | `ValueTask` | The asynchronous version of `Dispose(bool waitForSubscriptionTask)`. | +| **Run (overloads)** | `Task` | Starts the worker's batch processing. See [Run](../../data-subscriptions/consuming-subscription.mdx#methods) above. | + +**Events** + +| Event | Type | Description | +|-------|------|-------------| +| **AfterAcknowledgment** | `AfterAcknowledgmentAction` | Fires after the server acknowledges the progress of a processed batch. | +| **OnSubscriptionConnectionRetry** | `Action` | Fires when the worker tries to reconnect after a failure. Receives the exception that interrupted processing. | +| **OnUnexpectedSubscriptionError** | `Action` | Fires when an unexpected error interrupts the worker's connection to the server, before the worker reconnects automatically. Receives the exception. | +| **OnDisposed** | `Action>` | Fires after the worker is disposed. | + +**Properties** + +| Property | Type | Description | +|----------|------|-------------| +| **CurrentNodeTag** | `string` | The node tag of the RavenDB server currently handling the subscription. | +| **SubscriptionName** | `string` | The name of the subscription being processed. | +| **WorkerId** | `string` | The worker ID. | + + + + + + + diff --git a/docs/data-subscriptions/content/_consuming-subscription-java.mdx b/docs/data-subscriptions/content/_consuming-subscription-java.mdx new file mode 100644 index 0000000000..2742456371 --- /dev/null +++ b/docs/data-subscriptions/content/_consuming-subscription-java.mdx @@ -0,0 +1,669 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* A **subscription worker** is the client-side component that consumes a subscription. + +* The worker connects to the server, receives a batch of documents, hands the batch to your processing code, and + when done, acknowledges its progress to the server so the server can keep track and provide the next batch. + +* This page explains how to create and run a subscription worker using the client API. + +* In this article: + * [Creating a subscription worker](../../data-subscriptions/consuming-subscription.mdx#creating-a-subscription-worker) + * [Running the worker](../../data-subscriptions/consuming-subscription.mdx#running-the-worker) + * [At-least-once delivery](../../data-subscriptions/consuming-subscription.mdx#at-least-once-delivery) + * [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies) + * [Error handling](../../data-subscriptions/consuming-subscription.mdx#error-handling) + * [Examples](../../data-subscriptions/consuming-subscription.mdx#examples) + * [Full exception handling with retries](../../data-subscriptions/consuming-subscription.mdx#example-full-exception-handling-with-retries) + * [Setting the batch size](../../data-subscriptions/consuming-subscription.mdx#example-setting-the-batch-size) + * [Processing documents in a session](../../data-subscriptions/consuming-subscription.mdx#example-processing-documents-in-a-session) + * [Processing raw objects](../../data-subscriptions/consuming-subscription.mdx#example-processing-raw-objects) + * [End when no documents are left](../../data-subscriptions/consuming-subscription.mdx#example-end-when-no-documents-are-left) + * [Including documents in the batch](../../data-subscriptions/consuming-subscription.mdx#example-including-documents-in-the-batch) + * [Primary and secondary workers](../../data-subscriptions/consuming-subscription.mdx#example-primary-and-secondary-workers) + * [Syntax](../../data-subscriptions/consuming-subscription.mdx#syntax) + + + + + +Create a worker using the store's `subscriptions().getSubscriptionWorker` method, passing the +document class together with either the subscription name or a +[SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) object. + +```java +SubscriptionWorker subscriptionWorker = + store.subscriptions().getSubscriptionWorker(Order.class, subscriptionName); +``` + +This constructs the worker but does not run it yet. + + + + + +To start processing the documents the server sends, call the worker's `run` method, passing the +logic that handles each batch: + +```java +subscriptionWorker.run(batch -> { + // your batch-processing logic here +}); +``` + +
+ +From this point on, the worker receives and processes document batches, acknowledging each batch +so the server can keep track of the subscription's progress and provide the next batch if available. + +`run` returns a `CompletableFuture` that stays alive while the worker is processing. If processing is +aborted, this future completes with an exception. + + + + + +The server advances a subscription's progress only when the worker acknowledges a batch. +If a worker stops (or its connection fails) before acknowledging a batch, the server will re-send the +same batch the next time a worker connects to the subscription. + +Delivery is therefore at-least-once: documents may be processed more than once, but none are skipped. +Make your processing idempotent to make sure that reprocessing a document has no unintended effect. + +[Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx#connection-failure-and-reassignment) +add another way a document can be reprocessed, by reassigning a dropped worker's documents to the +other workers. + + + + + +A worker is configured with a strategy, set in the +[SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) +`strategy` property, that decides whether the worker may connect to the subscription +and how it behaves toward a worker that is already connected. + +The following strategies allow only one worker connected at a time: + +* **`OPEN_IF_FREE`** + The server lets the worker connect only if no other worker is connected. + If another worker is already connected, the incoming worker throws a `SubscriptionInUseException`. +* **`WAIT_FOR_FREE`** + If the subscription is in use, the worker waits for the connected worker to disconnect, + then takes its place. + This suits failover, where a standby worker waits to take over from the active one. +* **`TAKE_OVER`** + The incoming worker tries to take over the existing connection. + It succeeds if the connected worker does not itself use `TAKE_OVER`. + - If the takeover succeeds, the worker that has been taken over throws a `SubscriptionInUseException`. + - If the takeover fails, the incoming worker throws the exception instead. + +To let several workers consume the same subscription at the same time, use the `CONCURRENT` strategy. +See [Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx). + + + +### First worker to connect sets the subscription mode + +The first worker to connect sets whether the subscription runs in single-worker or concurrent mode, +and this mode holds until all workers disconnect: + +* A subscription running concurrent workers admits only other concurrent workers. A worker that uses + a single-worker strategy is rejected with `SubscriptionInUseException`. +* A subscription running a single-worker strategy admits no concurrent worker until the connected + worker disconnects. A concurrent worker is rejected with `SubscriptionInUseException`. + + + + + + + + + +### Connection failures + +When an unexpected error occurs during communication with the server, the worker tries to +reconnect. +It stops without reconnecting in these cases: + +* The subscription no longer exists. +* Another worker took over the subscription (see [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies)). +* The worker cannot connect to any of the servers. +* The worker could not learn which node is responsible for the task (e.g., when the cluster + has no leader). +* An authorization error occurred. +* An error occurred while establishing the connection. +* The database does not exist. + +--- + +When the worker tries to reconnect after a failure, the registered connection-retry listener is +called. Register one to log worker connection errors: + +```java +// Log connection-retry errors +subscriptionWorker.addOnSubscriptionConnectionRetry(exception -> { + logger.error("Subscription connection error: " + subscriptionName, exception); +}); +``` + + + + + +### Errors while processing a batch + +When your batch-processing code throws, the `ignoreSubscriberErrors` option in +`SubscriptionWorkerOptions` determines what happens: + +* **`false`** (default) + Processing stops. The worker wraps the exception in a `SubscriberErrorException` and rethrows + it without acknowledging progress, so the future returned by `run` completes in error. +* **`true`** + The failed batch is acknowledged without a retry, and the worker continues with the next + batches. + + + + + +### Reconnection and timeouts + +These `SubscriptionWorkerOptions` properties govern reconnection after a failure: + +* **`timeToWaitBeforeConnectionRetry`** + How long the worker waits before each reconnect attempt. + Default: 5 seconds. +* **`maxErroneousPeriod`** + How long the connection may stay in an erroneous state before the worker stops retrying. + Default: 5 minutes. + + + + + + + + + +### Example: Full exception handling with retries + +In this example, the client handles the exceptions the worker may throw, +recreating the worker to resume processing when the error is recoverable. + +```java +while (true) { + SubscriptionWorkerOptions options = new SubscriptionWorkerOptions(subscriptionName); + + // Allow a downtime of up to 2 hours, + // and wait 2 minutes before reconnecting + options.setMaxErroneousPeriod(Duration.ofHours(2)); + options.setTimeToWaitBeforeConnectionRetry(Duration.ofMinutes(2)); + + subscriptionWorker = store.subscriptions().getSubscriptionWorker(Order.class, options); + + try { + // Log any exception that occurs during processing + subscriptionWorker.addOnSubscriptionConnectionRetry(exception -> { + logger.error("Error during subscription processing: " + subscriptionName, exception); + }); + + subscriptionWorker.run(batch -> { + for (SubscriptionBatch.Item item : batch.getItems()) { + // Forcefully stop processing for a specific case, + // and throw an exception to let external logic handle it + if ("Europe".equalsIgnoreCase(item.getResult().getShipVia())) { + // The exception thrown here will be wrapped by `SubscriberErrorException` + throw new IllegalStateException("We cannot ship via Europe"); + } + processOrder(item.getResult()); + } + }).get(); + + // run completes normally if you have disposed the subscription + return; + } catch (Exception e) { + logger.error("Failure in subscription: " + subscriptionName, e); + + e = ExceptionsUtils.unwrapException(e); + + // The following exceptions are not recoverable + if (e instanceof DatabaseDoesNotExistException || + e instanceof SubscriptionDoesNotExistException || + e instanceof SubscriptionInvalidStateException || + e instanceof AuthorizationException) { + throw e; + } + + if (e instanceof SubscriptionClosedException) { + // Subscription probably closed explicitly by admin + return; + } + + if (e instanceof SubscriberErrorException) { + SubscriberErrorException se = (SubscriberErrorException) e; + // For IllegalStateException we want to throw, otherwise continue processing + if (se.getCause() != null && se.getCause() instanceof IllegalStateException) { + throw e; + } + continue; + } + + // Handle this depending on the subscription opening strategy + if (e instanceof SubscriptionInUseException) { + continue; + } + + return; + } finally { + subscriptionWorker.close(); + } +} +``` + + + + + +### Example: Setting the batch size + +In this example, the worker sets the maximum number of documents the server sends in each batch. + +```java +SubscriptionWorkerOptions options = new SubscriptionWorkerOptions(subscriptionName); +options.setMaxDocsPerBatch(20); + +SubscriptionWorker workerWBatch = + store.subscriptions().getSubscriptionWorker(Order.class, options); + +workerWBatch.run(x -> { + // your custom logic +}); +``` + + + + + +### Example: Processing documents in a session + +In this example, the subscription sends to the worker `Orders` that have no shipping date. +The worker sets each document's `ShippedAt` field and saves it back through a session. + + +Open the session using `batch.openSession`, not `store.openSession`. + + +```java +// Create the subscription task on the server: +SubscriptionCreationOptions subscriptionCreationOptions = new SubscriptionCreationOptions(); +subscriptionCreationOptions.setQuery("from Orders as o where o.ShippedAt = null"); +String subscriptionName = store.subscriptions().create(subscriptionCreationOptions); + +// Create the subscription worker that will consume the documents: +SubscriptionWorker subscriptionWorker = + store.subscriptions().getSubscriptionWorker(Order.class, subscriptionName); + +subscriptionWorker.run(batch -> { + // Open a session using 'batch.openSession' + try (IDocumentSession session = batch.openSession()) { + for (SubscriptionBatch.Item orderItem : batch.getItems()) { + transferOrderToShipmentCompany(orderItem.getResult()); // call your custom method + orderItem.getResult().setShippedAt(new Date()); // update the document field + } + + // Save the updated Order documents + session.saveChanges(); + } +}); +``` + + + + + +### Example: Processing raw objects + +In this example, the worker processes documents as raw `ObjectNode` objects, with no deserialization +to a document class. + +```java +// Create the subscription task on the server: +SubscriptionCreationOptions subscriptionCreationOptions = new SubscriptionCreationOptions(); +subscriptionCreationOptions.setName("My dynamic subscription"); +subscriptionCreationOptions.setQuery( + "from Orders as o\n" + + "select {\n" + + " DynamicField_1: 'Company: ' + o.Company + ' Employee: ' + o.Employee\n" + + "}"); + +// Create the subscription worker that will consume the documents: +// Omitting the document class yields a worker over raw 'ObjectNode' items +SubscriptionWorker worker = + store.subscriptions().getSubscriptionWorker("My dynamic subscription"); + +worker.run(x -> { + for (SubscriptionBatch.Item item : x.getItems()) { + ObjectNode result = item.getResult(); + + // Access the dynamic field in the document + processItem(result.get("DynamicField_1")); // call your custom method + } +}); +``` + + + + + +### Example: End when no documents are left + +In this example, the worker runs until no new documents remain, which suits ad-hoc, single-use processing where +you need every matching document processed once. +Set `closeWhenNoDocsLeft` to `true`; the worker then throws a `SubscriptionClosedException` when +it finishes. + +```java +SubscriptionWorkerOptions options = new SubscriptionWorkerOptions(subscriptionName); + +// Stop when there are no more documents left to send, +// throwing a SubscriptionClosedException when it finishes +options.setCloseWhenNoDocsLeft(true); + +SubscriptionWorker highValueOrdersWorker = + store.subscriptions().getSubscriptionWorker(OrderAndCompany.class, options); + +try { + highValueOrdersWorker.run(batch -> { + for (SubscriptionBatch.Item item : batch.getItems()) { + sendThankYouNoteToCompany(item.getResult()); // call your custom method + } + }).get(); +} catch (SubscriptionClosedException e) { + // That's expected, no more documents to process +} +``` + + + + + +### Example: Including documents in the batch + +In this example, the subscription sends to the worker every `Order` and includes in the batch +`Product` documents referenced in the order. +When the worker accesses an included `Product`, the product's data has already arrived in the batch, +so the worker doesn't need to fetch it from the server. + +```java +// Create the subscription task on the server: +SubscriptionCreationOptions subscriptionCreationOptions = new SubscriptionCreationOptions(); +subscriptionCreationOptions.setQuery("from Orders include Lines[].Product"); +String subscriptionName = store.subscriptions().create(subscriptionCreationOptions); + +// Create the subscription worker that will consume the documents: +SubscriptionWorker subscriptionWorker = + store.subscriptions().getSubscriptionWorker(Order.class, subscriptionName); + +subscriptionWorker.run(batch -> { + // Open a session using 'batch.openSession' to access the Product documents + try (IDocumentSession session = batch.openSession()) { + for (SubscriptionBatch.Item orderItem : batch.getItems()) { + Order order = orderItem.getResult(); + for (OrderLine orderLine : order.getLines()) { + // Calling load will not generate a request to the server, + // since orderLine.Product was included in the batch + Product product = session.load(Product.class, orderLine.getProduct()); + + processOrderAndProduct(order, product); // call your custom method + } + } + } +}); +``` + + + + + +### Example: Primary and secondary workers + +In this example, the two workers form an active-standby pair: the primary, using `TAKE_OVER`, establishes the +connection; the secondary, using `WAIT_FOR_FREE`, waits to take over if the primary fails. + +The primary worker: + +```java +SubscriptionWorkerOptions options1 = new SubscriptionWorkerOptions(subscriptionName); +options1.setStrategy(SubscriptionOpeningStrategy.TAKE_OVER); +SubscriptionWorker worker1 = store.subscriptions().getSubscriptionWorker(Order.class, options1); + +while (true) { + try { + worker1.run(x -> { + // your logic + }); + } catch (Exception e) { + // retry + } +} +``` + +
+ +The secondary worker: + +```java +SubscriptionWorkerOptions options2 = new SubscriptionWorkerOptions(subscriptionName); +options2.setStrategy(SubscriptionOpeningStrategy.WAIT_FOR_FREE); +SubscriptionWorker worker2 = store.subscriptions().getSubscriptionWorker(Order.class, options2); + +while (true) { + try { + worker2.run(x -> { + // your logic + }); + } catch (Exception e) { + // retry + } +} +``` + + + + + + + + + +### Methods + + + + +Create a subscription worker through `store.subscriptions()`. + +```java +SubscriptionWorker getSubscriptionWorker(String subscriptionName); +SubscriptionWorker getSubscriptionWorker(String subscriptionName, String database); +SubscriptionWorker getSubscriptionWorker(SubscriptionWorkerOptions options); +SubscriptionWorker getSubscriptionWorker(SubscriptionWorkerOptions options, String database); + + SubscriptionWorker getSubscriptionWorker(Class clazz, String subscriptionName); + SubscriptionWorker getSubscriptionWorker(Class clazz, String subscriptionName, String database); + SubscriptionWorker getSubscriptionWorker(Class clazz, SubscriptionWorkerOptions options); + SubscriptionWorker getSubscriptionWorker(Class clazz, SubscriptionWorkerOptions options, String database); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **clazz** | `Class` | The document class the worker deserializes items into. Omit it to receive raw `ObjectNode` items. | +| **subscriptionName** | `String` | The name of the subscription the worker connects to. | +| **options** | `SubscriptionWorkerOptions` | Options that affect how the worker interacts with the subscription. They do not change the subscription's own definition. | +| **database** | `String` | The database where the subscription task resides. If `null`, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `SubscriptionWorker` | The created worker. It is idle until you call `run`. | + + + + +Start the worker's batch processing. Pass the delegate that processes each received batch. + +```java +CompletableFuture run(Consumer> processDocuments); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **processDocuments** | `Consumer>` | Delegate for batch processing. | + +| Return value | | +|-----------|-------------| +| `CompletableFuture` | A future that stays alive while the worker is processing or trying to. If processing is aborted, the future completes with an exception. | + + + + + + + + +### Classes + + + + +When you create a worker using `SubscriptionWorkerOptions`, the only required property is +`subscriptionName`; the rest fall back to their defaults. + +| Property | Type | Description | +|----------|------|-------------| +| **subscriptionName** | `String` | The name of the subscription the worker connects to. | +| **maxDocsPerBatch** | `int` | The maximum number of documents the server tries to send in a batch. If it finds fewer, it sends what it has without waiting. Default: 4096. | +| **sendBufferSize** | `int` | The size of the TCP socket buffer used for sending data. Default: 32,768 bytes (32 KiB). | +| **receiveBufferSize** | `int` | The size of the TCP socket buffer used for receiving data. Default: 32,768 bytes (32 KiB). | +| **ignoreSubscriberErrors** | `boolean` | Whether processing continues when your batch-handling code throws. `true` continues; `false` (default) aborts. | +| **closeWhenNoDocsLeft** | `boolean` | Whether the connection closes when no new documents remain. `true` processes all available documents and then throws a `SubscriptionClosedException` (useful for one-time processing); `false` (default) keeps the worker waiting for new documents. | +| **timeToWaitBeforeConnectionRetry** | `Duration` | The wait before each reconnect attempt after a non-aborting failure. Default: 5 seconds. | +| **maxErroneousPeriod** | `Duration` | The maximum time the connection may stay in an erroneous state before it is terminated. Default: 5 minutes. | +| **strategy** | `SubscriptionOpeningStrategy` | How the server handles this worker's connection attempt. Default: `OPEN_IF_FREE`. | + + + + +The strategy that decides whether a worker may connect to a subscription. See +[Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies). + +```java +public enum SubscriptionOpeningStrategy { + // Connect if no other worker is connected + OPEN_IF_FREE, + + // Take over the connection + TAKE_OVER, + + // Wait for the currently connected worker to disconnect + WAIT_FOR_FREE, + + // Connect concurrently + CONCURRENT +} +``` + + + + + + + +The batch passed to your `run` delegate. + +| Property | Type | Description | +|----------|------|-------------| +| **items** | `List.Item>` | The items in the batch. | +| **numberOfItemsInBatch** | `int` | The number of items in the batch. | + +| Method | Return value | Description | +|--------|--------------|-------------| +| **openSession()** | `IDocumentSession` | Opens a session that tracks all items and their included items in the current batch. | + + +As long as no exception occurs, the worker keeps addressing the server it received the first +batch from. +If it cannot reach that node, it fails over to another node from the session's topology, see +[load balancing](../../client-api/configuration/load-balance/overview.mdx). The node it reaches +then tells it which node currently handles data subscriptions. + + + + + +A single item in a subscription batch. + + +If `T` is `ObjectNode`, no deserialization takes place. + + +| Property | Type | Description | +|----------|------|-------------| +| **result** | `T` | The batch item. | +| **exceptionMessage** | `String` | The exception message thrown while processing the document on the server side. | +| **id** | `String` | The document ID of the item's underlying document. | +| **changeVector** | `String` | The change vector of the item's underlying document. | +| **projection** | `boolean` | `true` when the item is a projection produced by the subscription query rather than a full document. | +| **revision** | `boolean` | `true` when the item is a document revision, for a subscription that streams revisions. | +| **rawResult** | `ObjectNode` | The item before deserialization to `T`. | +| **rawMetadata** | `ObjectNode` | The metadata of the item's underlying document. | +| **metadata** | `IMetadataDictionary` | The item's underlying metadata values. | + + + + + + + +The worker returned by `getSubscriptionWorker`. + +**Methods** + +| Method | Return type | Description | +|--------|-------------|-------------| +| **close()** | `void` | Aborts the worker, waiting for the future returned by `run` to finish. | +| **run (overloads)** | `CompletableFuture` | Starts the worker's batch processing. See [run](../../data-subscriptions/consuming-subscription.mdx#methods) above. | + +**Events** + +| Event | Type | Description | +|-------|------|-------------| +| **addAfterAcknowledgmentListener** | `Consumer>` | Fires after the server acknowledges the progress of a processed batch. | +| **addOnSubscriptionConnectionRetry** | `Consumer` | Fires when the worker tries to reconnect after a failure. Receives the exception that interrupted processing. | +| **onClosed** | `Consumer>` | Fires after the worker is disposed. | + +**Properties** + +| Property | Type | Description | +|----------|------|-------------| +| **currentNodeTag** | `String` | The node tag of the RavenDB server currently handling the subscription. | +| **subscriptionName** | `String` | The name of the subscription being processed. | + + + + + + + diff --git a/docs/data-subscriptions/content/_consuming-subscription-nodejs.mdx b/docs/data-subscriptions/content/_consuming-subscription-nodejs.mdx new file mode 100644 index 0000000000..0ec99bb377 --- /dev/null +++ b/docs/data-subscriptions/content/_consuming-subscription-nodejs.mdx @@ -0,0 +1,739 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* A **subscription worker** is the client-side component that consumes a subscription. + +* The worker connects to the server, receives a batch of documents, hands the batch to your processing code, and + when done, acknowledges its progress to the server so the server can keep track and provide the next batch. + +* This page explains how to create and run a subscription worker using the client API. + +* In this article: + * [Creating a subscription worker](../../data-subscriptions/consuming-subscription.mdx#creating-a-subscription-worker) + * [Running the worker](../../data-subscriptions/consuming-subscription.mdx#running-the-worker) + * [At-least-once delivery](../../data-subscriptions/consuming-subscription.mdx#at-least-once-delivery) + * [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies) + * [Error handling](../../data-subscriptions/consuming-subscription.mdx#error-handling) + * [Examples](../../data-subscriptions/consuming-subscription.mdx#examples) + * [Full exception handling with retries](../../data-subscriptions/consuming-subscription.mdx#example-full-exception-handling-with-retries) + * [Setting the batch size](../../data-subscriptions/consuming-subscription.mdx#example-setting-the-batch-size) + * [Processing documents in a session](../../data-subscriptions/consuming-subscription.mdx#example-processing-documents-in-a-session) + * [Processing dynamic objects](../../data-subscriptions/consuming-subscription.mdx#example-processing-dynamic-objects) + * [End when no documents are left](../../data-subscriptions/consuming-subscription.mdx#example-end-when-no-documents-are-left) + * [Including documents in the batch](../../data-subscriptions/consuming-subscription.mdx#example-including-documents-in-the-batch) + * [Primary and secondary workers](../../data-subscriptions/consuming-subscription.mdx#example-primary-and-secondary-workers) + * [Syntax](../../data-subscriptions/consuming-subscription.mdx#syntax) + + + + + +Create a worker using the store's `subscriptions.getSubscriptionWorker` method, passing either the +subscription name or a [subscription worker options](../../data-subscriptions/consuming-subscription.mdx#classes) object. + +```js +const subscriptionWorker = documentStore.subscriptions.getSubscriptionWorker(subscriptionName); +``` + +This constructs the worker but does not run it yet. + + + + + +To start processing the documents the server sends, attach a handler to the worker's `batch` event. +Attaching the first `batch` listener starts the worker: + +```js +subscriptionWorker.on("batch", (batch, callback) => { + try { + // your batch-processing logic here + + // Call callback() when done, so the worker acknowledges the batch + callback(); + } catch (err) { + callback(err); + } +}); +``` + +
+ +From this point on, the worker receives and processes document batches. Call `callback()` after +processing each batch so the server can keep track of the subscription's progress and provide the +next batch if available; call `callback(err)` to report a processing failure. + + + + + +The server advances a subscription's progress only when the worker acknowledges a batch. +If a worker stops (or its connection fails) before acknowledging a batch, the server will re-send the +same batch the next time a worker connects to the subscription. + +Delivery is therefore at-least-once: documents may be processed more than once, but none are skipped. +Make your processing idempotent to make sure that reprocessing a document has no unintended effect. + +[Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx#connection-failure-and-reassignment) +add another way a document can be reprocessed, by reassigning a dropped worker's documents to the +other workers. + + + + + +A worker is configured with a strategy, set in the +[subscription worker options](../../data-subscriptions/consuming-subscription.mdx#classes) +`strategy` property, that decides whether the worker may connect to the subscription +and how it behaves toward a worker that is already connected. + +The following strategies allow only one worker connected at a time: + +* **`"OpenIfFree"`** + The server lets the worker connect only if no other worker is connected. + If another worker is already connected, the incoming worker fails with a `SubscriptionInUseException`. +* **`"WaitForFree"`** + If the subscription is in use, the worker waits for the connected worker to disconnect, + then takes its place. + This suits failover, where a standby worker waits to take over from the active one. +* **`"TakeOver"`** + The incoming worker tries to take over the existing connection. + It succeeds if the connected worker does not itself use `"TakeOver"`. + - If the takeover succeeds, the worker that has been taken over fails with a `SubscriptionInUseException`. + - If the takeover fails, the incoming worker fails with the exception instead. + +To let several workers consume the same subscription at the same time, use the `"Concurrent"` strategy. +See [Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx). + + + +### First worker to connect sets the subscription mode + +The first worker to connect sets whether the subscription runs in single-worker or concurrent mode, +and this mode holds until all workers disconnect: + +* A subscription running concurrent workers admits only other concurrent workers. A worker that uses + a single-worker strategy is rejected with `SubscriptionInUseException`. +* A subscription running a single-worker strategy admits no concurrent worker until the connected + worker disconnects. A concurrent worker is rejected with `SubscriptionInUseException`. + + + + + + + + + +### Connection failures + +When an unexpected error occurs during communication with the server, the worker tries to +reconnect. +It stops without reconnecting in these cases: + +* The subscription no longer exists. +* Another worker took over the subscription (see [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies)). +* The worker cannot connect to any of the servers. +* The worker could not learn which node is responsible for the task (e.g., when the cluster + has no leader). +* An authorization error occurred. +* An error occurred while establishing the connection. +* The database does not exist. + +--- + +When the worker tries to reconnect after a failure, the `connectionRetry` event fires. +Subscribe to it to log worker connection errors: + +```js +// Log connection-retry errors +subscriptionWorker.on("connectionRetry", error => { + console.error("Subscription connection error: " + subscriptionName, error); +}); +``` + + + + + +### Errors while processing a batch + +When your batch-processing code throws (or calls `callback(err)`), the `ignoreSubscriberErrors` +option determines what happens: + +* **`false`** (default) + Processing stops. The worker wraps the exception in a `SubscriberErrorException` and emits it + through the `error` event, without acknowledging progress. +* **`true`** + The failed batch is acknowledged without a retry, and the worker continues with the next + batches. + + + + + +### Reconnection and timeouts + +These worker options govern reconnection after a failure: + +* **`timeToWaitBeforeConnectionRetry`** + How long the worker waits before each reconnect attempt, in milliseconds. + Default: 5 seconds. +* **`maxErroneousPeriod`** + How long the connection may stay in an erroneous state before the worker stops retrying, in + milliseconds. + Default: 5 minutes. + + + + + + + + + +### Example: Full exception handling with retries + +In this example, the client handles the exceptions the worker may emit, +recreating the worker to resume processing when the error is recoverable. + +```js +// Create the subscription task on the server: +const subscriptionName = await documentStore.subscriptions.create({ + name: "ProcessOrdersWithLowFreight", + query: "from Orders where Freight < 0.5" +}); + +// Create the subscription worker that will consume the documents: +await setupReconnectingWorker(subscriptionName); + +async function setupReconnectingWorker(subscriptionName) { + let subscriptionWorker; + + function closeWorker(worker) { + worker.dispose(); + } + + function reconnect() { + if (subscriptionWorker) { + closeWorker(subscriptionWorker); + } + + // Configure the worker: + const subscriptionWorkerOptions = { + subscriptionName: subscriptionName, + // Allow a downtime of up to 2 hours + maxErroneousPeriod: 2 * 3600 * 1000, + // Wait 2 minutes before reconnecting + timeToWaitBeforeConnectionRetry: 2 * 60 * 1000 + }; + + subscriptionWorker = documentStore.subscriptions.getSubscriptionWorker(subscriptionWorkerOptions); + + // Subscribe to connection retry events, and log any exceptions + subscriptionWorker.on("connectionRetry", error => { + console.error("Error during subscription processing: " + subscriptionName, error); + }); + + // Run the worker: + subscriptionWorker.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + const orderDocument = item.result; + + // Forcefully stop processing if the order's Company is "companies/46-A", + // and report an error to let external logic handle the specific case + if (orderDocument.Company && orderDocument.Company === "companies/46-A") { + // The error reported here will be wrapped by `SubscriberErrorException` + callback(new Error("Company ID can't be 'companies/46-A', please fix")); + return; + } + + // Process the order document - provide your own logic + processOrder(orderDocument); + } + // Call 'callback' once you're done so the worker acknowledges the batch + callback(); + } catch (err) { + callback(err); + } + }); + + // Handle errors: + subscriptionWorker.on("error", error => { + console.error("Failure in subscription: " + subscriptionName, error); + + // The following errors are not recoverable + if (error.name === "DatabaseDoesNotExistException" || + error.name === "SubscriptionDoesNotExistException" || + error.name === "SubscriptionInvalidStateException" || + error.name === "AuthorizationException") { + throw error; + } + + if (error.name === "SubscriptionClosedException") { + // Subscription probably closed explicitly by admin + return closeWorker(subscriptionWorker); + } + + // Otherwise, try reconnecting + setTimeout(reconnect, 1000); + }); + } + + reconnect(); +} +``` + + + + + +### Example: Setting the batch size + +In this example, the worker sets the maximum number of documents the server sends in each batch. + +```js +const workerOptions = { + subscriptionName: subscriptionName, + maxDocsPerBatch: 20 +}; + +const worker = documentStore.subscriptions.getSubscriptionWorker(workerOptions); + +worker.on("batch", (batch, callback) => { + try { + // your custom logic + callback(); + } catch (err) { + callback(err); + } +}); +``` + + + + + +### Example: Processing documents in a session + +In this example, the subscription sends to the worker `Orders` that have no shipping date. +The worker sets each document's `ShippedAt` field and saves it back through a session. + + +Open the session using `batch.openSession`, not `documentStore.openSession`. + + +```js +// Create the subscription task on the server: +const subscriptionName = await documentStore.subscriptions.create({ + name: "ProcessOrdersThatWereNotShipped", + query: "from Orders as o where o.ShippedAt = null" +}); + +// Create the subscription worker that will consume the documents: +const worker = documentStore.subscriptions.getSubscriptionWorker({ subscriptionName }); + +worker.on("batch", async (batch, callback) => { + try { + // Open a session using 'batch.openSession' + const session = batch.openSession(); + + for (const item of batch.items) { + const orderDocument = item.result; + + transferOrderToShipmentCompany(orderDocument); // call your custom method + orderDocument.ShippedAt = new Date(); // update the document field + } + + // Save the updated Order documents + await session.saveChanges(); + callback(); + } catch (err) { + callback(err); + } +}); +``` + + + + + +### Example: Processing dynamic objects + +In this example, the subscription projects each `Order` into a dynamic shape, and the worker processes the +dynamic objects it receives. + +```js +// Create the subscription task on the server: +const subscriptionName = await documentStore.subscriptions.create({ + name: "ProcessDynamicFields", + query: `from Orders as o + select { + dynamicField: "Company: " + o.Company + " Employee: " + o.Employee + }` +}); + +// Create the subscription worker that will consume the documents: +const worker = documentStore.subscriptions.getSubscriptionWorker({ subscriptionName }); + +worker.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + // Access the dynamic field in the document + const field = item.result.dynamicField; + + processItem(field); // call your custom method + } + callback(); + } catch (err) { + callback(err); + } +}); +``` + + + + + +### Example: End when no documents are left + +In this example, the worker runs until no new documents remain, which suits ad-hoc, single-use processing where +you need every matching document processed once. +Set `closeWhenNoDocsLeft` to `true`; the worker then emits a `SubscriptionClosedException` when +it finishes. + +```js +// Create the subscription task on the server: +const query = `declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + from Orders as o + where getOrderLinesSum(o) > 10000`; + +const subscriptionName = await documentStore.subscriptions.create({ query }); + +// Create the subscription worker that will consume the documents: +const workerOptions = { + subscriptionName: subscriptionName, + // Stop when there are no more documents left to send, + // emitting a SubscriptionClosedException when it finishes + closeWhenNoDocsLeft: true +}; + +const highValueOrdersWorker = documentStore.subscriptions.getSubscriptionWorker(workerOptions); + +highValueOrdersWorker.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + sendThankYouNoteToCompany(item.result); // call your custom method + } + callback(); + } catch (err) { + callback(err); + } +}); + +highValueOrdersWorker.on("error", err => { + if (err.name === "SubscriptionClosedException") { + // That's expected, no more documents to process + } +}); +``` + + + + + +### Example: Including documents in the batch + +In this example, the subscription sends to the worker every `Order` and includes in the batch +`Product` documents referenced in the order. +When the worker accesses an included `Product`, the product's data has already arrived in the batch, +so the worker doesn't need to fetch it from the server. + +```js +// Create the subscription task on the server: +const subscriptionName = await documentStore.subscriptions.create({ + name: "ProcessIncludedDocuments", + query: "from Orders include Lines[].Product" +}); + +// Create the subscription worker that will consume the documents: +const worker = documentStore.subscriptions.getSubscriptionWorker({ subscriptionName }); + +worker.on("batch", async (batch, callback) => { + try { + // Open a session using 'batch.openSession' to access the Product documents + const session = batch.openSession(); + + for (const item of batch.items) { + const orderDocument = item.result; + + for (const orderLine of orderDocument.Lines) { + // Calling 'load' will not generate a request to the server, + // because orderLine.Product was included in the batch + const product = await session.load(orderLine.Product); + + processOrderAndProduct(orderDocument, product); // call your custom method + } + } + callback(); + } catch (err) { + callback(err); + } +}); +``` + + + + + +### Example: Primary and secondary workers + +In this example, the two workers form an active-standby pair: the primary, using `"TakeOver"`, establishes the +connection; the secondary, using `"WaitForFree"`, waits to take over if the primary fails. + +The primary worker: + +```js +const worker1 = documentStore.subscriptions.getSubscriptionWorker({ + subscriptionName, + strategy: "TakeOver", + documentType: Order +}); + +worker1.on("batch", (batch, callback) => { + // your logic + callback(); +}); + +worker1.on("error", err => { + // retry +}); +``` + +
+ +The secondary worker: + +```js +const worker2 = documentStore.subscriptions.getSubscriptionWorker({ + subscriptionName, + strategy: "WaitForFree", + documentType: Order +}); + +worker2.on("batch", (batch, callback) => { + // your logic + callback(); +}); + +worker2.on("error", err => { + // retry +}); +``` + + + + + + + + + +### Methods + + + + +Create a subscription worker through `documentStore.subscriptions`. + +```js +// Overloads: +getSubscriptionWorker(subscriptionName); +getSubscriptionWorker(subscriptionName, database); +getSubscriptionWorker(options); +getSubscriptionWorker(options, database); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **subscriptionName** | `string` | The name of the subscription the worker connects to. | +| **options** | `object` | Options that affect how the worker interacts with the subscription. They do not change the subscription's own definition. See the [Classes](../../data-subscriptions/consuming-subscription.mdx#classes) below. | +| **database** | `string` | The database where the subscription task resides. If not specified, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `SubscriptionWorker` | The created worker. It starts processing once you attach a handler to its `batch` event. | + + + + + + + + +### Classes + + + + +When you create a worker with options, the only required property is `subscriptionName`; the rest +fall back to their defaults. + +```js +// The subscription worker options object: +{ + subscriptionName; + documentType; + ignoreSubscriberErrors; + closeWhenNoDocsLeft; + maxDocsPerBatch; + timeToWaitBeforeConnectionRetry; + maxErroneousPeriod; + strategy; +} +``` + +
+ +| Property | Type | Description | +|----------|------|-------------| +| **subscriptionName** | `string` | The name of the subscription the worker connects to. | +| **documentType** | `object` | The document class of the subscription's documents. | +| **ignoreSubscriberErrors** | `boolean` | Whether processing continues when your batch-handling code throws. `true` continues; `false` (default) aborts. | +| **closeWhenNoDocsLeft** | `boolean` | Whether the connection closes when no new documents remain. `true` processes all available documents and then emits a `SubscriptionClosedException` (useful for one-time processing); `false` (default) keeps the worker waiting for new documents. | +| **maxDocsPerBatch** | `number` | The maximum number of documents the server tries to send in a batch. If it finds fewer, it sends what it has without waiting. Default: 4096. | +| **timeToWaitBeforeConnectionRetry** | `number` | The wait before each reconnect attempt after a non-aborting failure, in milliseconds. Default: 5 seconds. | +| **maxErroneousPeriod** | `number` | The maximum time the connection may stay in an erroneous state before it is terminated, in milliseconds. Default: 5 minutes. | +| **strategy** | `string` | How the server handles this worker's connection attempt. Default: `"OpenIfFree"`. | + + + + +The strategy that decides whether a worker may connect to a subscription, set as a string on the +`strategy` option. See [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies). + +```js +// Available values: +"OpenIfFree" // Connect if no other worker is connected +"TakeOver" // Take over the connection +"WaitForFree" // Wait for the currently connected worker to disconnect +"Concurrent" // Connect concurrently +``` + + + + + + + +The batch passed to your `batch` event handler. + +| Property | Type | Description | +|----------|------|-------------| +| **items** | `Item[]` | The items in the batch. | + +| Method | Return value | Description | +|--------|--------------|-------------| +| **getNumberOfItemsInBatch()** | `number` | The number of items in the batch. | +| **getNumberOfIncludes()** | `number` | The number of included documents in the batch. | +| **openSession()** | `object` | Opens a session that tracks all items and their included items in the current batch. | +| **openSession(options)** | `object` | Opens a session with the given [session options](../../client-api/session/opening-a-session.mdx#session-options). | + + +As long as no exception occurs, the worker keeps addressing the server it received the first +batch from. +If it cannot reach that node, it fails over to another node from the session's topology, see +[load balancing](../../client-api/configuration/load-balance/overview.mdx). The node it reaches +then tells it which node currently handles data subscriptions. + + + + + +A single item in a subscription batch. + +```js +class Item +{ + result; + exceptionMessage; + id; + changeVector; + projection; + revision; + rawResult; + rawMetadata; + metadata; +} +``` + +
+ +| Property | Type | Description | +|----------|------|-------------| +| **result** | `object` | The batch item. | +| **exceptionMessage** | `string` | The exception message thrown while processing the document on the server side. | +| **id** | `string` | The document ID of the item's underlying document. | +| **changeVector** | `string` | The change vector of the item's underlying document. | +| **projection** | `boolean` | `true` when the item is a projection produced by the subscription query rather than a full document. | +| **revision** | `boolean` | `true` when the item is a document revision, for a subscription that streams revisions. | +| **rawResult** | `object` | The item before deserialization. | +| **rawMetadata** | `object` | The metadata of the item's underlying document. | +| **metadata** | `object` | The item's underlying metadata values. | + + + + + + + +The worker returned by `getSubscriptionWorker`. + +**Methods** + +| Method | Return type | Description | +|--------|-------------|-------------| +| **dispose()** | `void` | Aborts the worker. | +| **on(event, handler)** | `object` | Attaches an event listener. | +| **getWorkerId()** | `string` | The worker ID. | + +**Events** + +| Event | Listener signature | Description | +|-------|--------------------|-------------| +| **"batch"** | `(batch, callback) => void` | Fires when a batch of documents arrives from the server. Call `callback()` when done so the worker acknowledges the batch and the server sends the next one. | +| **"afterAcknowledgment"** | `(batch, callback) => void` | Fires after the server acknowledges the progress of a processed batch. | +| **"connectionRetry"** | `(error) => void` | Fires when the worker tries to reconnect after a failure. | +| **"unexpectedSubscriptionError"** | `(error) => void` | Fires when an unexpected error interrupts the worker's connection, before it reconnects automatically. | +| **"error"** | `(error) => void` | Fires on subscription errors. | +| **"end"** | `(error) => void` | Fires when the subscription finishes and no more batches will be emitted. | + +**Properties** + +| Property | Type | Description | +|----------|------|-------------| +| **currentNodeTag** | `string` | The node tag of the RavenDB server currently handling the subscription. | +| **subscriptionName** | `string` | The name of the subscription being processed. | + + + + + + + diff --git a/docs/data-subscriptions/content/_consuming-subscription-python.mdx b/docs/data-subscriptions/content/_consuming-subscription-python.mdx new file mode 100644 index 0000000000..8e60d33a01 --- /dev/null +++ b/docs/data-subscriptions/content/_consuming-subscription-python.mdx @@ -0,0 +1,608 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* A **subscription worker** is the client-side component that consumes a subscription. + +* The worker connects to the server, receives a batch of documents, hands the batch to your processing code, and + when done, acknowledges its progress to the server so the server can keep track and provide the next batch. + +* This page explains how to create and run a subscription worker using the client API. + +* In this article: + * [Creating a subscription worker](../../data-subscriptions/consuming-subscription.mdx#creating-a-subscription-worker) + * [Running the worker](../../data-subscriptions/consuming-subscription.mdx#running-the-worker) + * [At-least-once delivery](../../data-subscriptions/consuming-subscription.mdx#at-least-once-delivery) + * [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies) + * [Error handling](../../data-subscriptions/consuming-subscription.mdx#error-handling) + * [Examples](../../data-subscriptions/consuming-subscription.mdx#examples) + * [Full exception handling with retries](../../data-subscriptions/consuming-subscription.mdx#example-full-exception-handling-with-retries) + * [Setting the batch size](../../data-subscriptions/consuming-subscription.mdx#example-setting-the-batch-size) + * [Processing documents in a session](../../data-subscriptions/consuming-subscription.mdx#example-processing-documents-in-a-session) + * [Processing dynamic objects](../../data-subscriptions/consuming-subscription.mdx#example-processing-dynamic-objects) + * [End when no documents are left](../../data-subscriptions/consuming-subscription.mdx#example-end-when-no-documents-are-left) + * [Failing over to another node](../../data-subscriptions/consuming-subscription.mdx#example-failing-over-to-another-node) + * [Primary and secondary workers](../../data-subscriptions/consuming-subscription.mdx#example-primary-and-secondary-workers) + * [Syntax](../../data-subscriptions/consuming-subscription.mdx#syntax) + + + + + +Create a worker using the store's `subscriptions.get_subscription_worker` method (or +`get_subscription_worker_by_name` for default options), passing the document type together with the +subscription name or a [SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) object. + +```python +subscription_worker = store.subscriptions.get_subscription_worker_by_name(subscription_name, Order) +``` + +This constructs the worker but does not run it yet. + + + + + +To start processing the documents the server sends, call the worker's `run` method, passing a +function that handles each batch: + +```python +def process_batch(batch): + # your batch-processing logic here + ... + +subscription_worker.run(process_batch) +``` + +
+ +From this point on, the worker receives and processes document batches, acknowledging each batch +so the server can keep track of the subscription's progress and provide the next batch if available. + +`run` returns a `Future` that stays alive while the worker is processing. If processing is aborted, +the future completes with an exception. + + + + + +The server advances a subscription's progress only when the worker acknowledges a batch. +If a worker stops (or its connection fails) before acknowledging a batch, the server will re-send the +same batch the next time a worker connects to the subscription. + +Delivery is therefore at-least-once: documents may be processed more than once, but none are skipped. +Make your processing idempotent to make sure that reprocessing a document has no unintended effect. + + + + + +A worker is configured with a strategy, set in the +[SubscriptionWorkerOptions](../../data-subscriptions/consuming-subscription.mdx#classes) +`strategy` property, that decides whether the worker may connect to the subscription +and how it behaves toward a worker that is already connected: + +* **`OPEN_IF_FREE`** + The server lets the worker connect only if no other worker is connected. + If another worker is already connected, the incoming worker throws a `SubscriptionInUseException`. +* **`WAIT_FOR_FREE`** + If the subscription is in use, the worker waits for the connected worker to disconnect, + then takes its place. + This suits failover, where a standby worker waits to take over from the active one. +* **`TAKE_OVER`** + The incoming worker tries to take over the existing connection. + It succeeds if the connected worker does not itself use `TAKE_OVER`. + - If the takeover succeeds, the worker that has been taken over throws a `SubscriptionInUseException`. + - If the takeover fails, the incoming worker throws the exception instead. + + + + + + + +### Connection failures + +When an unexpected error occurs during communication with the server, the worker tries to +reconnect. +It stops without reconnecting in these cases: + +* The subscription no longer exists. +* Another worker took over the subscription (see [Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies)). +* The worker cannot connect to any of the servers. +* The worker could not learn which node is responsible for the task (e.g., when the cluster + has no leader). +* An authorization error occurred. +* An error occurred while establishing the connection. +* The database does not exist. + +--- + +When the worker tries to reconnect after a failure, the registered connection-retry handler is +called. Register one to log worker connection errors: + +```python +# Log connection-retry errors +subscription_worker.add_on_subscription_connection_retry( + lambda exception: logger.error( + f"Subscription connection error: {subscription_name}", exc_info=exception + ) +) +``` + + + + + +### Errors while processing a batch + +When your batch-processing code throws, the `ignore_subscriber_errors` option in +`SubscriptionWorkerOptions` determines what happens: + +* **`False`** (default) + Processing stops. The worker wraps the exception in a `SubscriberErrorException` and rethrows + it without acknowledging progress, so the future returned by `run` completes in error. +* **`True`** + The failed batch is acknowledged without a retry, and the worker continues with the next + batches. + + + + + +### Reconnection and timeouts + +These `SubscriptionWorkerOptions` properties govern reconnection after a failure: + +* **`time_to_wait_before_connection_retry`** + How long the worker waits before each reconnect attempt. + Default: 5 seconds. +* **`max_erroneous_period`** + How long the connection may stay in an erroneous state before the worker stops retrying. + Default: 5 minutes. + + + + + + + + + +### Example: Full exception handling with retries + +In this example, the client handles the exceptions the worker may throw, +recreating the worker to resume processing when the error is recoverable. + +```python +while True: + options = SubscriptionWorkerOptions(subscription_name) + + # Allow a downtime of up to 2 hours, and wait 2 minutes before reconnecting + options.max_erroneous_period = timedelta(hours=2) + options.time_to_wait_before_connection_retry = timedelta(minutes=2) + + subscription_worker = store.subscriptions.get_subscription_worker(options, Order) + + try: + # Log any exception that occurs during processing + subscription_worker.add_on_subscription_connection_retry( + lambda exception: logger.error( + f"Error during subscription processing: {subscription_name}", exc_info=exception + ) + ) + + def _process_documents_callback(batch): + for item in batch.items: + # Forcefully stop processing for a specific case, + # and raise an exception to let external logic handle it + if item.result.company == "companies/2-A": + # The exception raised here will be wrapped by `SubscriberErrorException` + raise UnsupportedCompanyException( + "Company Id can't be 'companies/2-A', you must fix this" + ) + process_order(item.result) + + # Block on the run future so processing exceptions surface here + subscription_worker.run(_process_documents_callback).result() + + # run completes normally if you have disposed the subscription + return + except Exception as e: + logger.error(f"Failure in subscription: {subscription_name}", exc_info=e) + exception_type = type(e) + + # The following exceptions are not recoverable + if exception_type in ( + DatabaseDoesNotExistException, + SubscriptionDoesNotExistException, + SubscriptionInvalidStateException, + AuthorizationException, + ): + raise + + if exception_type is SubscriptionClosedException: + # Subscription probably closed explicitly by admin + return + + if exception_type is SubscriberErrorException: + # For UnsupportedCompanyException we want to raise, otherwise continue processing + if e.args[1] is not None and type(e.args[1]) is UnsupportedCompanyException: + raise + continue + + # Handle this depending on the subscription opening strategy + if exception_type is SubscriptionInUseException: + continue + + return + finally: + subscription_worker.close(False) +``` + + + + + +### Example: Setting the batch size + +In this example, the worker sets the maximum number of documents the server sends in each batch. + +```python +worker_w_batch = store.subscriptions.get_subscription_worker( + SubscriptionWorkerOptions(subscription_name, max_docs_per_batch=20), Order +) + +# Pass a function that takes a SubscriptionBatch[Order], with your logic in it +worker_w_batch.run(lambda batch: ...) +``` + + + + + +### Example: Processing documents in a session + +In this example, the subscription sends to the worker `Orders` that have no shipping date. +The worker sets each document's `shipped_at` field and saves it back through a session. + + +Open the session using `batch.open_session`, not `store.open_session`. + + +```python +# Create the subscription task on the server: +subscription_name = store.subscriptions.create_for_options( + SubscriptionCreationOptions(query="from Orders as o where o.ShippedAt = null") +) + +# Create the subscription worker that will consume the documents: +subscription_worker = store.subscriptions.get_subscription_worker_by_name(subscription_name, Order) + +def _transfer_order_callback(batch): + # Open a session using 'batch.open_session' + with batch.open_session() as session: + for order in (item.result for item in batch.items): + transfer_order_to_shipment_company(order) # call your custom method + order.shipped_at = datetime.utcnow() # update the document field + + # Save the updated Order documents + session.save_changes() + +subscription_worker.run(_transfer_order_callback) +``` + + + + + +### Example: Processing dynamic objects + +In this example, the subscription projects each `Order` into a dynamic shape, and the worker processes the +dynamic objects it receives. + +```python +# Create the subscription task on the server: +subscription_name = "My dynamic subscription" +store.subscriptions.create_for_class( + Order, + SubscriptionCreationOptions( + subscription_name, + query=""" + from Orders as o + select { + dynamic_field_1: "Company: " + o.Company + " Employee: " + o.Employee + } + """, + ), +) + +# Create the subscription worker that will consume the documents: +subscription_worker = store.subscriptions.get_subscription_worker_by_name(subscription_name) + +def _raise_notification_callback(batch): + for item in batch.items: + # Access the dynamic field in the document + raise_notification(item.result.dynamic_field_1) # call your custom method + +subscription_worker.run(_raise_notification_callback) +``` + + + + + +### Example: End when no documents are left + +In this example, the worker runs until no new documents remain, which suits ad-hoc, single-use processing where +you need every matching document processed once. +Set `close_when_no_docs_left` to `True`; the worker then throws a `SubscriptionClosedException` when +it finishes. + +```python +high_value_orders_worker = store.subscriptions.get_subscription_worker( + SubscriptionWorkerOptions( + subscription_name, + # Stop when there are no more documents left to send, + # throwing a SubscriptionClosedException when it finishes + close_when_no_docs_left=True, + ), + OrderAndCompany, +) + +try: + def _subscription_batch_callback(batch): + for item in batch.items: + send_thank_you_note_to_company(item.result) # call your custom method + + high_value_orders_worker.run(_subscription_batch_callback).result() +except SubscriptionClosedException: + # That's expected, no more documents to process + ... +``` + + + + + +### Example: Failing over to another node + +With the `WAIT_FOR_FREE` strategy, any available node can create a worker, and another node takes +over the connection if the worker fails. + +```python +worker = store.subscriptions.get_subscription_worker( + SubscriptionWorkerOptions(subscription_name, strategy=SubscriptionOpeningStrategy.WAIT_FOR_FREE), + Order, +) +``` + + + + + +### Example: Primary and secondary workers + +In this example, the two workers form an active-standby pair: the primary, using `TAKE_OVER`, establishes the +connection; the secondary, using `WAIT_FOR_FREE`, waits to take over if the primary fails. + +The primary worker: + +```python +primary_worker = store.subscriptions.get_subscription_worker( + SubscriptionWorkerOptions(subscription_name, strategy=SubscriptionOpeningStrategy.TAKE_OVER), + Order, +) + +while True: + try: + primary_worker.run(lambda batch: ...).result() # your logic + except Exception: + ... # retry +``` + +
+ +The secondary worker: + +```python +secondary_worker = store.subscriptions.get_subscription_worker( + SubscriptionWorkerOptions(subscription_name, strategy=SubscriptionOpeningStrategy.WAIT_FOR_FREE), + Order, +) + +while True: + try: + secondary_worker.run(lambda batch: ...).result() # your logic + except Exception: + ... # retry +``` + + + + + + + + + +### Methods + + + + +Create a subscription worker through `store.subscriptions`. + +```python +def get_subscription_worker(self, options: SubscriptionWorkerOptions, + object_type: Optional[Type[_T]] = None, + database: Optional[str] = None) -> SubscriptionWorker[_T]: ... + +def get_subscription_worker_by_name(self, subscription_name: Optional[str] = None, + object_type: Optional[Type[_T]] = None, + database: Optional[str] = None) -> SubscriptionWorker[_T]: ... +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **options** | `SubscriptionWorkerOptions` | Options that affect how the worker interacts with the subscription. They do not change the subscription's own definition. See the [Classes](../../data-subscriptions/consuming-subscription.mdx#classes) below. | +| **subscription_name** | `str` | The name of the subscription the worker connects to. | +| **object_type** | `Type[_T]` | The document class the worker deserializes items into. | +| **database** | `str` | The database where the subscription task resides. If `None`, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `SubscriptionWorker` | The created worker. It is idle until you call `run`. | + + + + +Start the worker's batch processing. Pass the function that processes each received batch. + +```python +def run(self, process_documents: Optional[Callable[[SubscriptionBatch[_T]], Any]]) -> Future: ... +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **process_documents** | `Callable[[SubscriptionBatch[_T]], Any]` | Function for batch processing. | + +| Return value | | +|-----------|-------------| +| `Future` | A future that stays alive while the worker is processing or trying to. If processing is aborted, the future completes with an exception. | + + + + + + + + +### Classes + + + + +When you create a worker using `SubscriptionWorkerOptions`, the only required property is +`subscription_name`; the rest fall back to their defaults. + +| Property | Type | Description | +|----------|------|-------------| +| **subscription_name** | `str` | The name of the subscription the worker connects to. | +| **max_docs_per_batch** | `int` | The maximum number of documents the server tries to send in a batch. If it finds fewer, it sends what it has without waiting. Default: 4096. | +| **send_buffer_size** | `int` | The size of the TCP socket buffer used for sending data. Default: 32,768 bytes (32 KiB). | +| **receive_buffer_size** | `int` | The size of the TCP socket buffer used for receiving data. Default: 32,768 bytes (32 KiB). | +| **ignore_subscriber_errors** | `bool` | Whether processing continues when your batch-handling code throws. `True` continues; `False` (default) aborts. | +| **close_when_no_docs_left** | `bool` | Whether the connection closes when no new documents remain. `True` processes all available documents and then throws a `SubscriptionClosedException` (useful for one-time processing); `False` (default) keeps the worker waiting for new documents. | +| **time_to_wait_before_connection_retry** | `timedelta` | The wait before each reconnect attempt after a non-aborting failure. Default: 5 seconds. | +| **max_erroneous_period** | `timedelta` | The maximum time the connection may stay in an erroneous state before it is terminated. Default: 5 minutes. | +| **strategy** | `SubscriptionOpeningStrategy` | How the server handles this worker's connection attempt. Default: `OPEN_IF_FREE`. | + + + + +The strategy that decides whether a worker may connect to a subscription. See +[Worker strategies](../../data-subscriptions/consuming-subscription.mdx#worker-strategies). + +```python +class SubscriptionOpeningStrategy(Enum): + # Connect if no other worker is connected + OPEN_IF_FREE = "OpenIfFree" + + # Take over the connection + TAKE_OVER = "TakeOver" + + # Wait for the currently connected worker to disconnect + WAIT_FOR_FREE = "WaitForFree" +``` + + + + + + + +The batch passed to your `run` function. + +| Property | Type | Description | +|----------|------|-------------| +| **items** | `Item[]` | The items in the batch. | +| **number_of_items_in_batch** | `int` | The number of items in the batch. | + +| Method | Return value | Description | +|--------|--------------|-------------| +| **open_session()** | `DocumentSession` | Opens a session that tracks all items and their included items in the current batch. | + + +As long as no exception occurs, the worker keeps addressing the server it received the first +batch from. +If it cannot reach that node, it fails over to another node from the session's topology, see +[load balancing](../../client-api/configuration/load-balance/overview.mdx). The node it reaches +then tells it which node currently handles data subscriptions. + + + + + +A single item in a subscription batch. + +| Property | Type | Description | +|----------|------|-------------| +| **result** | `_T` | The batch item. | +| **key** | `str` | The document ID of the item's underlying document. | +| **change_vector** | `str` | The change vector of the item's underlying document. | +| **is_projection** | `bool` | `True` when the item is a projection produced by the subscription query rather than a full document. | +| **is_revision** | `bool` | `True` when the item is a document revision, for a subscription that streams revisions. | +| **raw_result** | `Dict` | The item before deserialization. | +| **raw_metadata** | `Dict` | The metadata of the item's underlying document. | +| **metadata** | `MetadataAsDictionary` | The item's underlying metadata values. | + + +Use these values only within the subscription's `run` function. Using them outside that scope may +cause unexpected behavior. + + + + + + + + +The worker returned by `get_subscription_worker`. + +**Methods** + +| Method | Return type | Description | +|--------|-------------|-------------| +| **close(wait_for_subscription_task=True)** | `None` | Aborts the worker, optionally waiting for the future returned by `run` to finish. | +| **run(process_documents)** | `Future` | Starts the worker's batch processing. See [run](../../data-subscriptions/consuming-subscription.mdx#methods) above. | + +**Events** + +| Event | Type | Description | +|-------|------|-------------| +| **add_on_subscription_connection_retry** | `Callable[[Exception], None]` | Fires when the worker tries to reconnect after a failure. Receives the exception that interrupted processing. | +| **after_acknowledgment** | `Callable[[SubscriptionBatch[_T]], None]` | Fires after the server acknowledges the progress of a processed batch. | + +**Properties** + +| Property | Type | Description | +|----------|------|-------------| +| **current_node_tag** | `str` | The node tag of the RavenDB server currently handling the subscription. | +| **subscription_name** | `str` | The name of the subscription being processed. | + + + + + + + diff --git a/docs/data-subscriptions/content/_maintenance-operations-csharp.mdx b/docs/data-subscriptions/content/_maintenance-operations-csharp.mdx new file mode 100644 index 0000000000..d8d952b4ed --- /dev/null +++ b/docs/data-subscriptions/content/_maintenance-operations-csharp.mdx @@ -0,0 +1,169 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* After creating a subscription, your client can manage it using the document store's `Subscriptions` property, of type `DocumentSubscriptions`. +* You can enable or disable a subscription, read its state, delete it, or drop worker connections. + +* In this article: + * [Deleting a subscription](../../data-subscriptions/maintenance-operations.mdx#deleting-a-subscription) + * [Enabling and disabling a subscription](../../data-subscriptions/maintenance-operations.mdx#enabling-and-disabling-a-subscription) + * [Dropping a connection](../../data-subscriptions/maintenance-operations.mdx#dropping-a-connection) + * [Listing subscriptions](../../data-subscriptions/maintenance-operations.mdx#listing-subscriptions) + * [Getting a subscription state](../../data-subscriptions/maintenance-operations.mdx#getting-a-subscription-state) + * [Updating a subscription](../../data-subscriptions/maintenance-operations.mdx#updating-a-subscription) + + + + + +Removing subscriptions from the database can be helpful when, for example, many single-use subscriptions +have accumulated and managing them becomes hard. + +```csharp +void Delete(string name, string database = null); +Task DeleteAsync(string name, string database = null, CancellationToken token = default); +``` + +
+ +Usage: + +```csharp +store.Subscriptions.Delete(subscriptionName); +``` + + + + + +Disable a subscription to stop it from accepting worker connections, or enable it to resume. +A disabled subscription keeps its definition and progress; only its ability to connect is shut down. + +```csharp +void Disable(string name, string database = null); +Task DisableAsync(string name, string database = null, CancellationToken token = default); + +void Enable(string name, string database = null); +Task EnableAsync(string name, string database = null, CancellationToken token = default); +``` + +
+ +Usage: + +```csharp +store.Subscriptions.Disable(subscriptionName); +store.Subscriptions.Enable(subscriptionName); +``` + + + + + +Drop a subscription's active connections using `DropConnection`, passing the subscription name. +Dropping a connection ends the worker's `Run` task, but leaves the subscription enabled. +Once dropped, the worker will not reconnect on its own. To resume, run it again. +To stop the subscription from accepting workers, +[disable it](../../data-subscriptions/maintenance-operations.mdx#enabling-and-disabling-a-subscription). + +```csharp +void DropConnection(string name, string database = null); +Task DropConnectionAsync(string name, string database = null, CancellationToken token = default); +``` + +
+ +Usage: + +```csharp +store.Subscriptions.DropConnection(subscriptionName); +``` + + +To disconnect a specific concurrent worker, use `DropSubscriptionWorker`. +See [Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx#dropping-a-worker). + + + + + + +Retrieve the database's subscriptions using `GetSubscriptions`. +The method returns a `List`, one entry per subscription, each holding the state +described in [Getting a subscription state](../../data-subscriptions/maintenance-operations.mdx#getting-a-subscription-state) below. +Use `start` to set how many subscriptions to skip and `take` to set the maximum number to return. + +```csharp +List GetSubscriptions(int start, int take, string database = null); +Task> GetSubscriptionsAsync(int start, int take, + string database = null, CancellationToken token = default); +``` + +
+ +Usage: + +```csharp +// List the first 10 subscriptions +List firstSubscriptions = store.Subscriptions.GetSubscriptions(0, 10); + +// List all subscriptions +List allSubscriptions = store.Subscriptions.GetSubscriptions(0, int.MaxValue); +``` + + + + + +To retrieve the state of a single subscription from the server use `GetSubscriptionState`, passing it the subscription's name. +The returned `SubscriptionState` lets you inspect the subscription. For example, read +`LastBatchAckTime` to see when the server last acknowledged progress, +`ChangeVectorForNextBatchStartingPoint` to see where it will resume, or `Disabled` to check whether +it currently accepts workers. + +```csharp +SubscriptionState GetSubscriptionState(string subscriptionName, string database = null); +Task GetSubscriptionStateAsync(string subscriptionName, + string database = null, CancellationToken token = default); +``` + +
+ +Usage: + +```csharp +SubscriptionState subscriptionState = store.Subscriptions.GetSubscriptionState(subscriptionName); +``` + +
+ +Its properties: + +| Property | Type | Description | +|----------|------|-------------| +| **Query** | `string` | The subscription's RQL query. | +| **SubscriptionName** | `string` | The subscription's name, which is also its unique identifier. | +| **SubscriptionId** | `long` | The subscription's internal identifier, assigned by the cluster at creation. | +| **MentorNode** | `string` | The mentor node, if one was set manually. | +| **NodeTag** | `string` | The tag of the server node processing the subscription. | +| **ChangeVectorForNextBatchStartingPoint** | `string` | The change vector from which the subscription will begin sending documents to the worker. Updated on batch acknowledgment, and can also be set manually. | +| **LastBatchAckTime** | `DateTime?` | The last time the server acknowledged progress on the subscription's documents. | +| **LastClientConnectionTime** | `DateTime?` | The last time a client connected (retained after the connection ends). | +| **Disabled** | `bool` | When `true`, the subscription does not allow workers to connect. | +| **ArchivedDataProcessingBehavior** | `ArchivedDataProcessingBehavior?` | Whether archived documents are included in the subscription. | + + + + + +Change the definition of an existing subscription using `Update` / `UpdateAsync`. +See [Updating a subscription](../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#updating-a-subscription) +on the Creating a data subscription page. + + diff --git a/docs/data-subscriptions/content/_maintenance-operations-java.mdx b/docs/data-subscriptions/content/_maintenance-operations-java.mdx new file mode 100644 index 0000000000..287684c37c --- /dev/null +++ b/docs/data-subscriptions/content/_maintenance-operations-java.mdx @@ -0,0 +1,179 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* After creating a subscription, your client can manage it using the document store's `subscriptions()` method, which returns a `DocumentSubscriptions` instance. +* You can enable or disable a subscription, read its state, delete it, or drop worker connections. + +* In this article: + * [Deleting a subscription](../../data-subscriptions/maintenance-operations.mdx#deleting-a-subscription) + * [Enabling and disabling a subscription](../../data-subscriptions/maintenance-operations.mdx#enabling-and-disabling-a-subscription) + * [Dropping a connection](../../data-subscriptions/maintenance-operations.mdx#dropping-a-connection) + * [Listing subscriptions](../../data-subscriptions/maintenance-operations.mdx#listing-subscriptions) + * [Getting a subscription state](../../data-subscriptions/maintenance-operations.mdx#getting-a-subscription-state) + * [Updating a subscription](../../data-subscriptions/maintenance-operations.mdx#updating-a-subscription) + + + + + +Removing subscriptions from the database can be helpful when, for example, many single-use subscriptions +have accumulated and managing them becomes hard. + +```java +void delete(String name); +void delete(String name, String database); +``` + +
+ +Usage: + +```java +store.subscriptions().delete(subscriptionName); +``` + + + + + +Disable a subscription to stop it from accepting worker connections, or enable it to resume. +A disabled subscription keeps its definition and progress; only its ability to connect is shut down. + +```java +void disable(String name); +void disable(String name, String database); + +void enable(String name); +void enable(String name, String database); +``` + +
+ +Usage: + +```java +store.subscriptions().disable(subscriptionName); +store.subscriptions().enable(subscriptionName); +``` + + + + + +Drop a subscription's active connections using `dropConnection`, passing the subscription name. +Dropping a connection ends the worker, but leaves the subscription enabled. +Once dropped, the worker will not reconnect on its own. To resume, run it again. +To stop the subscription from accepting workers, +[disable it](../../data-subscriptions/maintenance-operations.mdx#enabling-and-disabling-a-subscription). + +```java +void dropConnection(String name); +void dropConnection(String name, String database); +``` + +
+ +Usage: + +```java +store.subscriptions().dropConnection(subscriptionName); +``` + + + + + +Retrieve the database's subscriptions using `getSubscriptions`. +The method returns a `List`, one entry per subscription, each holding the state +described in [Getting a subscription state](../../data-subscriptions/maintenance-operations.mdx#getting-a-subscription-state) below. +Use `start` to set how many subscriptions to skip and `take` to set the maximum number to return. + +```java +List getSubscriptions(int start, int take); +List getSubscriptions(int start, int take, String database); +``` + +
+ +Usage: + +```java +// List the first 10 subscriptions +List firstSubscriptions = store.subscriptions().getSubscriptions(0, 10); + +// List all subscriptions +List allSubscriptions = store.subscriptions().getSubscriptions(0, Integer.MAX_VALUE); +``` + + + + + +To retrieve the state of a single subscription from the server use `getSubscriptionState`, passing it the subscription's name. +The returned `SubscriptionState` lets you inspect the subscription. For example, read +`lastBatchAckTime` to see when the server last acknowledged progress, +`changeVectorForNextBatchStartingPoint` to see where it will resume, or `disabled` to check whether +it currently accepts workers. + +```java +SubscriptionState getSubscriptionState(String subscriptionName); +SubscriptionState getSubscriptionState(String subscriptionName, String database); +``` + +
+ +Usage: + +```java +SubscriptionState subscriptionState = store.subscriptions().getSubscriptionState(subscriptionName); +``` + +
+ +Its properties: + +| Property | Type | Description | +|----------|------|-------------| +| **query** | `String` | The subscription's RQL query. | +| **subscriptionName** | `String` | The subscription's name, which is also its unique identifier. | +| **subscriptionId** | `long` | The subscription's internal identifier, assigned by the cluster at creation. | +| **mentorNode** | `String` | The mentor node, if one was set manually. | +| **nodeTag** | `String` | The tag of the server node processing the subscription. | +| **changeVectorForNextBatchStartingPoint** | `String` | The change vector from which the subscription will begin sending documents to the worker. Updated on batch acknowledgment, and can also be set manually. | +| **lastBatchAckTime** | `Date` | The last time the server acknowledged progress on the subscription's documents. | +| **lastClientConnectionTime** | `Date` | The last time a client connected (retained after the connection ends). | +| **disabled** | `boolean` | When `true`, the subscription does not allow workers to connect. | +| **archivedDataProcessingBehavior** | `ArchivedDataProcessingBehavior` | Whether archived documents are included in the subscription. | + + + + + +Change the definition of an existing subscription using `update`, passing a `SubscriptionUpdateOptions` +that identifies the subscription by name (or by the id the server assigned it at creation) and carries +the new definition. + +```java +String update(SubscriptionUpdateOptions options); +String update(SubscriptionUpdateOptions options, String database); +``` + +
+ +Usage: + +```java +SubscriptionUpdateOptions options = new SubscriptionUpdateOptions(); +options.setName(subscriptionName); +options.setQuery("from Orders where Total > 100"); + +store.subscriptions().update(options); +``` + + diff --git a/docs/data-subscriptions/content/_maintenance-operations-nodejs.mdx b/docs/data-subscriptions/content/_maintenance-operations-nodejs.mdx new file mode 100644 index 0000000000..c212a86a6e --- /dev/null +++ b/docs/data-subscriptions/content/_maintenance-operations-nodejs.mdx @@ -0,0 +1,170 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* After creating a subscription, your client can manage it using the document store's `subscriptions` property, of type `DocumentSubscriptions`. +* You can enable or disable a subscription, read its state, delete it, or drop worker connections. + +* In this article: + * [Deleting a subscription](../../data-subscriptions/maintenance-operations.mdx#deleting-a-subscription) + * [Enabling and disabling a subscription](../../data-subscriptions/maintenance-operations.mdx#enabling-and-disabling-a-subscription) + * [Dropping a connection](../../data-subscriptions/maintenance-operations.mdx#dropping-a-connection) + * [Listing subscriptions](../../data-subscriptions/maintenance-operations.mdx#listing-subscriptions) + * [Getting a subscription state](../../data-subscriptions/maintenance-operations.mdx#getting-a-subscription-state) + * [Updating a subscription](../../data-subscriptions/maintenance-operations.mdx#updating-a-subscription) + + + + + +Removing subscriptions from the database can be helpful when, for example, many single-use subscriptions +have accumulated and managing them becomes hard. + +```js +// Overloads: +delete(name); +delete(name, database); +``` + +
+ +Usage: + +```js +await documentStore.subscriptions.delete(subscriptionName); +``` + + + + + +Disable a subscription to stop it from accepting worker connections, or enable it to resume. +A disabled subscription keeps its definition and progress; only its ability to connect is shut down. + +```js +// Overloads: +disable(name); +disable(name, database); + +enable(name); +enable(name, database); +``` + +
+ +Usage: + +```js +await documentStore.subscriptions.disable(subscriptionName); +await documentStore.subscriptions.enable(subscriptionName); +``` + + + + + +Drop a subscription's active connections using `dropConnection`, passing the subscription name. +Dropping a connection ends the worker, but leaves the subscription enabled. +Once dropped, the worker will not reconnect on its own. To resume, run it again. +To stop the subscription from accepting workers, +[disable it](../../data-subscriptions/maintenance-operations.mdx#enabling-and-disabling-a-subscription). + +```js +// Overloads: +dropConnection(name); +dropConnection(name, database); +``` + +
+ +Usage: + +```js +await documentStore.subscriptions.dropConnection(subscriptionName); +``` + + +To disconnect a specific concurrent worker, use `dropSubscriptionWorker`. +See [Concurrent subscriptions](../../data-subscriptions/concurrent-subscriptions.mdx#dropping-a-worker). + + + + + + +Retrieve the database's subscriptions using `getSubscriptions`. +The method returns an array of `SubscriptionState`, one entry per subscription, each holding the state +described in [Getting a subscription state](../../data-subscriptions/maintenance-operations.mdx#getting-a-subscription-state) below. +Use `start` to set how many subscriptions to skip and `take` to set the maximum number to return. + +```js +// Overloads: +getSubscriptions(start, take); +getSubscriptions(start, take, database); +``` + +
+ +Usage: + +```js +// List the first 10 subscriptions +const firstSubscriptions = await documentStore.subscriptions.getSubscriptions(0, 10); +``` + + + + + +To retrieve the state of a single subscription from the server use `getSubscriptionState`, passing it the subscription's name. +The returned `SubscriptionState` lets you inspect the subscription. For example, read +`lastBatchAckTime` to see when the server last acknowledged progress, +`changeVectorForNextBatchStartingPoint` to see where it will resume, or `disabled` to check whether +it currently accepts workers. + +```js +// Overloads: +getSubscriptionState(subscriptionName); +getSubscriptionState(subscriptionName, database); +``` + +
+ +Usage: + +```js +const subscriptionState = + await documentStore.subscriptions.getSubscriptionState(subscriptionName); +``` + +
+ +Its properties: + +| Property | Type | Description | +|----------|------|-------------| +| **query** | `string` | The subscription's RQL query. | +| **subscriptionName** | `string` | The subscription's name, which is also its unique identifier. | +| **subscriptionId** | `number` | The subscription's internal identifier, assigned by the cluster at creation. | +| **mentorNode** | `string` | The mentor node, if one was set manually. | +| **nodeTag** | `string` | The tag of the server node processing the subscription. | +| **changeVectorForNextBatchStartingPoint** | `string` | The change vector from which the subscription will begin sending documents to the worker. Updated on batch acknowledgment, and can also be set manually. | +| **lastBatchAckTime** | `string` | The last time the server acknowledged progress on the subscription's documents. | +| **lastClientConnectionTime** | `string` | The last time a client connected (retained after the connection ends). | +| **disabled** | `boolean` | When `true`, the subscription does not allow workers to connect. | +| **archivedDataProcessingBehavior** | `ArchivedDataProcessingBehavior` | Whether archived documents are included in the subscription. | + + + + + +Change the definition of an existing subscription using `update`. +See [Updating a subscription](../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#updating-a-subscription) +on the Creating a data subscription page. + + diff --git a/docs/data-subscriptions/content/_revisions-support-csharp.mdx b/docs/data-subscriptions/content/_revisions-support-csharp.mdx new file mode 100644 index 0000000000..806c50c08a --- /dev/null +++ b/docs/data-subscriptions/content/_revisions-support-csharp.mdx @@ -0,0 +1,353 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* When the [Revisions feature](../../document-extensions/revisions/overview.mdx) is enabled, a + revision is created with each change made in a document. + Each revision is a snapshot of the document at the time of the change. Over time, a complete + audit trail is formed. + +* A data subscription can subscribe not only to documents but also to their revisions, allowing workers + to track the changes made in documents over time. + +* To subscribe to revisions, use [Revision<T>](../../data-subscriptions/revisions-support.mdx#creating-and-consuming-a-revisions-subscription) + as the subscription's item type, or add `(Revisions = true)` to its RQL query. + +* In this article: + * [Regular subscription vs revisions subscription](../../data-subscriptions/revisions-support.mdx#regular-subscription-vs-revisions-subscription) + * [Revisions processing order](../../data-subscriptions/revisions-support.mdx#revisions-processing-order) + * [Creating and consuming a revisions subscription](../../data-subscriptions/revisions-support.mdx#creating-and-consuming-a-revisions-subscription) + * [Filtering revisions](../../data-subscriptions/revisions-support.mdx#filtering-revisions) + * [Projecting fields from revisions](../../data-subscriptions/revisions-support.mdx#projecting-fields-from-revisions) + * [Syntax](../../data-subscriptions/revisions-support.mdx#syntax) + + + + + +A subscription can target a collection's documents or their revisions. The two behave differently +in what they process, what the server-side query can see, and what reaches the worker. + + + +### Regular subscription + +* **Processed items** + The subscription processes documents from the defined collection. Only the current version of + each document is processed, even if the document has revisions. +* **Query access scope** + The subscription query running on the server has access only to the current version of each document. +* **Data sent to the worker** + Each item in the batch contains a single document, or a projection of it, as defined in the subscription. + + + + + +### Revisions subscription + +* **Processed items** + The subscription processes all revisions of documents from the defined collection, including + revisions of deleted documents from the revisions bin if they have not been purged. +* **Query access scope** + For each revision, the subscription query running on the server has access to both the revision + being processed and its preceding revision, so it can compare the two revisions and act on what + changed between them. +* **Data sent to the worker** + Unless the query [projects specific fields](../../data-subscriptions/revisions-support.mdx#projecting-fields-from-revisions), + each item in the batch contains both the processed revision (`Result.Current`) and its preceding + revision (`Result.Previous`). For a document's first revision, `Result.Previous` is `null`. + + + + + +* For a revisions subscription to work, [Revisions must be configured](../../document-extensions/revisions/overview.mdx#defining-a-revisions-configuration) + and enabled for the collection the subscription targets. + +* A document that has no revisions is not processed, so make sure your revisions configuration does + not purge revisions before the subscription has had a chance to process them. + + + + + + + +In a revisions subscription, the worker receives revisions in consecutive pairs: each revision +together with the one immediately before it. +This gives the worker, and the subscription query, the document's state both before and after every +change, so they can detect what changed and act on it, for example to filter or project by the difference. + +For example, take this `User` document: + +```json +{ + "Name": "James", + "Age": "21" +} +``` + +
+ +Suppose its `Age` field is then updated twice, first to `22` and then to `23`. The worker receives +three batch items, each pairing a revision with the one before it: + +| Batch item | Previous | Current | +|------------|----------|---------| +| item #1 | `null` | `{ Name: "James", Age: "21" }` | +| item #2 | `{ Name: "James", Age: "21" }` | `{ Name: "James", Age: "22" }` | +| item #3 | `{ Name: "James", Age: "22" }` | `{ Name: "James", Age: "23" }` | + +The first item pairs the document's creation with a `null` previous. Each later item carries the +earlier revision as `Previous` and the next as `Current`. + + + + + +This basic revisions subscription delivers to the worker pairs of consecutive `Order` revisions. + +#### Example: Creating the subscription + + + + +```csharp +subscriptionName = store.Subscriptions.Create( + // Use Revision as the processed-item type, e.g. Revision + new SubscriptionCreationOptions>()); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + // Add (Revisions = true) to the subscription's RQL + Query = @"from Orders (Revisions = true)" +}); +``` + + + + +
+ +#### Example: Consuming the subscription + +```csharp +SubscriptionWorker> revisionsWorker = + store.Subscriptions.GetSubscriptionWorker>(subscriptionName); + +await revisionsWorker.Run((SubscriptionBatch> batch) => +{ + foreach (var item in batch.Items) + { + // The preceding revision (null for a document's first revision) + var previousRevision = item.Result.Previous; + + // The revision being processed + var currentRevision = item.Result.Current; + + ProcessOrderRevisions(previousRevision, currentRevision); // your processing logic + } +}); +``` + + + + + +This subscription sends to the worker only the revisions in which the order was shipped to Mexico. + +#### Example: Creating the subscription + + + + +```csharp +subscriptionName = store.Subscriptions.Create( + new SubscriptionCreationOptions>() + { + // Only revisions of orders shipped to Mexico are sent to the worker + Filter = revision => revision.Current.ShipTo.Country == "Mexico" + }); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"declare function isSentToMexico(doc) { + return doc.Current.ShipTo.Country == 'Mexico' + } + + from 'Orders' (Revisions = true) as doc + where isSentToMexico(doc) == true" +}); +``` + + + + +
+ +#### Example: Consuming the subscription + +```csharp +SubscriptionWorker> worker = + store.Subscriptions.GetSubscriptionWorker>(subscriptionName); + +await worker.Run(batch => +{ + foreach (var item in batch.Items) + { + Console.WriteLine($@" + This is a revision of document {item.Id}. + The order in this revision was shipped at {item.Result.Current.ShippedAt}."); + } +}); +``` + + + + + +This subscription filters the revisions and sends to the worker only the projected data rather than +the full revision pair. + +#### Example: Creating the subscription + + + + +```csharp +subscriptionName = store.Subscriptions.Create( + new SubscriptionCreationOptions>() + { + // Process only revisions whose revenue exceeds the preceding revision's by more than 2500 + Filter = revision => + revision.Previous != null && + revision.Current.Lines.Sum(x => x.PricePerUnit * x.Quantity) > + revision.Previous.Lines.Sum(x => x.PricePerUnit * x.Quantity) + 2500, + + // Project the fields sent to the worker + Projection = revision => new OrderRevenues() + { + PreviousRevenue = revision.Previous.Lines.Sum(x => x.PricePerUnit * x.Quantity), + CurrentRevenue = revision.Current.Lines.Sum(x => x.PricePerUnit * x.Quantity) + } + }); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"declare function isRevenueDeltaAboveThreshold(doc, threshold) { + return doc.Previous !== null && doc.Current.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0) > doc.Previous.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0) + threshold + } + + from 'Orders' (Revisions = true) as doc + where isRevenueDeltaAboveThreshold(doc, 2500) + + select { + PreviousRevenue: doc.Previous.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0), + + CurrentRevenue: doc.Current.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0) + }" +}); +``` + + + + +The projection targets this class: + +```csharp +public class OrderRevenues +{ + public decimal PreviousRevenue { get; set; } + public decimal CurrentRevenue { get; set; } +} +``` + +
+ +#### Example: Consuming the subscription + +Because the revision fields are projected into `OrderRevenues` in the subscription's definition, +each batch item has the shape of that projected class rather than the default `Result.Previous` and +`Result.Current` fields shown in the +[simple example](../../data-subscriptions/revisions-support.mdx#creating-and-consuming-a-revisions-subscription). + +```csharp +SubscriptionWorker revenuesComparisonWorker = + // Use the projected class type for the items the worker processes + store.Subscriptions.GetSubscriptionWorker(subscriptionName); + +await revenuesComparisonWorker.Run(batch => +{ + foreach (var item in batch.Items) + { + Console.WriteLine($@"Revenue for order {item.Id} + grew from {item.Result.PreviousRevenue} + to {item.Result.CurrentRevenue}"); + } +}); +``` + + + + + + + +### Classes + + + + +The processed-item type for a revisions subscription. Each batch item's `Result` is a +`Revision` holding the revision being processed and the one before it. + +```csharp +public sealed class Revision where T : class +{ + public T Previous; + public T Current; +} +``` + +
+ +| Field | Type | Description | +|-------|------|-------------| +| **Previous** | `T` | The preceding revision. `null` when `Current` is the document's first revision. | +| **Current** | `T` | The revision being processed. | + + + + + + + diff --git a/docs/data-subscriptions/content/_revisions-support-java.mdx b/docs/data-subscriptions/content/_revisions-support-java.mdx new file mode 100644 index 0000000000..067a61780d --- /dev/null +++ b/docs/data-subscriptions/content/_revisions-support-java.mdx @@ -0,0 +1,241 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* When the [Revisions feature](../../document-extensions/revisions/overview.mdx) is enabled, a + revision is created with each change made in a document. + Each revision is a snapshot of the document at the time of the change. Over time, a complete + audit trail is formed. + +* A data subscription can subscribe not only to documents but also to their revisions, allowing workers + to track the changes made in documents over time. + +* To subscribe to revisions, create it using `createForRevisions`, + or add `(Revisions = true)` to its RQL query. + +* In this article: + * [Regular subscription vs revisions subscription](../../data-subscriptions/revisions-support.mdx#regular-subscription-vs-revisions-subscription) + * [Revisions processing order](../../data-subscriptions/revisions-support.mdx#revisions-processing-order) + * [Creating and consuming a revisions subscription](../../data-subscriptions/revisions-support.mdx#creating-and-consuming-a-revisions-subscription) + * [Projecting fields from revisions](../../data-subscriptions/revisions-support.mdx#projecting-fields-from-revisions) + + + + + +A subscription can target a collection's documents or their revisions. The two behave differently +in what they process, what the server-side query can see, and what reaches the worker. + + + +### Regular subscription + +* **Processed items** + The subscription processes documents from the defined collection. Only the current version of + each document is processed, even if the document has revisions. +* **Query access scope** + The subscription query running on the server has access only to the current version of each document. +* **Data sent to the worker** + Each item in the batch contains a single document, or a projection of it, as defined in the subscription. + + + + + +### Revisions subscription + +* **Processed items** + The subscription processes all revisions of documents from the defined collection, including + revisions of deleted documents from the revisions bin if they have not been purged. +* **Query access scope** + For each revision, the subscription query running on the server has access to both the revision + being processed and its preceding revision, so it can compare the two revisions and act on what + changed between them. +* **Data sent to the worker** + Unless the query [projects specific fields](../../data-subscriptions/revisions-support.mdx#projecting-fields-from-revisions), + each item in the batch contains both the processed revision (`getResult().getCurrent()`) and its + preceding revision (`getResult().getPrevious()`). For a document's first revision, `getPrevious()` is `null`. + + + + + +* For a revisions subscription to work, [Revisions must be configured](../../document-extensions/revisions/overview.mdx#defining-a-revisions-configuration) + and enabled for the collection the subscription targets. + +* A document that has no revisions is not processed, so make sure your revisions configuration does + not purge revisions before the subscription has had a chance to process them. + + + + + + + +In a revisions subscription, the worker receives revisions in consecutive pairs: each revision +together with the one immediately before it. +This gives the worker, and the subscription query, the document's state both before and after every +change, so they can detect what changed and act on it, for example to filter or project by the difference. + +For example, take this `User` document: + +```json +{ + "Name": "James", + "Age": "21" +} +``` + +
+ +Suppose its `Age` field is then updated twice, first to `22` and then to `23`. The worker receives +three batch items, each pairing a revision with the one before it: + +| Batch item | Previous | Current | +|------------|----------|---------| +| item #1 | `null` | `{ Name: "James", Age: "21" }` | +| item #2 | `{ Name: "James", Age: "21" }` | `{ Name: "James", Age: "22" }` | +| item #3 | `{ Name: "James", Age: "22" }` | `{ Name: "James", Age: "23" }` | + +The first item pairs the document's creation with a `null` previous. Each later item carries the +earlier revision as `Previous` and the next as `Current`. + + + + + +This basic revisions subscription delivers to the worker pairs of consecutive `Order` revisions. + +#### Example: Creating the subscription + + + + +```java +String name = store.subscriptions().createForRevisions(Order.class); +``` + + + + +```java +SubscriptionCreationOptions options = new SubscriptionCreationOptions(); +options.setQuery("from Orders (Revisions = true)"); + +String name = store.subscriptions().createForRevisions(Order.class, options); +``` + + + + +
+ +#### Example: Consuming the subscription + +```java +SubscriptionWorker> revisionsWorker = store + .subscriptions().getSubscriptionWorkerForRevisions(Order.class, name); + +revisionsWorker.run(x -> { + for (SubscriptionBatch.Item> item : x.getItems()) { + // The preceding revision (null for a document's first revision) + Order previousRevision = item.getResult().getPrevious(); + + // The revision being processed + Order currentRevision = item.getResult().getCurrent(); + + processOrderRevisions(previousRevision, currentRevision); // your processing logic + } +}); +``` + + + + + +This subscription filters the revisions and sends to the worker only the projected data rather than +the full revision pair. + +#### Example: Creating the subscription + +```java +SubscriptionCreationOptions options = new SubscriptionCreationOptions(); +options.setQuery( + "declare function isRevenueDeltaAboveThreshold(doc, threshold) {\n" + + " return doc.Previous !== null && doc.Current.Lines.map(function(x) {\n" + + " return x.PricePerUnit * x.Quantity;\n" + + " }).reduce((a, b) => a + b, 0) > doc.Previous.Lines.map(function(x) {\n" + + " return x.PricePerUnit * x.Quantity;\n" + + " }).reduce((a, b) => a + b, 0) + threshold\n" + + "}\n" + + "from 'Orders' (Revisions = true) as doc\n" + + "where isRevenueDeltaAboveThreshold(doc, 2500)\n" + + "select {\n" + + " previousRevenue: doc.Previous.Lines.map(function(x) {\n" + + " return x.PricePerUnit * x.Quantity;\n" + + " }).reduce((a, b) => a + b, 0),\n" + + " currentRevenue: doc.Current.Lines.map(function(x) {\n" + + " return x.PricePerUnit * x.Quantity;\n" + + " }).reduce((a, b) => a + b, 0)\n" + + "}"); + +String name = store.subscriptions().create(options); +``` + +
+ +The projection targets this class: + +```java +public class OrderRevenues { + private double previousRevenue; + private double currentRevenue; + + public double getPreviousRevenue() { + return previousRevenue; + } + + public void setPreviousRevenue(double previousRevenue) { + this.previousRevenue = previousRevenue; + } + + public double getCurrentRevenue() { + return currentRevenue; + } + + public void setCurrentRevenue(double currentRevenue) { + this.currentRevenue = currentRevenue; + } +} +``` + +
+ +#### Example: Consuming the subscription + +Because the revision fields are projected into `OrderRevenues` in the subscription's definition, +each batch item has the shape of that projected class rather than the default `getPrevious()` and +`getCurrent()` fields shown in the +[simple example](../../data-subscriptions/revisions-support.mdx#creating-and-consuming-a-revisions-subscription). + +```java +SubscriptionWorker revenuesComparisonWorker = store + // Use the projected class type for the items the worker processes + .subscriptions().getSubscriptionWorker(OrderRevenues.class, name); + +revenuesComparisonWorker.run(x -> { + for (SubscriptionBatch.Item item : x.getItems()) { + OrderRevenues revenues = item.getResult(); + + System.out.printf("Revenue for order %s grew from %f to %f%n", + item.getId(), revenues.getPreviousRevenue(), revenues.getCurrentRevenue()); + } +}); +``` + + diff --git a/docs/data-subscriptions/content/_revisions-support-nodejs.mdx b/docs/data-subscriptions/content/_revisions-support-nodejs.mdx new file mode 100644 index 0000000000..d55cb5b694 --- /dev/null +++ b/docs/data-subscriptions/content/_revisions-support-nodejs.mdx @@ -0,0 +1,301 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* When the [Revisions feature](../../document-extensions/revisions/overview.mdx) is enabled, a + revision is created with each change made in a document. + Each revision is a snapshot of the document at the time of the change. Over time, a complete + audit trail is formed. + +* A data subscription can subscribe not only to documents but also to their revisions, allowing workers + to track the changes made in documents over time. + +* To subscribe to revisions, consume the subscription with `getSubscriptionWorkerForRevisions`, + or add `(Revisions = true)` to its RQL query. + +* In this article: + * [Regular subscription vs revisions subscription](../../data-subscriptions/revisions-support.mdx#regular-subscription-vs-revisions-subscription) + * [Revisions processing order](../../data-subscriptions/revisions-support.mdx#revisions-processing-order) + * [Creating and consuming a revisions subscription](../../data-subscriptions/revisions-support.mdx#creating-and-consuming-a-revisions-subscription) + * [Filtering revisions](../../data-subscriptions/revisions-support.mdx#filtering-revisions) + * [Projecting fields from revisions](../../data-subscriptions/revisions-support.mdx#projecting-fields-from-revisions) + + + + + +A subscription can target a collection's documents or their revisions. The two behave differently +in what they process, what the server-side query can see, and what reaches the worker. + + + +### Regular subscription + +* **Processed items** + The subscription processes documents from the defined collection. Only the current version of + each document is processed, even if the document has revisions. +* **Query access scope** + The subscription query running on the server has access only to the current version of each document. +* **Data sent to the worker** + Each item in the batch contains a single document, or a projection of it, as defined in the subscription. + + + + + +### Revisions subscription + +* **Processed items** + The subscription processes all revisions of documents from the defined collection, including + revisions of deleted documents from the revisions bin if they have not been purged. +* **Query access scope** + For each revision, the subscription query running on the server has access to both the revision + being processed and its preceding revision, so it can compare the two revisions and act on what + changed between them. +* **Data sent to the worker** + Unless the query [projects specific fields](../../data-subscriptions/revisions-support.mdx#projecting-fields-from-revisions), + each item in the batch contains both the processed revision (`result.current`) and its preceding + revision (`result.previous`). For a document's first revision, `result.previous` is `null`. + + + + + +* For a revisions subscription to work, [Revisions must be configured](../../document-extensions/revisions/overview.mdx#defining-a-revisions-configuration) + and enabled for the collection the subscription targets. + +* A document that has no revisions is not processed, so make sure your revisions configuration does + not purge revisions before the subscription has had a chance to process them. + + + + + + + +In a revisions subscription, the worker receives revisions in consecutive pairs: each revision +together with the one immediately before it. +This gives the worker, and the subscription query, the document's state both before and after every +change, so they can detect what changed and act on it, for example to filter or project by the difference. + +For example, take this `User` document: + +```json +{ + "Name": "James", + "Age": "21" +} +``` + +
+ +Suppose its `Age` field is then updated twice, first to `22` and then to `23`. The worker receives +three batch items, each pairing a revision with the one before it: + +| Batch item | Previous | Current | +|------------|----------|---------| +| item #1 | `null` | `{ Name: "James", Age: "21" }` | +| item #2 | `{ Name: "James", Age: "21" }` | `{ Name: "James", Age: "22" }` | +| item #3 | `{ Name: "James", Age: "22" }` | `{ Name: "James", Age: "23" }` | + +The first item pairs the document's creation with a `null` previous. Each later item carries the +earlier revision as `previous` and the next as `current`. + + + + + +This basic revisions subscription delivers to the worker pairs of consecutive `Order` revisions. + +#### Example: Creating the subscription + + + + +```js +const subscriptionName = await documentStore.subscriptions.createForRevisions({ + // Set the collection whose revisions are processed + documentType: Order +}); +``` + + + + +```js +const subscriptionName = await documentStore.subscriptions.create({ + // Add (Revisions = true) to the subscription's RQL + query: "from Orders (Revisions = true)" +}); +``` + + + + +
+ +#### Example: Consuming the subscription + +```js +const workerOptions = { subscriptionName }; + +const revisionsWorker = + // Use getSubscriptionWorkerForRevisions to receive revision pairs + documentStore.subscriptions.getSubscriptionWorkerForRevisions(workerOptions); + +revisionsWorker.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + // The preceding revision (null for a document's first revision) + const previousRevision = item.result.previous; + + // The revision being processed + const currentRevision = item.result.current; + + processOrderRevisions(previousRevision, currentRevision); // your processing logic + } + callback(); + } catch (err) { + callback(err); + } +}); +``` + + + + + +This subscription sends to the worker only the revisions in which the order was shipped to Mexico. + +#### Example: Creating the subscription + +```js +const subscriptionName = await documentStore.subscriptions.create({ + // Only revisions of orders shipped to Mexico are sent to the worker + query: `declare function isSentToMexico(doc) { + return doc.Current.ShipTo.Country == 'Mexico' + } + + from 'Orders' (Revisions = true) as doc + where isSentToMexico(doc) == true` +}); +``` + +
+ +#### Example: Consuming the subscription + +```js +const workerOptions = { subscriptionName }; + +const worker = + documentStore.subscriptions.getSubscriptionWorkerForRevisions(workerOptions); + +worker.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + console.log(` + This is a revision of document ${item.id}. + The order in this revision was shipped at ${item.result.current.ShippedAt}.`); + } + callback(); + } catch (err) { + callback(err); + } +}); +``` + + + + + +This subscription filters the revisions and sends to the worker only the projected data rather than +the full revision pair. + +#### Example: Creating the subscription + + + + +```js +const subscriptionName = await documentStore.subscriptions.create({ + // Process only revisions whose revenue exceeds the preceding revision's by more than 2500, + // and project the fields sent to the worker + query: `declare function isRevenueDeltaAboveThreshold(doc, threshold) { + return doc.Previous !== null && doc.Current.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0) > doc.Previous.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0) + threshold + } + + from 'Orders' (Revisions = true) as doc + where isRevenueDeltaAboveThreshold(doc, 2500) + + select { + previousRevenue: doc.Previous.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0), + + currentRevenue: doc.Current.Lines.map(function(x) { + return x.PricePerUnit * x.Quantity; + }).reduce((a, b) => a + b, 0) + }` +}); +``` + + + + +```js +class OrderRevenues { + constructor() { + this.previousRevenue; + this.currentRevenue; + } +} +``` + + + + +
+ +#### Example: Consuming the subscription + +Because the revision fields are projected into `OrderRevenues` in the subscription's definition, +each batch item has the shape of that projected class rather than the default `result.previous` and +`result.current` fields shown in the +[simple example](../../data-subscriptions/revisions-support.mdx#creating-and-consuming-a-revisions-subscription). + +```js +const workerOptions = { + subscriptionName, + // Use the projected class as the item type, so use getSubscriptionWorker (not ...ForRevisions) + documentType: OrderRevenues +}; + +const revenuesComparisonWorker = + documentStore.subscriptions.getSubscriptionWorker(workerOptions); + +revenuesComparisonWorker.on("batch", (batch, callback) => { + try { + for (const item of batch.items) { + console.log(` + Revenue for order ${item.id} + grew from ${item.result.previousRevenue} + to ${item.result.currentRevenue}`); + } + callback(); + } catch (err) { + callback(err); + } +}); +``` + + diff --git a/docs/data-subscriptions/creating-subscription/_category_.json b/docs/data-subscriptions/creating-subscription/_category_.json new file mode 100644 index 0000000000..981c15d76f --- /dev/null +++ b/docs/data-subscriptions/creating-subscription/_category_.json @@ -0,0 +1,4 @@ +{ + "position": 2, + "label": "Creating a data subscription" +} diff --git a/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_define-task.snagx b/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_define-task.snagx new file mode 100644 index 0000000000..cd4031dd2d Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_define-task.snagx differ diff --git a/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_ongoing-tasks-view.snagx b/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_ongoing-tasks-view.snagx new file mode 100644 index 0000000000..7c6a941dcf Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_ongoing-tasks-view.snagx differ diff --git a/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_test-subscription.snagx b/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_test-subscription.snagx new file mode 100644 index 0000000000..3ca9ac949a Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/snagit/subscriptions_test-subscription.snagx differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_archived-data-processing.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_archived-data-processing.png new file mode 100644 index 0000000000..a8258a27ed Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_archived-data-processing.png differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_define-task.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_define-task.png new file mode 100644 index 0000000000..ff8ffeb6f4 Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_define-task.png differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_ongoing-tasks-view.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_ongoing-tasks-view.png new file mode 100644 index 0000000000..cbab11bbe0 Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_ongoing-tasks-view.png differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_set-responsible-node.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_set-responsible-node.png new file mode 100644 index 0000000000..39c1ba4824 Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_set-responsible-node.png differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_set-responsible-node_pin-node.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_set-responsible-node_pin-node.png new file mode 100644 index 0000000000..ca2cb173e8 Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_set-responsible-node_pin-node.png differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_subscription-task-selection.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_subscription-task-selection.png new file mode 100644 index 0000000000..dea0c57b4f Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_subscription-task-selection.png differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_test-subscription.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_test-subscription.png new file mode 100644 index 0000000000..243c965a07 Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_test-subscription.png differ diff --git a/docs/data-subscriptions/creating-subscription/assets/subscriptions_use-defined-starting-point.png b/docs/data-subscriptions/creating-subscription/assets/subscriptions_use-defined-starting-point.png new file mode 100644 index 0000000000..6992e2b93f Binary files /dev/null and b/docs/data-subscriptions/creating-subscription/assets/subscriptions_use-defined-starting-point.png differ diff --git a/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-csharp.mdx b/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-csharp.mdx new file mode 100644 index 0000000000..86d90588bd --- /dev/null +++ b/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-csharp.mdx @@ -0,0 +1,931 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* A data subscription is a server-side task that sends documents matching a query to connected + clients ("workers"), in batches. +* This page explains how to create and update a subscription using the Client API. + You can also [create a subscription using Studio](../../../data-subscriptions/creating-subscription/creating-subscription_studio.mdx). +* A subscription's definition and its processing progress are kept on the cluster rather than on + a single node, and the cluster assigns one node to manage the task. + +* In this article: + * [Creating a subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#creating-a-subscription) + * [Subscription name](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-name) + * [Responsible node](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#responsible-node) + * [Updating a subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#updating-a-subscription) + * [Subscription query](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-query) + * [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) + * [Sending an entire collection](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-sending-an-entire-collection) + * [Filtering documents](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-documents) + * [Filtering with a regular expression](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-with-a-regular-expression) + * [Filtering and projecting fields](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-and-projecting-fields) + * [Projecting from a related document](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-projecting-from-a-related-document) + * [Including documents](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-documents) + * [Including counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters) + * [Including time series](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-time-series) + * [Creating a subscription via update](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-creating-a-subscription-via-update) + * [Updating an existing subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-updating-an-existing-subscription) + * [Syntax](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#syntax) + + + + + +To create a subscription, define which documents the server should send to the worker, then register +this definition with the server using the `Create` method on the document store's `Subscriptions` +property. + +You can define a subscription in three ways: + +* **A typed filter and projection** + Pass a [SubscriptionCreationOptions<T>](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) whose `Filter`, `Projection`, and + `Includes` expressions are written in C#. +* **A predicate** + Pass a lambda predicate directly to [Create<T>](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#methods). +* **A raw RQL (Raven Query Language) query** + Set the `Query` string on [SubscriptionCreationOptions](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes). + +The simplest definition sends an entire collection: + +```csharp +// The server sends every document in the 'Orders' collection +// to any client that connects to this subscription. +subscriptionName = store.Subscriptions.Create(); +``` + +
+ +For filtering, projections, includes, and revisions, see the [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) below. + + + + + +Every subscription has a name, which clients use to identify the subscription they want to +consume. +If you do not provide a name when creating the subscription, the server generates one for you. +To set your own: + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions +{ + // Set a custom name for the subscription + Name = "OrdersProcessingSubscription" +}); +``` + +
+ +Setting your own name is useful for long-running batch processes: a readable name is easier to +work with in code, and you can reuse the same name across environments, as long as the +subscription is created in advance. + + +A subscription name must be unique within its database. You cannot create two subscriptions with +the same name in the same database. + + + + + + +When a subscription is created, the cluster assigns one node to manage it. As long as this node is available, +it remains responsible for the subscription. If a worker connects to another node, that node will redirect it +to the responsible node. + +If the responsible node goes down and your license includes +[highly available tasks](../../../server/clustering/distribution/highly-available-tasks.mdx), +the task is reassigned to another node. +While the assigned node is online, it remains the one managing the subscription. + +To choose the responsible node yourself, set its tag in the `MentorNode` property: + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions +{ + // Make node D responsible for the subscription + MentorNode = "D" +}); +``` + +
+ +Choosing the node manually lets you pick a server based on its hardware, proximity to the +consuming clients, or other considerations. + + + + + +You can change an existing subscription's definition using the [Update](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#methods) method. +Pass a [SubscriptionUpdateOptions](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes), which extends the creation options with two +additional fields, `Id` and `CreateNew`: + +* **`Id`** targets the subscription by the ID the server assigned it at creation, instead of by + name. + Because the ID identifies the subscription independently of its name, you can use it to rename + a subscription: provide the ID and a new `Name`. +* **`CreateNew`** controls what happens when the subscription you are updating does not exist. + Set it to `true` to create the subscription instead, or leave it `false`, the default, to + throw an exception. + +```csharp +// Rename a subscription by targeting it with its Id +store.Subscriptions.Update(new SubscriptionUpdateOptions +{ + Id = subscriptionId, + Name = "NewSubscriptionName" +}); +``` + +
+ +For a full update example, see the [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) below. + + + + + +Every subscription is ultimately expressed as an RQL statement. +A subscription query can have these parts: + +* **A `declare` functions section**, as in ordinary RQL. + These functions can hold any JavaScript code, and support `load` and `include` operations. +* **A `from` clause** naming the document source, for example `from Orders`. + This clause can address only a collection; indexes are not supported. +* **A `where` clause** deciding the documents the server sends to the worker. + This clause supports RQL equality operators (`=`, `==`), plain JavaScript expressions, + and calls to declared functions, so you can express complex filtering. + Subscription RQL does not support RQL's search keywords. +* **A `select` clause** defining the projection. + This clause can call functions, allowing complex transformations. +* **An `include` clause** defining paths to include. + + +Although a subscription query has an RQL-like structure, it supports only the `declare`, +`select`, and `where` keywords; no other RQL keywords are supported. JavaScript ES5 syntax is +supported. For more details, see [JavaScript engine](../../../server/kb/javascript-engine.mdx). + + + + +Paths in a subscription query are handled as JavaScript expressions rather than as regular RQL +paths. +A query that in RQL would be written as: + +```rql +from Orders as o +where o.Lines[].Product = "products/1-A" +``` + +
+ +is written in a subscription query as follows: + +```rql +declare function filterLines(doc, productId) +{ + if (!!doc.Lines){ + return doc.Lines.filter(x => x.Product == productId).length > 0; + } + return false; +} + +from Orders as o +where filterLines(o, "products/1-A") +``` + + + +To create a subscription over document revisions, see +[Revisions support](../../../data-subscriptions/revisions-support.mdx). + + + + + + + +### Example: Sending an entire collection + +In this example, the subscription sends to the worker every document in the `Orders` collection, with no filter or projection. + + + + +```csharp +subscriptionName = store.Subscriptions.Create(); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = "from Orders" +}); +``` + + + + + + + + +### Example: Filtering documents + +In this example, the subscription sends to the worker only `Orders` whose total revenue exceeds 100. +The server evaluates the filter and sends just the matching documents. + + + + +```csharp +subscriptionName = store.Subscriptions.Create(x => + // Only documents matching this filter will be sent + x.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 100); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + from Orders as o + where getOrderLinesSum(o) > 100" +}); +``` + + + + + + + + +### Example: Filtering with a regular expression + +In this example, the subscription filters documents with a regular expression. +Use `Regex.IsMatch` inside the filter expression to match a field against a pattern on the +server, here `Orders` whose `ShipTo.City` starts with `New`. + + + + +```csharp +// Filter using Regex.IsMatch in the subscription's LINQ filter +subscriptionName = store.Subscriptions.Create(x => + Regex.IsMatch(x.ShipTo.City, "^New")); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"from Orders + where regex(ShipTo.City, '^New')" +}); +``` + + + + +
+ +Pass supported `RegexOptions` as a third argument: + +```csharp +// Filter with case-insensitive regex matching +subscriptionName = store.Subscriptions.Create(x => + Regex.IsMatch(x.ShipTo.City, "^new", RegexOptions.IgnoreCase)); +``` + +
+ + + +The following `RegexOptions` are supported: `RegexOptions.IgnoreCase`, `RegexOptions.Multiline`, +and `RegexOptions.Singleline`. +Other values, such as `Compiled` or `ExplicitCapture`, and the timeout overload are not +supported and throw a `NotSupportedException`. +The pattern must be a constant expression; it cannot be a value computed at runtime from +document fields. + + + + + + + +### Example: Filtering and projecting fields + +In this example, the subscription filters `Orders` by total revenue, and projects only two fields into each item +sent to the client: the document ID and the computed total. + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + // The subscription criteria: + Filter = x => x.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 100, + + // The object properties that will be sent for each matching document: + Projection = x => new + { + Id = x.Id, + Total = x.Lines.Sum(line => line.PricePerUnit * line.Quantity) + } +}); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + declare function projectOrder(doc) { + return { + Id: doc.Id, + Total: getOrderLinesSum(doc) + }; + } + + from Orders as o + where getOrderLinesSum(o) > 100 + select projectOrder(o)" +}); +``` + + + + + + + + +### Example: Projecting from a related document + +In this example, the subscription projects fields from each `Order` and also pulls data from a +[related document](../../../indexes/indexing-related-documents.mdx#what-are-related-documents), +loaded using `Load`. + + + + +```csharp +subscriptionName = store.Subscriptions.Create( + new SubscriptionCreationOptions() + { + // The subscription criteria: + Filter = x => x.Lines.Sum(line => line.PricePerUnit * line.Quantity) > 100, + + // The object properties that will be sent for each matching document: + Projection = x => new + { + Id = x.Id, + Total = x.Lines.Sum(line => line.PricePerUnit * line.Quantity), + ShipTo = x.ShipTo, + + // 'Load' the related Employee document and use its data in the projection + // Both Load calls resolve to a single server-side load + EmployeeName = RavenQuery.Load(x.Employee).FirstName + " " + + RavenQuery.Load(x.Employee).LastName + } + }); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + declare function projectOrder(doc) { + var employee = load(doc.Employee); + return { + Id: doc.Id, + Total: getOrderLinesSum(doc), + ShipTo: doc.ShipTo, + EmployeeName: employee.FirstName + ' ' + employee.LastName + }; + } + + from Orders as o + where getOrderLinesSum(o) > 100 + select projectOrder(o)" +}); +``` + + + + + + + + +### Example: Including documents + +In this example, the subscription sends every `Order` and includes the `Product` documents related to the order +in the same batch, so the consuming worker can later read each `Product` without an extra call to +the server. + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Includes = builder => builder + // The documents whose IDs are in each order line's 'Product' property + // will be included in the batch + .IncludeDocuments(x => x.Lines.Select(y => y.Product)) +}); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"from Orders include Lines[].Product" +}); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"declare function includeProducts(doc) { + let linesCount = doc.Lines.length; + + for (let i = 0; i < linesCount; i++) { + include(doc.Lines[i].Product); + } + + return doc; + } + + from Orders as o select includeProducts(o)" +}); +``` + + + + +
+ +Using the builder, assign an `ISubscriptionIncludeBuilder` to the `Includes` property of +`SubscriptionCreationOptions`. +The builder includes related documents through `IncludeDocuments`, and its methods can be +chained. + +In RQL, write the include in one of two ways: append the `include` keyword followed by the paths +to the fields holding the IDs to include (preferred, for both clarity and slightly better +performance), or call `include` inside a JavaScript function invoked from the `select` clause. + + +When you include documents alongside a projection, the `include` resolves paths against the +projected fields rather than the original document. + + + + + + +### Example: Including counters + +In this example, the subscription sends to the worker every `Order` and includes the values of the +named counters in the batch. + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Includes = builder => builder + // Values for the specified counters will be included in the batch + .IncludeCounters(new[] { "Pros", "Cons" }) +}); +``` + + + + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Query = @"from Orders include counters('Pros'), counters('Cons')" +}); +``` + + + + +
+ +`ISubscriptionIncludeBuilder` offers three methods for including counters: + +```csharp +// Include a single counter +ISubscriptionIncludeBuilder IncludeCounter(string name); + +// Include multiple counters +ISubscriptionIncludeBuilder IncludeCounters(string[] names); + +// Include ALL counters from ALL documents that match the subscription criteria +ISubscriptionIncludeBuilder IncludeAllCounters(); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **name** | `string` | The name of a counter. The subscription includes every counter with this name in the documents it retrieves. | +| **names** | `string[]` | An array of counter names. | + +All of these methods can be chained. +The following subscription includes several counters and a related document: + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Includes = builder => builder + .IncludeCounter("Likes") + .IncludeCounters(new[] { "Pros", "Cons" }) + .IncludeDocuments("Employee") +}); +``` + +
+ + +Changing the value of an existing counter after the document was sent does not trigger a +re-send. +Adding a new counter to the document, or removing one, does trigger re-sending the document. + + + + + + +### Example: Including time series + +In this example, the subscription sends every `Order` and includes time-series entries for each +document in the batch, so the worker has those entries without issuing additional requests to the server. + +```csharp +subscriptionName = store.Subscriptions.Create(new SubscriptionCreationOptions() +{ + Includes = builder => builder + // Include the last 10 entries of each document's "HeartRate" time series + .IncludeTimeSeries("HeartRate", TimeSeriesRangeType.Last, 10) +}); +``` + +
+ +`ISubscriptionIncludeBuilder` includes time series through these methods: + +```csharp +// Include entries of a named time series, by a time range or by a number of entries +ISubscriptionIncludeBuilder IncludeTimeSeries(string name, TimeSeriesRangeType type, TimeValue time); +ISubscriptionIncludeBuilder IncludeTimeSeries(string name, TimeSeriesRangeType type, int count); + +// The same, for several named time series +ISubscriptionIncludeBuilder IncludeTimeSeries(string[] names, TimeSeriesRangeType type, TimeValue time); +ISubscriptionIncludeBuilder IncludeTimeSeries(string[] names, TimeSeriesRangeType type, int count); + +// Include all time series +ISubscriptionIncludeBuilder IncludeAllTimeSeries(TimeSeriesRangeType type, TimeValue time); +ISubscriptionIncludeBuilder IncludeAllTimeSeries(TimeSeriesRangeType type, int count); +ISubscriptionIncludeBuilder IncludeAllTimeSeries(DateTime? from, DateTime? to); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **name** / **names** | `string` / `string[]` | The time series to include. | +| **type** | `TimeSeriesRangeType` | The range type: `None` or `Last`. | +| **time** | `TimeValue` | A time range to include, for example the last 7 days. | +| **count** | `int` | A number of entries to include, counted from the most recent. | + + + + + +### Example: Creating a subscription via update + +When the subscription you update does not exist, set `CreateNew` to `true` to create it instead, +using the provided query. + +```csharp +subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() +{ + Name = "my subscription", + Query = "from Products where PricePerUnit > 20", + + // Set to true so that a new subscription is created + // if a subscription named "my subscription" does not exist + CreateNew = true +}); +``` + + + + + +### Example: Updating an existing subscription + +You can change a subscription's definition after creation. Target it by name, or by the ID the +server assigned it at creation. The ID also lets you rename the subscription. + +Update by name, here changing the filter query of a subscription named "my subscription": + +```csharp +subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() +{ + // Specify the subscription you wish to modify + Name = "my subscription", + + // Provide a new query + Query = "from Products where PricePerUnit > 50" +}); +``` + +
+ +Update by ID, retrieving the ID from the subscription's state: + +```csharp +// Get the subscription's ID +SubscriptionState mySubscription = store.Subscriptions.GetSubscriptionState("my subscription"); +long subscriptionId = mySubscription.SubscriptionId; + +// Update the subscription +subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() +{ + Id = subscriptionId, + Query = "from Products where PricePerUnit > 50" +}); +``` + +
+ +Targeting by ID also lets you rename the subscription: + +```csharp +// Get the subscription's ID +mySubscription = store.Subscriptions.GetSubscriptionState("my subscription"); +subscriptionId = mySubscription.SubscriptionId; + +// Update the subscription name +subscriptionName = store.Subscriptions.Update(new SubscriptionUpdateOptions() +{ + Id = subscriptionId, + Name = "New name" +}); +``` + + + + + + + + + +### Methods + + + + +Create a subscription on the server. + +```csharp +string Create(SubscriptionCreationOptions options, + string database = null); + +string Create(SubscriptionCreationOptions options = null, + string database = null); + +string Create(SubscriptionCreationOptions options, + string database = null); + +string Create(Expression> predicate, + PredicateSubscriptionCreationOptions options = null, + string database = null); + +Task CreateAsync(SubscriptionCreationOptions options, + string database = null, + CancellationToken token = default); + +Task CreateAsync(SubscriptionCreationOptions options = null, + string database = null, + CancellationToken token = default); + +Task CreateAsync(SubscriptionCreationOptions options, + string database = null, + CancellationToken token = default); + +Task CreateAsync(Expression> predicate, + PredicateSubscriptionCreationOptions options = null, + string database = null, + CancellationToken token = default); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **predicate** | `Expression>` | An optional lambda expression returning a boolean that defines the subscription's filter. | +| **options** | `SubscriptionCreationOptions`, `SubscriptionCreationOptions`, or `PredicateSubscriptionCreationOptions` | The subscription creation options. See the [Classes](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) below. | +| **database** | `string` | The database where the subscription is created. If `null`, the document store's default database is used. | +| **token** | `CancellationToken` | A token used to cancel the creation operation. | + +| Return value | | +|-----------|-------------| +| `string` | The name of the created subscription. If a name was provided in the options it is returned; otherwise the server generates a unique name. | + + + + +Update an existing subscription's definition. + +```csharp +string Update(SubscriptionUpdateOptions options, string database = null); + +Task UpdateAsync(SubscriptionUpdateOptions options, string database = null, + CancellationToken token = default); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **options** | `SubscriptionUpdateOptions` | The subscription update options. See the [Classes](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) below. | +| **database** | `string` | The database where the subscription resides. If `null`, the document store's default database is used. | +| **token** | `CancellationToken` | A token used to cancel the update operation. | + +| Return value | | +|-----------|-------------| +| `string` | The name of the updated subscription. | + + + + + + + + +### Classes + + + + +```csharp +public class SubscriptionCreationOptions +{ + public string Name { get; set; } + public Expression> Filter { get; set; } + public Expression> Projection { get; set; } + public Action> Includes { get; set; } + public string ChangeVector { get; set; } + public bool Disabled { get; set; } + public string MentorNode { get; set; } + public bool PinToMentorNode { get; set; } + public ArchivedDataProcessingBehavior? ArchivedDataProcessingBehavior { get; set; } +} +``` + + + + +```csharp +public class SubscriptionCreationOptions +{ + public string Name { get; set; } + public string Query { get; set; } + public string ChangeVector { get; set; } + public virtual bool Disabled { get; set; } + public string MentorNode { get; set; } + public virtual bool PinToMentorNode { get; set; } + public ArchivedDataProcessingBehavior? ArchivedDataProcessingBehavior { get; set; } +} +``` + + + + + + + +```csharp +public sealed class PredicateSubscriptionCreationOptions +{ + public string Name { get; set; } + public string ChangeVector { get; set; } + public bool Disabled { get; set; } + public string MentorNode { get; set; } + public bool PinToMentorNode { get; set; } + public ArchivedDataProcessingBehavior? ArchivedDataProcessingBehavior { get; set; } +} +``` + + + + +```csharp +public class SubscriptionUpdateOptions : SubscriptionCreationOptions +{ + public long? Id { get; set; } + public bool CreateNew { get; set; } +} +``` + + + + +
+ +| Property | Type | Description | +|----------|------|-------------| +| **<T>** | `T` | The type whose collection the subscription draws its documents from. | +| **Name** | `string` | A user-defined name for the subscription. Must be unique within the database. | +| **Query** | `string` | An RQL query that defines the subscription. Supports JavaScript functions inside the `where` clause and special semantics for subscriptions on document revisions. | +| **Filter** | `Expression>` | A lambda expression defining the subscription's filter. Translated to a JavaScript function. | +| **Projection** | `Expression>` | A lambda expression defining the projection sent for each matching document. Translated to a JavaScript function. | +| **Includes** | `Action>` | An action defining the subscription's include clauses. Included documents, counters, and time series are part of the batch the subscription sends. The include methods can be chained. | +| **ChangeVector** | `string` | A change vector marking where the subscription starts processing. See the note below. | +| **Disabled** | `bool` | `true` creates the task in a disabled state; `false` creates it enabled. | +| **MentorNode** | `string` | The tag of the node you want to handle the subscription. Useful when you prefer a specific server for its hardware, its proximity to clients, or other reasons. | +| **PinToMentorNode** | `bool` | `true` pins the task to the responsible node; `false` lets another node run it if the responsible node is down. | +| **ArchivedDataProcessingBehavior** | `ArchivedDataProcessingBehavior?` | Whether archived documents are included in the subscription. See [Archived documents and subscriptions](../../../data-archival/archived-documents-and-other-features.mdx#archived-documents-and-subscriptions). | + +
+ +`SubscriptionUpdateOptions` adds two properties to the creation options: + +| Property | Type | Description | +|----------|------|-------------| +| **Id** | `long?` | The ID the server assigned the subscription at creation. Retrieve it from the subscription's state. When updating, the `Id` can be used instead of the `Name` and takes precedence over the `Name`, which lets you rename the subscription: provide the `Id` and a new `Name`. | +| **CreateNew** | `bool` | What to do when the target subscription does not exist. `true` creates a new subscription from the provided options; `false` throws. Default: `false`. | + +
+ + + +The `ChangeVector` property sets a starting point for processing, which is useful for an ad-hoc +process that needs only recent changes. +Set it to `"LastDocument"` to start from the latest document in the collection, or pass an actual +change vector to start from a specific point. +By default the subscription sends all documents matching its query, regardless of when they were +created. + + + + + + diff --git a/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-nodejs.mdx b/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-nodejs.mdx new file mode 100644 index 0000000000..8ad57f41c9 --- /dev/null +++ b/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-nodejs.mdx @@ -0,0 +1,744 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* A data subscription is a server-side task that sends documents matching a query to connected + clients ("workers"), in batches. +* This page explains how to create and update a subscription using the Client API. + You can also [create a subscription using Studio](../../../data-subscriptions/creating-subscription/creating-subscription_studio.mdx). +* A subscription's definition and its processing progress are kept on the cluster rather than on + a single node, and the cluster assigns one node to manage the task. + +* In this article: + * [Creating a subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#creating-a-subscription) + * [Subscription name](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-name) + * [Responsible node](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#responsible-node) + * [Updating a subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#updating-a-subscription) + * [Subscription query](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-query) + * [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) + * [Sending an entire collection](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-sending-an-entire-collection) + * [Filtering documents](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-documents) + * [Filtering with a regular expression](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-with-a-regular-expression) + * [Filtering and projecting fields](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-and-projecting-fields) + * [Projecting from a related document](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-projecting-from-a-related-document) + * [Including documents](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-documents) + * [Including counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters) + * [Including time series](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-time-series) + * [Creating a subscription via update](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-creating-a-subscription-via-update) + * [Updating an existing subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-updating-an-existing-subscription) + * [Syntax](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#syntax) + + + + + +To create a subscription, define which documents the server should send to the worker, then register +this definition with the server using the `create` method on the document store's `subscriptions` +property. + +You can define a subscription in two ways: + +* **By document type** + Pass the collection's document type, so the server sends that entire collection. +* **By a raw RQL (Raven Query Language) query** + Set the `query` string on the options object, which lets you filter, project, and include. + +The simplest definition sends an entire collection: + +```js +// The server sends every document in the 'Orders' collection +// to any client that connects to this subscription. +const subscriptionName = await documentStore.subscriptions.create(Order); +``` + +
+ +For filtering, projections, and includes, see the [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) below. +To subscribe to document revisions, see [Revisions support](../../../data-subscriptions/revisions-support.mdx). + + + + + +Every subscription has a name, which clients use to identify the subscription they want to +consume. +If you do not provide a name when creating the subscription, the server generates one for you. +To set your own: + +```js +const subscriptionName = await documentStore.subscriptions.create({ + // Set a custom name for the subscription + name: "OrdersProcessingSubscription", + documentType: Order +}); +``` + +
+ +Setting your own name is useful for long-running batch processes: a readable name is easier to +work with in code, and you can reuse the same name across environments, as long as the +subscription is created in advance. + + +A subscription name must be unique within its database. You cannot create two subscriptions with +the same name in the same database. + + + + + + +When a subscription is created, the cluster assigns one node to manage it. As long as this node is available, +it remains responsible for the subscription. If a worker connects to another node, that node will redirect it +to the responsible node. + +If the responsible node goes down and your license includes +[highly available tasks](../../../server/clustering/distribution/highly-available-tasks.mdx), +the task is reassigned to another node. +While the assigned node is online, it remains the one managing the subscription. + +To choose the responsible node yourself, set its tag in the `mentorNode` property: + +```js +const subscriptionName = await documentStore.subscriptions.create({ + // Make node D responsible for the subscription + mentorNode: "D", + documentType: Order +}); +``` + +
+ +Choosing the node manually lets you pick a server based on its hardware, proximity to the +consuming clients, or other considerations. + + + + + +You can change an existing subscription's definition using the [update](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#methods) method. +Pass an [update options](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) object, which extends the creation options with two +additional fields, `id` and `createNew`: + +* **`id`** targets the subscription by the ID the server assigned it at creation, instead of by + name. + Because the ID identifies the subscription independently of its name, you can use it to rename + a subscription: provide the ID and a new `name`. +* **`createNew`** controls what happens when the subscription you are updating does not exist. + Set it to `true` to create the subscription instead, or leave it `false`, the default, to + throw an exception. + +```js +// Rename a subscription by targeting it with its id +await documentStore.subscriptions.update({ + id: subscriptionId, + name: "NewSubscriptionName" +}); +``` + +
+ +For a full update example, see the [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) below. + + + + + +Every subscription is ultimately expressed as an RQL statement. +A subscription query can have these parts: + +* **A `declare` functions section**, as in ordinary RQL. + These functions can hold any JavaScript code, and support `load` and `include` operations. +* **A `from` clause** naming the document source, for example `from Orders`. + This clause can address only a collection; indexes are not supported. +* **A `where` clause** deciding the documents the server sends to the worker. + This clause supports RQL equality operators (`=`, `==`), plain JavaScript expressions, + and calls to declared functions, so you can express complex filtering. + Subscription RQL does not support RQL's search keywords. +* **A `select` clause** defining the projection. + This clause can call functions, allowing complex transformations. +* **An `include` clause** defining paths to include. + + +Although a subscription query has an RQL-like structure, it supports only the `declare`, +`select`, and `where` keywords; no other RQL keywords are supported. JavaScript ES5 syntax is +supported. For more details, see [JavaScript engine](../../../server/kb/javascript-engine.mdx). + + + + +Paths in a subscription query are handled as JavaScript expressions rather than as regular RQL +paths. +A query that in RQL would be written as: + +```rql +from Orders as o +where o.Lines[].Product = "products/1-A" +``` + +
+ +is written in a subscription query as follows: + +```rql +declare function filterLines(doc, productId) +{ + if (!!doc.Lines){ + return doc.Lines.filter(x => x.Product == productId).length > 0; + } + return false; +} + +from Orders as o +where filterLines(o, "products/1-A") +``` + + + +To create a subscription over document revisions, see +[Revisions support](../../../data-subscriptions/revisions-support.mdx). + + + + + + + +### Example: Sending an entire collection + +In this example, the subscription sends to the worker every document in the `Orders` collection, with no filter or projection. + + + + +```js +const subscriptionName = await documentStore.subscriptions.create(Order); +``` + + + + +```js +const subscriptionName = await documentStore.subscriptions.create({ + query: "from Orders" +}); +``` + + + + + + + + +### Example: Filtering documents + +In this example, the subscription sends to the worker only `Orders` whose total revenue exceeds 100. +The server evaluates the filter and sends just the matching documents. + +```js +const query = `declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + from Orders as o + where getOrderLinesSum(o) > 100`; + +const subscriptionName = await documentStore.subscriptions.create({ query }); +``` + + + + + +### Example: Filtering with a regular expression + +In this example, the subscription filters documents with a regular expression. +Use the RQL `regex` function to match a field against a pattern on the server, here `Orders` whose +`ShipTo.City` starts with `New`. + +```js +const query = "from Orders where regex(ShipTo.City, '^New')"; + +const subscriptionName = await documentStore.subscriptions.create({ query }); +``` + + + + + +### Example: Filtering and projecting fields + +In this example, the subscription filters `Orders` by total revenue, and projects only two fields into each item +sent to the client: the document ID and the computed total. + +```js +const query = `declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + declare function projectOrder(doc) { + return { + Id: doc.Id, + Total: getOrderLinesSum(doc) + }; + } + + from Orders as o + where getOrderLinesSum(o) > 100 + select projectOrder(o)`; + +const subscriptionName = await documentStore.subscriptions.create({ query }); +``` + + + + + +### Example: Projecting from a related document + +In this example, the subscription projects fields from each `Order` and also pulls data from a +[related document](../../../indexes/indexing-related-documents.mdx#what-are-related-documents), +loaded using `load`. + +```js +const query = `declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + declare function projectOrder(doc) { + var employee = load(doc.Employee); + return { + Id: doc.Id, + Total: getOrderLinesSum(doc), + ShipTo: doc.ShipTo, + EmployeeName: employee.FirstName + ' ' + employee.LastName + }; + } + + from Orders as o + where getOrderLinesSum(o) > 100 + select projectOrder(o)`; + +const subscriptionName = await documentStore.subscriptions.create({ query }); +``` + + + + + +### Example: Including documents + +In this example, the subscription sends every `Order` and includes the `Product` documents related to the order +in the same batch, so the consuming worker can later read each `Product` without an extra call to +the server. + + + + +```js +const options = { + // The documents whose IDs are in each order line's 'Product' property + // will be included in the batch + includes: builder => builder.includeDocuments("Lines[].Product"), + documentType: Order +}; + +const subscriptionName = await documentStore.subscriptions.create(options); +``` + + + + +```js +const subscriptionName = await documentStore.subscriptions.create({ + query: "from Orders include Lines[].Product" +}); +``` + + + + +```js +const query = `declare function includeProducts(doc) { + let linesCount = doc.Lines.length; + + for (let i = 0; i < linesCount; i++) { + include(doc.Lines[i].Product); + } + + return doc; + } + + from Orders as o select includeProducts(o)`; + +const subscriptionName = await documentStore.subscriptions.create({ query }); +``` + + + + +
+ +Using the builder, assign an include function to the `includes` property of the options object. +The builder includes related documents through `includeDocuments`, and its methods can be chained. + +In RQL, write the include in one of two ways: append the `include` keyword followed by the paths +to the fields holding the IDs to include (preferred, for both clarity and slightly better +performance), or call `include` inside a JavaScript function invoked from the `select` clause. + + +When you include documents alongside a projection, the `include` resolves paths against the +projected fields rather than the original document. + + + + + + +### Example: Including counters + +In this example, the subscription sends to the worker every `Order` and includes the values of the +named counters in the batch. + + + + +```js +const options = { + // Values for the specified counters will be included in the batch + includes: builder => builder.includeCounters(["Pros", "Cons"]), + documentType: Order +}; + +const subscriptionName = await documentStore.subscriptions.create(options); +``` + + + + +```js +const subscriptionName = await documentStore.subscriptions.create({ + query: "from Orders include counters('Pros'), counters('Cons')" +}); +``` + + + + +
+ +The include builder offers three methods for including counters: + +```js +// Include a single counter +includeCounter(name); + +// Include multiple counters +includeCounters(names); + +// Include ALL counters from ALL documents that match the subscription criteria +includeAllCounters(); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **name** | `string` | The name of a counter. The subscription includes every counter with this name in the documents it retrieves. | +| **names** | `string[]` | An array of counter names. | + +All of these methods can be chained. +The following subscription includes several counters and a related document: + +```js +const options = { + includes: builder => builder + .includeCounter("Likes") + .includeCounters(["Pros", "Cons"]) + .includeDocuments("Employee"), + documentType: Order +}; + +const subscriptionName = await documentStore.subscriptions.create(options); +``` + +
+ + +Changing the value of an existing counter after the document was sent does not trigger a +re-send. +Adding a new counter to the document, or removing one, does trigger re-sending the document. + + + + + + +### Example: Including time series + +In this example, the subscription sends every `Order` and includes time-series entries for each +document in the batch, so the worker has those entries without issuing additional requests to the server. + +```js +const options = { + // Include the last 10 entries of each document's "HeartRate" time series + includes: builder => builder.includeTimeSeries("HeartRate", "Last", 10), + documentType: Order +}; + +const subscriptionName = await documentStore.subscriptions.create(options); +``` + +
+ +The include builder includes time series through these methods: + +```js +// Include entries of a named time series, by a time range or by a number of entries +includeTimeSeries(name, type, time); +includeTimeSeries(name, type, count); + +// The same, for several named time series +includeTimeSeries(names, type, time); +includeTimeSeries(names, type, count); + +// Include all time series +includeAllTimeSeries(type, time); +includeAllTimeSeries(type, count); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **name** / **names** | `string` / `string[]` | The time series to include. | +| **type** | `string` | The range type: `"None"` or `"Last"`. | +| **time** | `TimeValue` | A time range to include, for example `TimeValue.ofDays(7)`. | +| **count** | `number` | A number of entries to include, counted from the most recent. | + + + + + +### Example: Creating a subscription via update + +When the subscription you update does not exist, set `createNew` to `true` to create it instead, +using the provided query. + +```js +const subscriptionName = await documentStore.subscriptions.update({ + name: "my subscription", + query: "from Products where PricePerUnit > 20", + + // Set to true so that a new subscription is created + // if a subscription named "my subscription" does not exist + createNew: true +}); +``` + + + + + +### Example: Updating an existing subscription + +You can change a subscription's definition after creation. Target it by name, or by the ID the +server assigned it at creation. The ID also lets you rename the subscription. + +Update by name, here changing the filter query of a subscription named "my subscription": + +```js +const subscriptionName = await documentStore.subscriptions.update({ + // Specify the subscription you wish to modify + name: "my subscription", + + // Provide a new query + query: "from Products where PricePerUnit > 50" +}); +``` + +
+ +Update by ID, retrieving the ID from the subscription's state: + +```js +// Get the subscription's ID +const mySubscription = await documentStore.subscriptions.getSubscriptionState("my subscription"); +const subscriptionId = mySubscription.subscriptionId; + +// Update the subscription +const subscriptionName = await documentStore.subscriptions.update({ + id: subscriptionId, + query: "from Products where PricePerUnit > 50" +}); +``` + +
+ +Targeting by ID also lets you rename the subscription: + +```js +// Get the subscription's ID +const mySubscription = await documentStore.subscriptions.getSubscriptionState("my subscription"); +const subscriptionId = mySubscription.subscriptionId; + +// Update the subscription name +const subscriptionName = await documentStore.subscriptions.update({ + id: subscriptionId, + name: "New name" +}); +``` + + + + + + + + + +### Methods + + + + +Create a subscription on the server. + +```js +// Overloads: +create(options); +create(options, database); +create(documentType); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **options** | `object` | The subscription creation options. See the [Classes](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) below. | +| **database** | `string` | The database where the subscription is created. If not specified, the document store's default database is used. | +| **documentType** | `object` | The document class whose collection the subscription draws from. | + +| Return value | | +|-----------|-------------| +| `Promise` | Resolves to the name of the created subscription. If a name was provided in the options it is returned; otherwise the server generates a unique name. | + + + + +Update an existing subscription's definition. + +```js +// Overloads: +update(options); +update(options, database); +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **options** | `object` | The subscription update options. See the [Classes](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) below. | +| **database** | `string` | The database where the subscription resides. If not specified, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `Promise` | Resolves to the name of the updated subscription. | + + + + + + + + +### Classes + + + + +```js +// The subscription creation options object: +{ + name; + query; + includes; + changeVector; + mentorNode; + pinToMentorNode; + disabled; + documentType; +} +``` + + + + +```js +// The subscription update options object extends the creation options +// object and adds two fields: +{ + id; + createNew; +} +``` + + + + +
+ +| Property | Type | Description | +|----------|------|-------------| +| **name** | `string` | A user-defined name for the subscription. Must be unique within the database. | +| **query** | `string` | An RQL query that defines the subscription. Supports JavaScript functions inside the `where` clause and special semantics for subscriptions on document revisions. | +| **includes** | `(builder) => void` | A function receiving an include builder, used to include related documents, counters, and time series in the batch. The include methods can be chained. | +| **changeVector** | `string` | A change vector marking where the subscription starts processing. See the note below. | +| **mentorNode** | `string` | The tag of the node you want to handle the subscription. Useful when you prefer a specific server for its hardware, its proximity to clients, or other reasons. | +| **pinToMentorNode** | `boolean` | `true` pins the task to the responsible node; `false` lets another node run it if the responsible node is down. | +| **disabled** | `boolean` | `true` creates the task in a disabled state; `false` creates it enabled. | +| **documentType** | `object` | The document class whose collection the subscription draws from. | + +
+ +The update options object adds two properties to the creation options: + +| Property | Type | Description | +|----------|------|-------------| +| **id** | `number` | The ID the server assigned the subscription at creation. Retrieve it from the subscription's state. When updating, the `id` can be used instead of the `name` and takes precedence over it, which lets you rename the subscription: provide the `id` and a new `name`. | +| **createNew** | `boolean` | What to do when the target subscription does not exist. `true` creates a new subscription from the provided options; `false` throws. Default: `false`. | + +
+ + + +The `changeVector` property sets a starting point for processing, which is useful for an ad-hoc +process that needs only recent changes. +Set it to `"LastDocument"` to start from the latest document in the collection, or pass an actual +change vector to start from a specific point. +By default the subscription sends all documents matching its query, regardless of when they were +created. + + + + + + diff --git a/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-python.mdx b/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-python.mdx new file mode 100644 index 0000000000..e25fd90316 --- /dev/null +++ b/docs/data-subscriptions/creating-subscription/content/_creating-subscription_api-python.mdx @@ -0,0 +1,720 @@ +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + + + +* A data subscription is a server-side task that sends documents matching a query to connected + clients ("workers"), in batches. +* This page explains how to create and update a subscription using the Client API. + You can also [create a subscription using Studio](../../../data-subscriptions/creating-subscription/creating-subscription_studio.mdx). +* A subscription's definition and its processing progress are kept on the cluster rather than on + a single node, and the cluster assigns one node to manage the task. + +* In this article: + * [Creating a subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#creating-a-subscription) + * [Subscription name](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-name) + * [Responsible node](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#responsible-node) + * [Updating a subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#updating-a-subscription) + * [Subscription query](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#subscription-query) + * [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) + * [Sending an entire collection](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-sending-an-entire-collection) + * [Filtering documents](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-documents) + * [Filtering with a regular expression](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-with-a-regular-expression) + * [Filtering and projecting fields](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-and-projecting-fields) + * [Projecting from a related document](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-projecting-from-a-related-document) + * [Including documents](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-documents) + * [Including counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters) + * [Including time series](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-time-series) + * [Updating an existing subscription](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-updating-an-existing-subscription) + * [Syntax](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#syntax) + + + + + +To create a subscription, define which documents the server should send to the worker, then register +this definition with the server using the `create_for_class` or `create_for_options` method on the +document store's `subscriptions` property. + +You can define a subscription in two ways: + +* **By document type** + Pass the collection's document type to `create_for_class`, so the server sends that entire collection. +* **By a raw RQL (Raven Query Language) query** + Set the `query` string on `SubscriptionCreationOptions` and pass it to `create_for_options`, which + lets you filter, project, and include. + +The simplest definition sends an entire collection: + +```python +# The server sends every document in the 'Orders' collection +# to any client that connects to this subscription. +name = store.subscriptions.create_for_class(Order) +``` + +
+ +For filtering, projections, and includes, see the [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) below. + + + + + +Every subscription has a name, which clients use to identify the subscription they want to +consume. +If you do not provide a name when creating the subscription, the server generates one for you. +To set your own: + +```python +name = store.subscriptions.create_for_class( + Order, SubscriptionCreationOptions(name="OrdersProcessingSubscription") +) +``` + +
+ +Setting your own name is useful for long-running batch processes: a readable name is easier to +work with in code, and you can reuse the same name across environments, as long as the +subscription is created in advance. + + +A subscription name must be unique within its database. You cannot create two subscriptions with +the same name in the same database. + + + + + + +When a subscription is created, the cluster assigns one node to manage it. As long as this node is available, +it remains responsible for the subscription. If a worker connects to another node, that node will redirect it +to the responsible node. + +If the responsible node goes down and your license includes +[highly available tasks](../../../server/clustering/distribution/highly-available-tasks.mdx), +the task is reassigned to another node. +While the assigned node is online, it remains the one managing the subscription. + +To choose the responsible node yourself, set its tag in the `mentor_node` property: + +```python +name = store.subscriptions.create_for_class( + # Make node D responsible for the subscription + Order, SubscriptionCreationOptions(mentor_node="D") +) +``` + +
+ +Choosing the node manually lets you pick a server based on its hardware, proximity to the +consuming clients, or other considerations. + + + + + +You can change an existing subscription's definition using the [update](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#methods) method. +Pass a [SubscriptionUpdateOptions](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes), which extends the creation options with two +additional fields, `key` and `create_new`: + +* **`key`** targets the subscription by the ID the server assigned it at creation, instead of by + name. + Because the ID identifies the subscription independently of its name, you can use it to rename + a subscription: provide the `key` and a new `name`. +* **`create_new`** controls what happens when the subscription you are updating does not exist. + Set it to `True` to create the subscription instead, or leave it `False`, the default, to + throw an exception. + +```python +# Rename a subscription by targeting it with its key (the server-assigned ID) +store.subscriptions.update(SubscriptionUpdateOptions(key=subscription_id, name="NewSubscriptionName")) +``` + +
+ +For a full update example, see the [Examples](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#examples) below. + + + + + +Every subscription is ultimately expressed as an RQL statement. +A subscription query can have these parts: + +* **A `declare` functions section**, as in ordinary RQL. + These functions can hold any JavaScript code, and support `load` and `include` operations. +* **A `from` clause** naming the document source, for example `from Orders`. + This clause can address only a collection; indexes are not supported. +* **A `where` clause** deciding the documents the server sends to the worker. + This clause supports RQL equality operators (`=`, `==`), plain JavaScript expressions, + and calls to declared functions, so you can express complex filtering. + Subscription RQL does not support RQL's search keywords. +* **A `select` clause** defining the projection. + This clause can call functions, allowing complex transformations. +* **An `include` clause** defining paths to include. + + +Although a subscription query has an RQL-like structure, it supports only the `declare`, +`select`, and `where` keywords; no other RQL keywords are supported. JavaScript ES5 syntax is +supported. For more details, see [JavaScript engine](../../../server/kb/javascript-engine.mdx). + + + + +Paths in a subscription query are handled as JavaScript expressions rather than as regular RQL +paths. +A query that in RQL would be written as: + +```rql +from Orders as o +where o.Lines[].Product = "products/1-A" +``` + +
+ +is written in a subscription query as follows: + +```rql +declare function filterLines(doc, productId) +{ + if (!!doc.Lines){ + return doc.Lines.filter(x => x.Product == productId).length > 0; + } + return false; +} + +from Orders as o +where filterLines(o, "products/1-A") +``` + + + +To create a subscription over document revisions, see +[Revisions support](../../../data-subscriptions/revisions-support.mdx). + + + + + + + +### Example: Sending an entire collection + +In this example, the subscription sends to the worker every document in the `Orders` collection, with no filter or projection. + + + + +```python +name = store.subscriptions.create_for_class(Order) +``` + + + + +```python +name = store.subscriptions.create_for_options( + SubscriptionCreationOptions(query="from Orders") +) +``` + + + + + + + + +### Example: Filtering documents + +In this example, the subscription sends to the worker only `Orders` whose total revenue exceeds 100. +The server evaluates the filter and sends just the matching documents. + +```python +name = store.subscriptions.create_for_options( + SubscriptionCreationOptions( + query=""" + declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + from Orders as o + where getOrderLinesSum(o) > 100 + """ + ) +) +``` + + + + + +### Example: Filtering with a regular expression + +In this example, the subscription filters documents with a regular expression. +Use the RQL `regex` function to match a field against a pattern on the server, here `Orders` whose +`ShipTo.City` starts with `New`. + +```python +name = store.subscriptions.create_for_options( + SubscriptionCreationOptions(query="from Orders where regex(ShipTo.City, '^New')") +) +``` + + + + + +### Example: Filtering and projecting fields + +In this example, the subscription filters `Orders` by total revenue, and projects only two fields into each item +sent to the client: the document ID and the computed total. + +```python +name = store.subscriptions.create_for_options( + SubscriptionCreationOptions( + query=""" + declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + declare function projectOrder(doc) { + return { + Id: doc.Id, + Total: getOrderLinesSum(doc) + }; + } + + from Orders as o + where getOrderLinesSum(o) > 100 + select projectOrder(o) + """ + ) +) +``` + + + + + +### Example: Projecting from a related document + +In this example, the subscription projects fields from each `Order` and also pulls data from a +[related document](../../../indexes/indexing-related-documents.mdx#what-are-related-documents), +loaded using `load`. + +```python +name = store.subscriptions.create_for_options( + SubscriptionCreationOptions( + query=""" + declare function getOrderLinesSum(doc) { + var sum = 0; + for (var i in doc.Lines) { + sum += doc.Lines[i].PricePerUnit * doc.Lines[i].Quantity; + } + return sum; + } + + declare function projectOrder(doc) { + var employee = load(doc.Employee); + return { + Id: doc.Id, + Total: getOrderLinesSum(doc), + ShipTo: doc.ShipTo, + EmployeeName: employee.FirstName + ' ' + employee.LastName + }; + } + + from Orders as o + where getOrderLinesSum(o) > 100 + select projectOrder(o) + """ + ) +) +``` + + + + + +### Example: Including documents + +In this example, the subscription sends every `Order` and includes the `Product` documents related to the order +in the same batch, so the consuming worker can later read each `Product` without an extra call to +the server. + + + + +```python +store.subscriptions.create_for_class( + Order, + # The documents whose IDs are in each order line's 'Product' property + # will be included in the batch + SubscriptionCreationOptions(includes=lambda builder: builder.include_documents("Lines[].Product")), +) +``` + + + + +```python +store.subscriptions.create_for_options( + SubscriptionCreationOptions(query="from Orders include Lines[].Product") +) +``` + + + + +```python +store.subscriptions.create_for_options( + SubscriptionCreationOptions( + query=""" + declare function includeProducts(doc) { + let linesCount = doc.Lines.length; + + for (let i = 0; i < linesCount; i++) { + include(doc.Lines[i].Product); + } + + return doc; + } + + from Orders as o select includeProducts(o) + """ + ) +) +``` + + + + +
+ +Using the builder, assign an include function to the `includes` property of `SubscriptionCreationOptions`. +The builder includes related documents through `include_documents`, and its methods can be chained. + +In RQL, write the include in one of two ways: append the `include` keyword followed by the paths +to the fields holding the IDs to include (preferred, for both clarity and slightly better +performance), or call `include` inside a JavaScript function invoked from the `select` clause. + + +When you include documents alongside a projection, the `include` resolves paths against the +projected fields rather than the original document. + + + + + + +### Example: Including counters + +In this example, the subscription sends to the worker every `Order` and includes the values of the +named counters in the batch. + + + + +```python +store.subscriptions.create_for_class( + Order, + # Values for the specified counters will be included in the batch + SubscriptionCreationOptions(includes=lambda builder: builder.include_counters("Pros", "Cons")), +) +``` + + + + +```python +store.subscriptions.create_for_options( + SubscriptionCreationOptions(query="from Orders include counters('Pros'), counters('Cons')") +) +``` + + + + +
+ +The include builder offers three methods for including counters: + +```python +# Include a single counter +def include_counter(self, name: str) -> SubscriptionIncludeBuilder: ... + +# Include multiple counters +def include_counters(self, *names: str) -> SubscriptionIncludeBuilder: ... + +# Include ALL counters from ALL documents that match the subscription criteria +def include_all_counters(self) -> SubscriptionIncludeBuilder: ... +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **name** | `str` | The name of a counter. The subscription includes every counter with this name in the documents it retrieves. | +| **\*names** | `str` | Counter names. | + +All of these methods can be chained. +The following subscription includes several counters and a related document: + +```python +store.subscriptions.create_for_class( + Order, + SubscriptionCreationOptions( + includes=lambda builder: builder + .include_counter("Likes") + .include_counters("Pros", "Cons") + .include_documents("Employee") + ), +) +``` + +
+ + +Changing the value of an existing counter after the document was sent does not trigger a +re-send. +Adding a new counter to the document, or removing one, does trigger re-sending the document. + + + + + + +### Example: Including time series + +In this example, the subscription sends every `Order` and includes time-series entries for each +document in the batch, so the worker has those entries without issuing additional requests to the server. + +```python +store.subscriptions.create_for_class( + Order, + SubscriptionCreationOptions( + # Include the last 10 entries of each document's "HeartRate" time series + includes=lambda builder: builder.include_time_series_by_range_type_and_count( + "HeartRate", TimeSeriesRangeType.LAST, 10 + ) + ), +) +``` + +
+ +`SubscriptionIncludeBuilder` includes time series through these methods: + +```python +# Include entries of a named time series, by a time range or by a number of entries +def include_time_series_by_range_type_and_time(self, name, ts_type, time) -> SubscriptionIncludeBuilder: ... +def include_time_series_by_range_type_and_count(self, name, ts_type, count) -> SubscriptionIncludeBuilder: ... + +# Include all time series +def include_all_time_series_by_range_type_and_time(self, ts_type, time) -> SubscriptionIncludeBuilder: ... +def include_all_time_series_by_range_type_and_count(self, ts_type, count) -> SubscriptionIncludeBuilder: ... +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **name** | `str` | The time series to include. | +| **ts_type** | `TimeSeriesRangeType` | The range type: `NONE` or `LAST`. | +| **time** | `TimeValue` | A time range to include. | +| **count** | `int` | A number of entries to include, counted from the most recent. | + + + + + +### Example: Updating an existing subscription + +You can change a subscription's definition after creation. Target it by name, or by the ID the +server assigned it at creation. The ID also lets you rename the subscription. + +Update by name, here changing the filter query of a subscription named "my subscription": + +```python +store.subscriptions.update( + SubscriptionUpdateOptions(name="my subscription", query="from Products where PricePerUnit > 50") +) +``` + +
+ +Update by ID, retrieving the ID from the subscription's state: + +```python +# Get the subscription's ID +my_subscription = store.subscriptions.get_subscription_state("my subscription") +subscription_id = my_subscription.subscription_id + +# Update the subscription +store.subscriptions.update( + SubscriptionUpdateOptions(key=subscription_id, query="from Products where PricePerUnit > 50") +) +``` + +
+ +Targeting by ID also lets you rename the subscription: + +```python +# Get the subscription's ID +my_subscription = store.subscriptions.get_subscription_state("my subscription") +subscription_id = my_subscription.subscription_id + +# Update the subscription name +store.subscriptions.update(SubscriptionUpdateOptions(key=subscription_id, name="New name")) +``` + + + + + + + + + +### Methods + + + + +Create a subscription on the server. + +```python +def create_for_options(self, options: SubscriptionCreationOptions, + database: Optional[str] = None) -> str: ... + +def create_for_class(self, object_type: Type[_T], + options: Optional[SubscriptionCreationOptions] = None, + database: Optional[str] = None) -> str: ... +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **options** | `SubscriptionCreationOptions` | The subscription creation options. See the [Classes](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) below. | +| **object_type** | `Type[_T]` | The document class whose collection the subscription draws from. | +| **database** | `str` | The database where the subscription is created. If `None`, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `str` | The name of the created subscription. If a name was provided in the options it is returned; otherwise the server generates a unique name. | + + + + +Update an existing subscription's definition. + +```python +def update(self, options: SubscriptionUpdateOptions, + database: Optional[str] = None) -> str: ... +``` + +
+ +| Parameter | Type | Description | +|-----------|------|-------------| +| **options** | `SubscriptionUpdateOptions` | The subscription update options. See the [Classes](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) below. | +| **database** | `str` | The database where the subscription resides. If `None`, the document store's default database is used. | + +| Return value | | +|-----------|-------------| +| `str` | The name of the updated subscription. | + + + + + + + + +### Classes + + + + +```python +class SubscriptionCreationOptions: + def __init__( + self, + name: Optional[str] = None, + query: Optional[str] = None, + includes: Optional[Callable[[SubscriptionIncludeBuilder], None]] = None, + change_vector: Optional[str] = None, + mentor_node: Optional[str] = None, + ): ... +``` + + + + +```python +class SubscriptionUpdateOptions(SubscriptionCreationOptions): + def __init__( + self, + name: Optional[str] = None, + query: Optional[str] = None, + includes: Optional[Callable[[SubscriptionIncludeBuilder], None]] = None, + change_vector: Optional[str] = None, + mentor_node: Optional[str] = None, + key: Optional[int] = None, + create_new: Optional[bool] = None, + ): ... +``` + + + + +
+ +| Property | Type | Description | +|----------|------|-------------| +| **name** | `str` | A user-defined name for the subscription. Must be unique within the database. | +| **query** | `str` | An RQL query that defines the subscription. Supports JavaScript functions inside the `where` clause and special semantics for subscriptions on document revisions. | +| **includes** | `Callable[[SubscriptionIncludeBuilder], None]` | A function receiving an include builder, used to include related documents, counters, and time series in the batch. The include methods can be chained. | +| **change_vector** | `str` | A change vector marking where the subscription starts processing. See the note below. | +| **mentor_node** | `str` | The tag of the node you want to handle the subscription. Useful when you prefer a specific server for its hardware, its proximity to clients, or other reasons. | + +
+ +`SubscriptionUpdateOptions` adds two properties to the creation options: + +| Property | Type | Description | +|----------|------|-------------| +| **key** | `int` | The ID the server assigned the subscription at creation. Retrieve it from the subscription's state. When updating, the `key` can be used instead of the `name` and takes precedence over it, which lets you rename the subscription: provide the `key` and a new `name`. | +| **create_new** | `bool` | What to do when the target subscription does not exist. `True` creates a new subscription from the provided options; `False` throws. Default: `False`. | + +
+ + + +The `change_vector` property sets a starting point for processing, which is useful for an ad-hoc +process that needs only recent changes. +Set it to `"LastDocument"` to start from the latest document in the collection, or pass an actual +change vector to start from a specific point. +By default the subscription sends all documents matching its query, regardless of when they were +created. + + + + + + diff --git a/docs/data-subscriptions/creating-subscription/creating-subscription_api.mdx b/docs/data-subscriptions/creating-subscription/creating-subscription_api.mdx new file mode 100644 index 0000000000..52daf529db --- /dev/null +++ b/docs/data-subscriptions/creating-subscription/creating-subscription_api.mdx @@ -0,0 +1,29 @@ +--- +title: "Creating a data subscription: API" +sidebar_label: "Client API" +description: "Create and update RavenDB data subscriptions using the Client API, with filtering, projections, includes, and configurable subscription options." +sidebar_position: 1 +--- + +import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; +import LanguageContent from "@site/src/components/LanguageContent"; + +import CreatingSubscriptionApiCsharp from './content/_creating-subscription_api-csharp.mdx'; +import CreatingSubscriptionApiNodejs from './content/_creating-subscription_api-nodejs.mdx'; +import CreatingSubscriptionApiPython from './content/_creating-subscription_api-python.mdx'; + +export const supportedLanguages = ["csharp", "nodejs", "python"]; + + + + + + + + + + + + + + diff --git a/docs/data-subscriptions/creating-subscription/creating-subscription_studio.mdx b/docs/data-subscriptions/creating-subscription/creating-subscription_studio.mdx new file mode 100644 index 0000000000..465bfeb202 --- /dev/null +++ b/docs/data-subscriptions/creating-subscription/creating-subscription_studio.mdx @@ -0,0 +1,90 @@ +--- +title: "Creating a data subscription: Studio" +sidebar_label: "Studio" +description: "Create a data subscription task in RavenDB Studio: write the subscription query, set the starting point and responsible node, test the query, and save the task." +sidebar_position: 2 +--- + +import Admonition from '@theme/Admonition'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + +# Creating a data subscription: Studio + + +* This page explains how to create a data subscription using Studio. + You can also [create a subscription using the Client API](../../data-subscriptions/creating-subscription/creating-subscription_api.mdx). + +* In this article: + * [Defining the subscription task](../../data-subscriptions/creating-subscription/creating-subscription_studio.mdx#defining-the-subscription-task) + + + + + + + +To create a subscription, open **`Tasks` > `Ongoing Tasks` > `Add a database task`** and select **Subscription**. + +![Open the ongoing tasks view](./assets/subscriptions_ongoing-tasks-view.png) + +1. Open the **Tasks** menu. +2. Select **Ongoing Tasks**. +3. Click **Add a database task** and select **Subscription**. + ![Open the ongoing tasks view](./assets/subscriptions_subscription-task-selection.png) + + + + + +Enter a task name, then define the subscription: + +![Defining the subscription task](./assets/subscriptions_define-task.png) + +1. **Task name** + Enter a name for the subscription task. + +2. **Task state** + Enable or disable the task. + +3. **Subscription RQL** + The RQL query that defines the documents the server sends to the worker. + e.g., `from Orders` + Click **Syntax** for help writing the query. + +4. **Archived data processing behavior** + ![Defining the subscription task](./assets/subscriptions_archived-data-processing.png) + Set how the subscription processes archived documents. + * **Default**: use the archived-data behavior [configured for the server or database](../../data-subscriptions/configuration.mdx#subscriptionsarchiveddataprocessingbehavior). + * **Exclude archived**: Only non-archived documents will be included in processing. + * **Include archived**: Both archived and non-archived documents will be included in processing. + * **Archived only**: Only archived documents will be included in processing. + +5. **Use a defined starting point** + ![Defining the subscription task](./assets/subscriptions_use-defined-starting-point.png) + Set the point from which the server sends the first batch. + * **Beginning of Time** (default): the server sends every document matching the query, regardless of its creation time. + * **Latest Document**: start from the first document added after the subscription is created. + * **Change Vector**: start from a specified document change vector. + +6. **Set responsible node** + ![Defining the subscription task](./assets/subscriptions_set-responsible-node.png) + Toggle **Set responsible node** to choose the responsible node yourself. + By default, the server selects the responsible node. If that node goes down and your license includes + [highly available tasks](../../server/clustering/distribution/highly-available-tasks.mdx), the server + automatically reassigns the subscription to another node. + * When you select a responsible node manually, you can also **pin** it to the task. Pinning prevents + another node from taking over if the responsible node goes down. + ![Defining the subscription task](./assets/subscriptions_set-responsible-node_pin-node.png) + +7. **Test subscription** + ![Testing the subscription](./assets/subscriptions_test-subscription.png) + Testing shows the documents that match the subscription's query, so you can verify it before saving the task. + * **A. Results limit** - The maximum number of documents to retrieve for the test. + * **B. Time limit** - The test run time, in seconds. Default: 15. + * **C. Run Test** - Click to run the test and preview the matching documents in the results table. + +Click **Save** to create the task, or **Cancel** to discard it. + + + diff --git a/docs/data-subscriptions/faq.mdx b/docs/data-subscriptions/faq.mdx new file mode 100644 index 0000000000..d44f46fabd --- /dev/null +++ b/docs/data-subscriptions/faq.mdx @@ -0,0 +1,202 @@ +--- +title: "Data subscriptions: FAQ" +sidebar_label: "FAQ" +description: "Answers to common questions about RavenDB data subscriptions: delivery guarantees, document order, failure handling, concurrent workers, permissions, and limits." +sidebar_position: 10 +--- + +import Admonition from '@theme/Admonition'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + +# Data subscriptions: FAQ + + + +* In this article: + * [Can the same document be processed more than once?](../data-subscriptions/faq.mdx#can-the-same-document-be-processed-more-than-once) + * [In what order are documents sent?](../data-subscriptions/faq.mdx#in-what-order-are-documents-sent) + * [What happens if a worker disconnects or crashes?](../data-subscriptions/faq.mdx#what-happens-if-a-worker-disconnects-or-crashes) + * [Can several workers consume the same subscription at once?](../data-subscriptions/faq.mdx#can-several-workers-consume-the-same-subscription-at-once) + * [What access level is needed to create or consume a subscription?](../data-subscriptions/faq.mdx#what-access-level-is-needed-to-create-or-consume-a-subscription) + * [What happens if the responsible node goes down?](../data-subscriptions/faq.mdx#what-happens-if-the-responsible-node-goes-down) + * [Can a subscription's query be changed after creation?](../data-subscriptions/faq.mdx#can-a-subscriptions-query-be-changed-after-creation) + * [Does a subscription stop when no documents are left?](../data-subscriptions/faq.mdx#does-a-subscription-stop-when-no-documents-are-left) + * [Can a subscription deliver document revisions?](../data-subscriptions/faq.mdx#can-a-subscription-deliver-document-revisions) + * [How is a subscription different from ETL or external replication?](../data-subscriptions/faq.mdx#how-is-a-subscription-different-from-etl-or-external-replication) + * [Is there a limit on the number of concurrent connections?](../data-subscriptions/faq.mdx#is-there-a-limit-on-the-number-of-concurrent-connections) + + + + + + + +### Can the same document be processed more than once? + +Yes. Data subscriptions provide at-least-once delivery, so a document can reach a worker more than +once in two situations: + +* A worker fails before acknowledging a batch it has processed. The server re-sends that batch the + next time a worker connects. + +* In a concurrent subscription, a worker drops, and the server reassigns the documents it was + processing to the other workers. + +Make your processing idempotent so that reprocessing a document has no unintended effect. + +See: [At-least-once delivery](../data-subscriptions/consuming-subscription.mdx#at-least-once-delivery) + + + + + +### In what order are documents sent? + +The server sends documents to the worker in Etag order, which reflects the order in which the +documents were last modified. + +A document the worker has already acknowledged is not sent again, except in these cases: + +* It was modified after the server sent it. + +* It belongs to a batch the worker received but did not acknowledge. + +* The subscription was reassigned to another node. + +See: [Processing order and re-sending](../data-subscriptions/overview.mdx#processing-order-and-re-sending) + + + + + +### What happens if a worker disconnects or crashes? + +The server keeps track of the batches the worker has acknowledged, so a worker that reconnects +continues after the last acknowledged batch instead of starting over. + +A batch the worker received but did not acknowledge is sent again, so some documents may be +processed more than once. + +See: [Progress persistence and failover](../data-subscriptions/overview.mdx#progress-persistence-and-failover) + + + + + +### Can several workers consume the same subscription at once? + +By default, only one worker consumes a subscription at a time. A worker's strategy decides whether +it connects only when free, takes over, or waits for the current worker to disconnect. + +To let several workers consume the subscription simultaneously and divide the processing load, use +the **concurrent** strategy. + +See: [Concurrent subscriptions](../data-subscriptions/concurrent-subscriptions.mdx) and +[Worker strategies](../data-subscriptions/consuming-subscription.mdx#worker-strategies) + + + + + +### What access level is needed to create or consume a subscription? + +Most ongoing tasks can be created, updated, and deleted only by a client with database admin +clearance. + +A data subscription is the exception: any client with Read/Write access to the database can create, +update, and delete one. + +Consuming a subscription requires **Read Only** access. + +See: [Security considerations](../data-subscriptions/overview.mdx#security-considerations) + + + + + +### What happens if the responsible node goes down? + +The subscription's progress is stored at the cluster level, so the subscription can continue on +another node. + +If your license includes **highly available tasks**, the cluster reassigns it to another node +automatically. Otherwise, it resumes on its original node once that node is back online. + +See: [highly available tasks](../server/clustering/distribution/highly-available-tasks.mdx) + + + + + +### Can a subscription's query be changed after creation? + +Yes. Change the subscription's definition, including its query, using the `Update` method. + +You can target the subscription by name, or by the ID the server assigned it at creation. + +See: [Updating a subscription](../data-subscriptions/creating-subscription/creating-subscription_api.mdx#updating-a-subscription) + + + + + +### Does a subscription stop when no documents are left? + +Not by default. + +When no matching documents remain, the worker waits, and the server sends new matching documents as +they arrive. + +To process every current match once and then stop, set `CloseWhenNoDocsLeft` so the worker throws a +`SubscriptionClosedException` when it finishes. + +See: [End when no documents are left](../data-subscriptions/consuming-subscription.mdx#example-end-when-no-documents-are-left) + + + + + +### Can a subscription deliver document revisions? + +Yes. When the Revisions feature is enabled for the collection, a subscription can process +consecutive pairs of document revisions instead of current documents. + +The worker can then act on what changed between revisions. + +See: [Revisions support](../data-subscriptions/revisions-support.mdx) + + + + + +### How is a subscription different from ETL or external replication? + +All three send data out of the database, but to different destinations: + +* **ETL** pushes transformed data to an external system, such as a relational database, a message + queue, or another RavenDB instance. + +* **External replication** keeps another RavenDB database in sync. + +* **A data subscription** streams matching documents to a worker in your own application, in batches + it acknowledges, while the server tracks progress so processing can pause and resume. + +See: [What is a data subscription](../data-subscriptions/overview.mdx#what-is-a-data-subscription) + + + + + +### Is there a limit on the number of concurrent connections? + +Yes. The number of concurrent connections allowed per database is capped by a configuration key that +defaults to 1000. + +The key can be set server-wide or per database. + +See: [Subscriptions.MaxNumberOfConcurrentConnections](../data-subscriptions/configuration.mdx#subscriptionsmaxnumberofconcurrentconnections) + + + + diff --git a/docs/data-subscriptions/maintenance-operations.mdx b/docs/data-subscriptions/maintenance-operations.mdx new file mode 100644 index 0000000000..4d1db9c447 --- /dev/null +++ b/docs/data-subscriptions/maintenance-operations.mdx @@ -0,0 +1,29 @@ +--- +title: "Maintenance operations" +sidebar_label: "Maintenance operations" +description: "Manage existing data subscriptions from the client API: delete, enable or disable, drop a connection, list subscriptions, and read a subscription's state." +sidebar_position: 6 +--- + +import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; +import LanguageContent from "@site/src/components/LanguageContent"; + +import MaintenanceOperationsCsharp from './content/_maintenance-operations-csharp.mdx'; +import MaintenanceOperationsJava from './content/_maintenance-operations-java.mdx'; +import MaintenanceOperationsNodejs from './content/_maintenance-operations-nodejs.mdx'; + +export const supportedLanguages = ["csharp", "java", "nodejs"]; + + + + + + + + + + + + + + diff --git a/docs/data-subscriptions/monitoring.mdx b/docs/data-subscriptions/monitoring.mdx new file mode 100644 index 0000000000..6dcba81513 --- /dev/null +++ b/docs/data-subscriptions/monitoring.mdx @@ -0,0 +1,151 @@ +--- +title: "Data subscriptions: Monitoring" +sidebar_label: "Monitoring" +description: "Monitor a running data subscription in RavenDB Studio: track its status, connected clients, and progress in the Ongoing Tasks view, and its connection activity over time in the Ongoing Task Stats graph." +sidebar_position: 7 +--- + +import Admonition from '@theme/Admonition'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + +# Monitoring a data subscription + + + +* This page explains how to monitor a running data subscription using Studio. + +* The Ongoing Tasks view shows a subscription's current status, its progress, and the clients + currently connected to it. + +* The Ongoing Task Stats graph tracks a subscription's activity over time, from each client + connection down to the individual batches and the time spent on each. + +* In this article: + * [Subscription details](../data-subscriptions/monitoring.mdx#subscription-details) + * [Status and connected clients](../data-subscriptions/monitoring.mdx#status-and-connected-clients) + * [Subscription performance](../data-subscriptions/monitoring.mdx#subscription-performance) + * [Connection event indicators](../data-subscriptions/monitoring.mdx#connection-event-indicators) + + + + + + + +Open **`Tasks` > `Ongoing Tasks`**, where subscription tasks are listed under **Subscriptions**. +Expand a subscription bar to view its current status and connected clients. + +![Find the subscription in the ongoing tasks view](./assets/subscriptions_ongoing-tasks_task-bar_collapsed.png) + + + + + +### Status and connected clients + +![Subscription details](./assets/subscriptions_ongoing-tasks_task-bar_expanded.png) + +1. **Last batch acknowledgment time** + The last time a worker acknowledged a batch. + +2. **Last client connection time** + The last time a worker connected to the subscription. + +3. **Change vector for next batch** + The point from which the server sends the next batch. Hover the info icon for the value. + +4. **Task details** + * **Node** - the node in the database group responsible for running the subscription, shown in + the database group topology. + * **Status** - whether a worker is connected: **Active** when one is, otherwise not active. + * **Error** - any error reported for the task; a dash when there are none. + * **State** - the task's overall state, shown as **OK** when the task is healthy. + +5. **Connected clients** + The workers currently consuming the subscription. + The **subscription mode** is shown (here **Concurrent**), and each connected worker is listed with + its connection strategy and client URI. Click **Disconnect** to drop an individual worker. + +For the API equivalents of disconnecting, see +[Dropping a worker](../data-subscriptions/concurrent-subscriptions.mdx#dropping-a-worker) +and [Dropping a connection](../data-subscriptions/maintenance-operations.mdx#dropping-a-connection). + + + + + + + + + +Open **`Stats` > `Ongoing Task Stats`**, where a bar on the performance timeline shows the +subscription's activity. + +![The subscription on the ongoing task stats timeline](./assets/subscriptions_task-stats_collapsed.png) + +1. **Subscription bar** + The subscription's bar on the performance timeline. Click it to open the breakdown below. + +2. **Query** + The server-side query that selects the documents. Click the eye icon to view it. + + + + + +Expanding the bar breaks each client connection into the batches it processed and the time spent +in each phase. Hover any bar for its metrics: + +![The expanded subscription bar](./assets/subscriptions_task-stats_expanded.png) + +* **Client connection** + The worker's entire connection: duration, client URI, strategy, number of batches acknowledged, + and the total size of all batches. + +* **Documents batch** + A single batch the server sent: total duration, the number and size of documents sent, and the + number and size of any included documents, counters, and time-series entries. + +* **Sending documents** + Within a batch, the time the server spent sending the documents to the worker. + +* **Waiting for ACK** + Within a batch, the time spent waiting for the worker to acknowledge the batch. + + + + + +### Connection event indicators + +In the stats view, **connection event indicators** signify events related to the initiation, state, +or termination of a worker's connection. + +Available indicators: + +| Indicator | Meaning | +|:---:|---| +| ![Connection initiated](./assets/subscriptions_task-stats_indicator-initiated.png) | **Connection initiated.** A worker has connected to the subscription. | +| ![Connection aborted](./assets/subscriptions_task-stats_indicator-aborted.png) | **Connection aborted.** The worker forcibly closed its connection. | +| ![Connection rejected](./assets/subscriptions_task-stats_indicator-rejected.png) | **Connection rejected.** A worker using the [OpenIfFree](../data-subscriptions/consuming-subscription.mdx#worker-strategies) strategy tried to connect while another worker held the subscription, so the server rejected it. | +| ![Pending connection](./assets/subscriptions_task-stats_indicator-pending.png) | **Pending connection.** A worker using the [WaitForFree](../data-subscriptions/consuming-subscription.mdx#worker-strategies) strategy tried to connect while another worker held the subscription. Instead of being rejected, it waits for the current worker to disconnect, then connects. | + +--- + +Example: + +![Connection event indicators on the timeline](./assets/subscriptions_task-stats_indicators-pending.png) + +In this example: + +1. Worker **A**, using the `OpenIfFree` strategy, connects to the subscription. +2. Worker **B**, using the `WaitForFree` strategy, attempts to connect to the same subscription and rejected. +3. Because A holds the subscription, B does not connect immediately and transitions to a **pending** state. +4. Worker **A** ends its connection with the subscription. +5. Worker **B** now connects. +6. Worker **B** aborts its connection. + + + + diff --git a/docs/data-subscriptions/overview.mdx b/docs/data-subscriptions/overview.mdx new file mode 100644 index 0000000000..a73f478503 --- /dev/null +++ b/docs/data-subscriptions/overview.mdx @@ -0,0 +1,258 @@ +--- +title: "Data subscriptions: Overview" +sidebar_label: "Overview" +description: "A data subscription is a server-side task that selects documents by a query. The server sends them in batches to client workers, which process and acknowledge each batch while the server persists the progress." +sidebar_position: 1 +--- + +import Admonition from '@theme/Admonition'; +import CodeBlock from '@theme/CodeBlock'; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; + +# Data subscriptions: Overview + + + +* A data subscription is a server-side task that selects documents by a defined query. The server + sends the selected documents in batches to consuming clients (**workers**) for processing. + +* The server sends a batch, the worker processes and acknowledges it, and only then does the + server send the next batch. + +* The server keeps track of the batches the worker has acknowledged, so a worker that stops and + reconnects to the subscription continues after the last acknowledged batch instead of starting + over. + +* In this article: + * [What is a data subscription](../data-subscriptions/overview.mdx#what-is-a-data-subscription) + * [How documents are processed](../data-subscriptions/overview.mdx#how-documents-are-processed) + * [Processing order and re-sending](../data-subscriptions/overview.mdx#processing-order-and-re-sending) + * [Progress persistence and failover](../data-subscriptions/overview.mdx#progress-persistence-and-failover) + * [How the worker communicates with the server](../data-subscriptions/overview.mdx#how-the-worker-communicates-with-the-server) + * [What the worker receives](../data-subscriptions/overview.mdx#what-the-worker-receives) + * [Consuming and managing the subscription](../data-subscriptions/overview.mdx#consuming-and-managing-the-subscription) + * [Security considerations](../data-subscriptions/overview.mdx#security-considerations) + * [Usage example](../data-subscriptions/overview.mdx#usage-example) + + + + + +A data subscription is an ongoing task that runs on the server and selects documents for clients to +process. The subscription's query defines the documents the server sends to the worker. You can +shape the query to filter the documents, project specific fields, and include related data. + +A client consumes the subscription using a **worker**. The worker connects to the subscription, +receives a batch of documents, and runs your processing code over the batch. Processing can take +anywhere from seconds to hours, depending on your code. When the worker finishes processing a batch, +it acknowledges the batch to the server, and the server then sends the next batch to the worker. + +Two definitions shape every subscription: + +* **The subscription definition**, set when the subscription is created, determines the documents + the server sends to the worker and their shape. +* **The worker options**, set when a worker connects to the subscription, determine the batch + size, the processing logic, and how the worker interacts with other workers of the same + subscription. + +To get started, create a data subscription using the +[Client API](../data-subscriptions/creating-subscription/creating-subscription_api.mdx) or +[Studio](../data-subscriptions/creating-subscription/creating-subscription_studio.mdx), and +[consume it using a worker](../data-subscriptions/consuming-subscription.mdx). + + +Data subscriptions are also supported on [sharded databases](../sharding/subscriptions.mdx), where +documents from all shards are processed in a single subscription stream. + + + + + + + + +### Processing order and re-sending + +The server sends documents to the worker in batches, ordered by each document's [Etag](../glossary/etag.mdx), +which reflects the order in which the documents were last modified. The server persists the progress +only after the worker has processed and acknowledged the whole batch. + +Because the server sends documents in Etag order, a document the worker has already acknowledged is +not sent again, except in these cases: + +* The document was modified after the server sent it. +* The document belongs to a batch the worker received but did not acknowledge. +* The subscription was reassigned to another node, for example after a node failure. The new node + cannot always locate the exact same starting point, so some documents may be processed again. + + +Plan for at-least-once delivery. Make your processing idempotent, so reprocessing a document that +the server sends again has no unintended effect. + + + + +--- + + + +### Progress persistence and failover + +The server persists each subscription's progress. This lets a subscription pause and resume, avoid +missing documents, and continue on another node after a failure: + +* Because the progress is stored on the server, you can pause the subscription and later resume it + from the last acknowledged point. +* No documents are missed even when a failure occurs, whether on the client, in the connection, or + elsewhere. +* The progress is stored at the cluster level, so it is available on all database nodes. If the + responsible node goes down, the subscription can continue on another node. + +The server uses **change vectors** to track progress, so when a subscription fails over to another +node, processing resumes close to the last acknowledged point rather than starting over. + + + + + + + +A worker communicates with its subscription over a long-lived TCP connection, using a dedicated +protocol. Each successful batch goes through these stages: + +1. The server sends a batch of documents to the worker. +2. The worker processes the batch and sends an acknowledgment to the server. +3. The server persists the acknowledgment and notifies the worker that it is ready to send the next + batch. + +The server also uses the connection to tell whether a worker is active. The connection is kept alive +and monitored with heartbeat messages. If the connection breaks, processing of the current batch +restarts. + +The diagram below traces the full lifecycle of a subscription connection, from the worker's +connection request through the processing and acknowledgment of each batch: + +![Subscription document-processing lifecycle](./assets/subscriptions_document-processing.png) + + +When the node responsible for the subscription goes down, the subscription can continue on another +node. If your license includes [highly available tasks](../server/clustering/distribution/highly-available-tasks.mdx), +the cluster reassigns the subscription to another node automatically. Otherwise, the subscription +resumes on its original node once that node is back online. + + + + + + +You can set these options while creating a subscription: + +* **Filtering** + Send to the worker only the documents that match a condition, for example only orders shipped to a + specific country. + [Filtering documents](../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-documents) + +* **Projections** + Send to the worker only selected fields, or computed values, instead of the full document. + [Projecting fields](../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-and-projecting-fields) + [Projecting from a related document](../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-projecting-from-a-related-document) + +* **Includes** + Include related documents or counters alongside each item, so the worker has them without + issuing additional requests to the server. + [Including documents](../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-documents) + [Including counters](../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters) + +* **Revisions** + Process consecutive pairs of document revisions instead of current documents, so the worker can + act on what changed between revisions. + [Revisions support](../data-subscriptions/revisions-support.mdx) + + + + + +These options and operations control how a subscription is consumed and managed: + +* **Worker strategies** + Control how a worker connects to a subscription that another worker is already consuming: connect + only if free, take over from the current worker, or wait for the current worker to disconnect. + [Worker strategies](../data-subscriptions/consuming-subscription.mdx#worker-strategies) + +* **Concurrent subscriptions** + Let multiple workers consume the same subscription at once and divide the processing load between + them to speed it up. + [Concurrent subscriptions](../data-subscriptions/concurrent-subscriptions.mdx) + +* **Maintenance operations** + You can list, update, enable, disable, and delete subscriptions, and drop their active + connections. + [Maintenance operations](../data-subscriptions/maintenance-operations.mdx) + +* **Monitoring** + Watch a running subscription's status, connected clients, and performance in Studio. + [Monitoring](../data-subscriptions/monitoring.mdx) + +* **Configuration** + Set server-wide or per-database defaults, such as how archived documents are handled and the + maximum number of concurrent connections per subscription. + [Configuration](../data-subscriptions/configuration.mdx) + + + + + +Two things shape a subscription's security: who is allowed to manage and consume it, and what its +query exposes to a consumer. + +* **Access control** + Unlike most ongoing tasks, which require database admin clearance to manage, a data subscription + can be created, updated, and deleted by any client with + [Read/Write access](../server/security/authorization/security-clearance-and-permissions.mdx#readwrite) + to the database, and consumed by any client with + [Read Only access](../server/security/authorization/security-clearance-and-permissions.mdx#read-only). + There is no permission specific to the subscription itself. + +* **Connection security** + On a secure server, a worker authenticates with a + [client certificate](../server/security/authentication/client-certificate-usage.mdx), and its + connection to the server is encrypted like any other client connection. + +* **Data exposure** + Because a subscription's filter, projection, and includes run on the server, a consumer receives + whatever the query exposes: not only the matching documents but also any related documents and + counters the query loads alongside them. Define the query to expose only the data its consumers + are meant to see. + + + + + +The following example creates a subscription for a specific company's orders, then runs a worker +that processes the documents the server sends. + +```csharp +public async Task ConsumeSubscription(IDocumentStore store, CancellationToken cancellationToken) +{ + // Create the subscription task on the server. + // The server will send every Order placed by company "companies/11". + string subscriptionName = await store.Subscriptions + .CreateAsync(order => order.Company == "companies/11"); + + // Create a worker on the client to consume the subscription + SubscriptionWorker worker = + store.Subscriptions.GetSubscriptionWorker(subscriptionName); + + // Run the worker and process each batch the server sends + await worker.Run(batch => + { + foreach (var item in batch.Items) + { + Console.WriteLine($"Order {item.Result.Id} will be shipped via {item.Result.ShipVia}"); + } + }, cancellationToken); +} +``` + + diff --git a/docs/data-subscriptions/revisions-support.mdx b/docs/data-subscriptions/revisions-support.mdx new file mode 100644 index 0000000000..9949baad55 --- /dev/null +++ b/docs/data-subscriptions/revisions-support.mdx @@ -0,0 +1,29 @@ +--- +title: "Revisions support" +sidebar_label: "Revisions support" +description: "Subscribe to a collection's document revisions instead of its documents, so a worker receives consecutive revision pairs and can filter and project over the changes." +sidebar_position: 5 +--- + +import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; +import LanguageContent from "@site/src/components/LanguageContent"; + +import RevisionsSupportCsharp from './content/_revisions-support-csharp.mdx'; +import RevisionsSupportJava from './content/_revisions-support-java.mdx'; +import RevisionsSupportNodejs from './content/_revisions-support-nodejs.mdx'; + +export const supportedLanguages = ["csharp", "java", "nodejs"]; + + + + + + + + + + + + + + diff --git a/docs/document-extensions/attachments/attachments-and-other-features.mdx b/docs/document-extensions/attachments/attachments-and-other-features.mdx index 583f20e6bd..b57e0c86cd 100644 --- a/docs/document-extensions/attachments/attachments-and-other-features.mdx +++ b/docs/document-extensions/attachments/attachments-and-other-features.mdx @@ -21,7 +21,7 @@ import Panel from '@site/src/components/Panel'; * This metadata update triggers any features that respond to document changes - such as [Indexing](../../studio/database/indexes/indexes-overview.mdx), any defined [ongoing tasks](../../studio/database/tasks/ongoing-tasks/general-info.mdx), - [Subscriptions](../../client-api/data-subscriptions/what-are-data-subscriptions.mdx), etc. + [Subscriptions](../../data-subscriptions/overview.mdx), etc. * In this article: * [Attachments & Replication](../../document-extensions/attachments/attachments-and-other-features.mdx#attachments-replication) diff --git a/docs/document-extensions/counters/content/_counters-and-other-features-csharp.mdx b/docs/document-extensions/counters/content/_counters-and-other-features-csharp.mdx index 863dfb6f88..259ee95125 100644 --- a/docs/document-extensions/counters/content/_counters-and-other-features-csharp.mdx +++ b/docs/document-extensions/counters/content/_counters-and-other-features-csharp.mdx @@ -187,7 +187,7 @@ Each [ongoing task](../../../studio/database/tasks/ongoing-tasks/general-info.md * Counters can be [exported using a script](../../../server/ongoing-tasks/etl/raven.mdx#adding-counter-explicitly-in-a-script). **Default behavior**: When an ETL script is not provided, Counters are exported. -* **Counters** and the **[Data Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscriptions) task** +* **Counters** and the **[Data Subscriptions](../../../data-subscriptions/overview.mdx) task** Data Subscriptions can be initiated by document changes, including those caused by **Counter Name updates**. Documents will **not** be delivered in reaction to Counter Value modification. diff --git a/docs/document-extensions/counters/content/_counters-and-other-features-java.mdx b/docs/document-extensions/counters/content/_counters-and-other-features-java.mdx index 70d4cd51cd..864f2d545b 100644 --- a/docs/document-extensions/counters/content/_counters-and-other-features-java.mdx +++ b/docs/document-extensions/counters/content/_counters-and-other-features-java.mdx @@ -182,7 +182,7 @@ Each [ongoing task](../../../studio/database/tasks/ongoing-tasks/general-info.md * Counters can be [exported using a script](../../../server/ongoing-tasks/etl/raven.mdx#adding-counter-explicitly-in-a-script). **Default behavior**: When an ETL script is not provided, Counters are exported. -* **Counters** and the **[Data Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscriptions) task** +* **Counters** and the **[Data Subscriptions](../../../data-subscriptions/overview.mdx) task** Data Subscriptions can be initiated by document changes, including those caused by **Counter Name updates**. Documents will **not** be delivered in reaction to Counter Value modification. diff --git a/docs/document-extensions/counters/content/_counters-and-other-features-nodejs.mdx b/docs/document-extensions/counters/content/_counters-and-other-features-nodejs.mdx index 09be7ff504..ac1992cf0b 100644 --- a/docs/document-extensions/counters/content/_counters-and-other-features-nodejs.mdx +++ b/docs/document-extensions/counters/content/_counters-and-other-features-nodejs.mdx @@ -183,7 +183,7 @@ Each [ongoing task](../../../studio/database/tasks/ongoing-tasks/general-info.md * Counters can be [exported using a script](../../../server/ongoing-tasks/etl/raven.mdx#adding-counter-explicitly-in-a-script). **Default behavior**: When an ETL script is not provided, Counters are exported. -* **Counters** and the **[Data Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscriptions) task** +* **Counters** and the **[Data Subscriptions](../../../data-subscriptions/overview.mdx) task** Data Subscriptions can be initiated by document changes, including those caused by **Counter Name updates**. Documents will **not** be delivered in reaction to Counter Value modification. diff --git a/docs/document-extensions/counters/content/_counters-and-other-features-python.mdx b/docs/document-extensions/counters/content/_counters-and-other-features-python.mdx index 891730050c..458fedc745 100644 --- a/docs/document-extensions/counters/content/_counters-and-other-features-python.mdx +++ b/docs/document-extensions/counters/content/_counters-and-other-features-python.mdx @@ -143,7 +143,7 @@ Each [ongoing task](../../../studio/database/tasks/ongoing-tasks/general-info.md * Counters can be [exported using a script](../../../server/ongoing-tasks/etl/raven.mdx#adding-counter-explicitly-in-a-script). **Default behavior**: When an ETL script is not provided, Counters are exported. -* **Counters** and the **[Data Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx#data-subscriptions) task** +* **Counters** and the **[Data Subscriptions](../../../data-subscriptions/overview.mdx) task** Data Subscriptions can be initiated by document changes, including those caused by **Counter Name updates**. Documents will **not** be delivered in reaction to Counter Value modification. diff --git a/docs/document-extensions/counters/content/_including-counters-csharp.mdx b/docs/document-extensions/counters/content/_including-counters-csharp.mdx index 5e151c5ee0..9eb03d03b8 100644 --- a/docs/document-extensions/counters/content/_including-counters-csharp.mdx +++ b/docs/document-extensions/counters/content/_including-counters-csharp.mdx @@ -14,7 +14,7 @@ import Panel from '@site/src/components/Panel'; so their values can be accessed later in the same session without making additional requests to the server. * To see how to include counters when creating a subscription, see - [Create subscription - include counters](../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). + [Create subscription - include counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters). * In this article: * [Sample data](../../../document-extensions/counters/including-counters.mdx#sample-data) diff --git a/docs/document-extensions/counters/content/_including-counters-java.mdx b/docs/document-extensions/counters/content/_including-counters-java.mdx index baf4489657..f1e07d17bd 100644 --- a/docs/document-extensions/counters/content/_including-counters-java.mdx +++ b/docs/document-extensions/counters/content/_including-counters-java.mdx @@ -12,7 +12,7 @@ import CodeBlock from '@theme/CodeBlock'; so their values can be accessed later in the same session without making additional requests to the server. * To see how to include counters when creating a subscription, see - [Create subscription - include counters](../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). + [Create subscription - include counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters). diff --git a/docs/document-extensions/counters/content/_including-counters-nodejs.mdx b/docs/document-extensions/counters/content/_including-counters-nodejs.mdx index fb594c3473..9cf5b7f72f 100644 --- a/docs/document-extensions/counters/content/_including-counters-nodejs.mdx +++ b/docs/document-extensions/counters/content/_including-counters-nodejs.mdx @@ -14,7 +14,7 @@ import Panel from '@site/src/components/Panel'; so their values can be accessed later in the same session without making additional requests to the server. * To see how to include counters when creating a subscription, see - [Create subscription - include counters](../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). + [Create subscription - include counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters). * In this article: * [Sample data](../../../document-extensions/counters/including-counters.mdx#sample-data) diff --git a/docs/document-extensions/counters/content/_including-counters-php.mdx b/docs/document-extensions/counters/content/_including-counters-php.mdx index 181206c2ec..374b9df846 100644 --- a/docs/document-extensions/counters/content/_including-counters-php.mdx +++ b/docs/document-extensions/counters/content/_including-counters-php.mdx @@ -12,7 +12,7 @@ import CodeBlock from '@theme/CodeBlock'; so their values can be accessed later in the same session without making additional requests to the server. * To see how to include counters when creating a subscription, see - [Create subscription - include counters](../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). + [Create subscription - include counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters). diff --git a/docs/document-extensions/counters/content/_including-counters-python.mdx b/docs/document-extensions/counters/content/_including-counters-python.mdx index 7960886750..ae1a460971 100644 --- a/docs/document-extensions/counters/content/_including-counters-python.mdx +++ b/docs/document-extensions/counters/content/_including-counters-python.mdx @@ -12,7 +12,7 @@ import CodeBlock from '@theme/CodeBlock'; so their values can be accessed later in the same session without making additional requests to the server. * To see how to include counters when creating a subscription, see - [Create subscription - include counters](../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription---include-counters). + [Create subscription - include counters](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-including-counters). diff --git a/docs/document-extensions/revisions/content/_revisions-and-other-features-csharp.mdx b/docs/document-extensions/revisions/content/_revisions-and-other-features-csharp.mdx index e5f4cd51a5..a215e3229c 100644 --- a/docs/document-extensions/revisions/content/_revisions-and-other-features-csharp.mdx +++ b/docs/document-extensions/revisions/content/_revisions-and-other-features-csharp.mdx @@ -245,7 +245,7 @@ Revisions are [backed up](../../../backup/overview.mdx#backup-content) both by a ## Revisions and Data Subscriptions -* Learn about revisions and data subscriptions [here](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx). +* Learn about revisions and data subscriptions [here](../../../data-subscriptions/revisions-support.mdx). diff --git a/docs/document-extensions/revisions/content/_revisions-and-other-features-nodejs.mdx b/docs/document-extensions/revisions/content/_revisions-and-other-features-nodejs.mdx index c6cdd003bc..50dbfc0cda 100644 --- a/docs/document-extensions/revisions/content/_revisions-and-other-features-nodejs.mdx +++ b/docs/document-extensions/revisions/content/_revisions-and-other-features-nodejs.mdx @@ -283,7 +283,7 @@ When a document is [reverted](../../../document-extensions/revisions/revert-docu -* Learn about revisions and data subscriptions [here](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx). +* Learn about revisions and data subscriptions [here](../../../data-subscriptions/revisions-support.mdx). diff --git a/docs/document-extensions/revisions/content/_revisions-and-other-features-python.mdx b/docs/document-extensions/revisions/content/_revisions-and-other-features-python.mdx index 7101843df4..424cf131e4 100644 --- a/docs/document-extensions/revisions/content/_revisions-and-other-features-python.mdx +++ b/docs/document-extensions/revisions/content/_revisions-and-other-features-python.mdx @@ -206,7 +206,7 @@ Revisions are [backed up](../../../backup/overview.mdx#backup-content) both by a ## Revisions and Data Subscriptions -* Learn about revisions and data subscriptions [here](../../../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx). +* Learn about revisions and data subscriptions [here](../../../data-subscriptions/revisions-support.mdx). diff --git a/docs/document-extensions/revisions/revisions-and-other-features.mdx b/docs/document-extensions/revisions/revisions-and-other-features.mdx index a8299ab261..47b7aa458f 100644 --- a/docs/document-extensions/revisions/revisions-and-other-features.mdx +++ b/docs/document-extensions/revisions/revisions-and-other-features.mdx @@ -46,11 +46,11 @@ see_also: source: "docs" path: "Studio > Database Management > Document Extensions > Revisions" - title: "What Are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" + link: "data-subscriptions/overview" source: "docs" path: "Client API > Data Subscriptions" - title: "Revisions and Data Subscriptions" - link: "client-api/data-subscriptions/advanced-topics/subscription-with-revisioning" + link: "data-subscriptions/revisions-support" source: "docs" path: "Client API > Data Subscriptions > Advanced topics" --- diff --git a/docs/document-extensions/timeseries/content/_time-series-and-other-features-csharp.mdx b/docs/document-extensions/timeseries/content/_time-series-and-other-features-csharp.mdx index 3faaf40cfb..0a6fa6d2b9 100644 --- a/docs/document-extensions/timeseries/content/_time-series-and-other-features-csharp.mdx +++ b/docs/document-extensions/timeseries/content/_time-series-and-other-features-csharp.mdx @@ -62,7 +62,7 @@ Some of these apply to time series data, while others do not. #### Tasks that cannot be applied to time series * [SQL ETL](../../../server/ongoing-tasks/etl/basics.mdx), another type of ETL that can set a relational database as its target. -* [Data Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) send data to "worker" clients in batches. +* [Data Subscriptions](../../../data-subscriptions/overview.mdx) send data to "worker" clients in batches. diff --git a/docs/document-extensions/timeseries/content/_time-series-and-other-features-nodejs.mdx b/docs/document-extensions/timeseries/content/_time-series-and-other-features-nodejs.mdx index 9ee3ed9c2f..38eff24852 100644 --- a/docs/document-extensions/timeseries/content/_time-series-and-other-features-nodejs.mdx +++ b/docs/document-extensions/timeseries/content/_time-series-and-other-features-nodejs.mdx @@ -63,7 +63,7 @@ Some of these apply to time series data, while others do not. #### Tasks that cannot be applied to time series * [SQL ETL](../../../server/ongoing-tasks/etl/basics.mdx), another type of ETL that can set a relational database as its target. -* [Data Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) send data to "worker" clients in batches. +* [Data Subscriptions](../../../data-subscriptions/overview.mdx) send data to "worker" clients in batches. diff --git a/docs/migration/client-api/previous-versions-client-breaking-changes.mdx b/docs/migration/client-api/previous-versions-client-breaking-changes.mdx index 7b3b533f17..dd56fe88b1 100644 --- a/docs/migration/client-api/previous-versions-client-breaking-changes.mdx +++ b/docs/migration/client-api/previous-versions-client-breaking-changes.mdx @@ -49,7 +49,7 @@ Each breaking change indicates a behavior change or a function that is no longer which could cause errors and confusion. * To eliminate this ambiguity, starting from **7.0**, the `Create` overload for predicate-based subscriptions now accepts `PredicateSubscriptionCreationOptions`, which no longer includes a `Query` property. -* Refer to the [Subscription creation API overview](../../client-api/data-subscriptions/creation/api-overview.mdx) for the complete list of available `Create` method overloads. +* Refer to the [Subscription creation API overview](../../data-subscriptions/creating-subscription/creating-subscription_api.mdx) for the complete list of available `Create` method overloads. diff --git a/docs/querying/filtering-query-results/content/_filter-by-date-and-time-csharp.mdx b/docs/querying/filtering-query-results/content/_filter-by-date-and-time-csharp.mdx index 9c706bdb39..6b26d5d8a3 100644 --- a/docs/querying/filtering-query-results/content/_filter-by-date-and-time-csharp.mdx +++ b/docs/querying/filtering-query-results/content/_filter-by-date-and-time-csharp.mdx @@ -39,7 +39,7 @@ import Panel from '@site/src/components/Panel'; In the C# examples below, [DateTimeKind](https://learn.microsoft.com/dotnet/api/system.datetimekind) is passed to the `DateTime` constructor to mark the value as UTC. * Unlike the [Restrictions](../../../querying/filtering-query-results/filter-by-date-and-time.mdx#restrictions) on `now()` and `today()`, - explicit date values are deterministic and **can be used** in [Exploration queries](../../../indexes/querying/exploration-queries.mdx) and [Subscription queries](../../../client-api/data-subscriptions/creation/examples.mdx). + explicit date values are deterministic and **can be used** in [Exploration queries](../../../indexes/querying/exploration-queries.mdx) and [Subscription queries](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx). --- @@ -826,7 +826,7 @@ where RequireAt <= now($offset) --- * **Subscription queries** - Both `now()` and `today()` are not supported in [subscription queries](../../../client-api/data-subscriptions/creation/examples.mdx). + Both `now()` and `today()` are not supported in [subscription queries](../../../data-subscriptions/creating-subscription/creating-subscription_api.mdx). For example, the following query is not allowed: diff --git a/docs/querying/rql/rql-code-assistance.mdx b/docs/querying/rql/rql-code-assistance.mdx index c7fc9aa006..da916b80de 100644 --- a/docs/querying/rql/rql-code-assistance.mdx +++ b/docs/querying/rql/rql-code-assistance.mdx @@ -29,7 +29,7 @@ import ContentFrame from "@site/src/components/ContentFrame"; * **RQL Code Assistance** is available whenever you write a query in the following Studio views: * [Query view](../../studio/database/queries/query-view.mdx#query-view) * [Patch view](../../studio/database/documents/patch-view.mdx#patch-view) - * [Subscription task view](../../studio/database/tasks/ongoing-tasks/subscription-task.mdx#subscription-task-definition) + * [Subscription task view](../../data-subscriptions/creating-subscription/creating-subscription_studio.mdx#defining-the-subscription-task) --- diff --git a/docs/server/configuration/subscription-configuration.mdx b/docs/server/configuration/subscription-configuration.mdx deleted file mode 100644 index d52a001d98..0000000000 --- a/docs/server/configuration/subscription-configuration.mdx +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: "Configuration: Subscriptions" -sidebar_label: Subscription Configuration -description: "Configure RavenDB data subscription settings including batch sizes, connection timeout intervals, and concurrent subscription limits." -sidebar_position: 24 ---- - -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -# Configuration: Subscriptions - - -* The following configuration keys control various aspects of subscription behavior in RavenDB. - Learn more about subscriptions in [Data subscriptions](../../client-api/data-subscriptions/what-are-data-subscriptions.mdx). - -* In this article: - * [Subscriptions.ArchivedDataProcessingBehavior](../../server/configuration/subscription-configuration.mdx#subscriptionsarchiveddataprocessingbehavior) - * [Subscriptions.MaxNumberOfConcurrentConnections](../../server/configuration/subscription-configuration.mdx#subscriptionsmaxnumberofconcurrentconnections) - - - -## Subscriptions.ArchivedDataProcessingBehavior - -The default processing behavior for archived documents in a subscription query. - -- **Type**: `enum ArchivedDataProcessingBehavior`: - * `ExcludeArchived`: only non-archived documents are processed by the subscription query. - * `IncludeArchived`: both archived and non-archived documents are processed by the subscription query. - * `ArchivedOnly`: only archived documents are processed by the subscription query. -- **Default**: `ExcludeArchived` -- **Scope**: Server-wide, or per database - - - -## Subscriptions.MaxNumberOfConcurrentConnections - -The maximum number of concurrent subscription connections allowed per database. - -- **Type**: `int` -- **Default**: `1000` -- **Scope**: Server-wide or per database - - diff --git a/docs/server/extensions/content/_refresh-csharp.mdx b/docs/server/extensions/content/_refresh-csharp.mdx index 7b5e51c496..31aa2e25d0 100644 --- a/docs/server/extensions/content/_refresh-csharp.mdx +++ b/docs/server/extensions/content/_refresh-csharp.mdx @@ -48,7 +48,7 @@ import CodeBlock from '@theme/CodeBlock'; This triggers any features that react to document updating, including but not limited to: - Re-indexing of the document by indexes that cover it - [Replication](../../../server/ongoing-tasks/external-replication.mdx), - [Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx), + [Subscriptions](../../../data-subscriptions/overview.mdx), and [ETL](../../../server/ongoing-tasks/etl/basics.mdx) triggering - Creation of a [document revision](../../../document-extensions/revisions/overview.mdx) diff --git a/docs/server/kb/javascript-engine.mdx b/docs/server/kb/javascript-engine.mdx index cb97d5cfa1..223c6fab35 100644 --- a/docs/server/kb/javascript-engine.mdx +++ b/docs/server/kb/javascript-engine.mdx @@ -18,7 +18,7 @@ import ContentFrame from "@site/src/components/ContentFrame"; * RavenDB integrates **JavaScript scripting** across various features, including: [RQL projections](../../indexes/querying/projections.mdx), - [Subscriptions](../../client-api/data-subscriptions/creation/examples.mdx#create-subscription-with-filtering-and-projection), + [Subscriptions](../../data-subscriptions/creating-subscription/creating-subscription_api.mdx#example-filtering-and-projecting-fields), [ETL](../../server/ongoing-tasks/etl/basics.mdx), [Smuggler (data import/export)](../../client-api/smuggler/what-is-smuggler.mdx#transformscript), [Single](../../client-api/operations/patching/single-document.mdx) or [Set based](../../client-api/operations/patching/set-based.mdx) document patches, diff --git a/docs/server/ongoing-tasks/external-replication.mdx b/docs/server/ongoing-tasks/external-replication.mdx index eb442b619b..9b944a91bc 100644 --- a/docs/server/ongoing-tasks/external-replication.mdx +++ b/docs/server/ongoing-tasks/external-replication.mdx @@ -56,7 +56,7 @@ import LanguageContent from "@site/src/components/LanguageContent"; * [Indexes](../../indexes/creating-and-deploying.mdx) * [Conflict resolution scripts](../../server/clustering/replication/replication-conflicts.mdx#conflict-resolution-script) * [Compare-Exchange](../../compare-exchange/overview.mdx) - * [Subscriptions](../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + * [Subscriptions](../../data-subscriptions/overview.mdx) * [Identities](../../server/kb/document-identifier-generation.mdx#identity-id) * Ongoing tasks * [ETL](../../server/ongoing-tasks/etl/basics.mdx) diff --git a/docs/server/ongoing-tasks/hub-sink-replication.mdx b/docs/server/ongoing-tasks/hub-sink-replication.mdx index cfe147a6cc..a17034a447 100644 --- a/docs/server/ongoing-tasks/hub-sink-replication.mdx +++ b/docs/server/ongoing-tasks/hub-sink-replication.mdx @@ -73,7 +73,7 @@ After the data is in the destination server, setting up a hub/sink replication o * [Indexes](../../indexes/creating-and-deploying.mdx) * [Conflict resolver definitions](../../server/clustering/replication/replication-conflicts.mdx#conflict-resolution-script) * [Compare-Exchange](../../compare-exchange/overview.mdx) - * [Subscriptions](../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + * [Subscriptions](../../data-subscriptions/overview.mdx) * [Identities](../../server/kb/document-identifier-generation.mdx#identity-id) * Ongoing tasks * [ETL](../../server/ongoing-tasks/etl/basics.mdx) diff --git a/docs/server/tcp-compression.mdx b/docs/server/tcp-compression.mdx index 4c1acf5818..0dc5218aa6 100644 --- a/docs/server/tcp-compression.mdx +++ b/docs/server/tcp-compression.mdx @@ -19,7 +19,7 @@ import LanguageContent from "@site/src/components/LanguageContent"; are compressed, including nodes' [database replication](../server/clustering/replication/replication-overview.mdx) and the data submitted by - [Data Subscriptions](../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + [Data Subscriptions](../data-subscriptions/overview.mdx) to their workers. * Especially on the cloud, the significant reduction in the amount of transferred @@ -46,7 +46,7 @@ With the **TCP compression** feature enabled, data **in transit** is compressed Replication makes a large portion of a cluster's traffic, and compressing replicated data will minimize the traffic volume and expedite data delivery. -* [Data Subscriptions](../client-api/data-subscriptions/what-are-data-subscriptions.mdx) +* [Data Subscriptions](../data-subscriptions/overview.mdx) will also transfer compressed data to their workers. Data subscriptions are used to automate documents processing, and may transfer large quantities of documents on a regular basis. diff --git a/docs/sharding/subscriptions.mdx b/docs/sharding/subscriptions.mdx index 3711a48ca2..4021827e2f 100644 --- a/docs/sharding/subscriptions.mdx +++ b/docs/sharding/subscriptions.mdx @@ -15,8 +15,8 @@ import LanguageContent from "@site/src/components/LanguageContent"; # Sharding: Data Subscriptions -* From a user's point of view, [Data Subscriptions](../client-api/data-subscriptions/what-are-data-subscriptions.mdx) - are created and [consumed](../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx) +* From a user's point of view, [Data Subscriptions](../data-subscriptions/overview.mdx) + are created and [consumed](../data-subscriptions/consuming-subscription.mdx) in the exact same way when the database is sharded and when it is not. * Old clients are supported and can continue communicating with RavenDB @@ -37,7 +37,7 @@ import LanguageContent from "@site/src/components/LanguageContent"; To allow data subscriptions in a sharded database: * From a user's point of view, creating a data subscription is - done once, [just like it is done](../client-api/data-subscriptions/creation/how-to-create-data-subscription.mdx) + done once, [just like it is done](../data-subscriptions/creating-subscription/creating-subscription_api.mdx) under a non-sharded database. * Behind the scenes, though, the [orchestrator](../sharding/overview.mdx#client-server-communication) that was appointed to handle this client and received its @@ -91,11 +91,11 @@ how large the overall database gets. Data subscriptions features that are not supported yet under a sharded database: -* [Concurrent Subscriptions](../client-api/data-subscriptions/concurrent-subscriptions.mdx) +* [Concurrent Subscriptions](../data-subscriptions/concurrent-subscriptions.mdx) Allowing multiple workers to connect a common subscription simultaneously. -* [Data Subscriptions Revisions Support](../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx) +* [Data Subscriptions Revisions Support](../data-subscriptions/revisions-support.mdx) Subscribing to document revisions. -* [SubscriptionCreationOptions.ChangeVector](../client-api/data-subscriptions/creation/api-overview.mdx#subscriptioncreationoptionst) +* [SubscriptionCreationOptions.ChangeVector](../data-subscriptions/creating-subscription/creating-subscription_api.mdx#classes) Providing an arbitrary change vector from which the subscription would start processing is currently not supported. diff --git a/docs/sharding/unsupported.mdx b/docs/sharding/unsupported.mdx index 1e641c3d6c..5e10c0ac00 100644 --- a/docs/sharding/unsupported.mdx +++ b/docs/sharding/unsupported.mdx @@ -99,8 +99,8 @@ import LanguageContent from "@site/src/components/LanguageContent"; | Unsupported Feature | Comment | | ------------- | ------------- | -| [Concurrent Subscriptions](../client-api/data-subscriptions/concurrent-subscriptions.mdx) | | -| [Data Subscriptions Revisions Support](../client-api/data-subscriptions/advanced-topics/subscription-with-revisioning.mdx) | Subscribing to document revisions | +| [Concurrent Subscriptions](../data-subscriptions/concurrent-subscriptions.mdx) | | +| [Data Subscriptions Revisions Support](../data-subscriptions/revisions-support.mdx) | Subscribing to document revisions | | [SubscriptionCreationOptions.ChangeVector](../sharding/subscriptions.mdx#unsupported-features) | Providing a change vector to start the processing from is not supported
except for these special cases:
`"LastDocument"`, `"BeginningOfTime"`, `"DoNotChange"` | ## Unsupported Integrations Features diff --git a/docs/studio/database/indexes/create-map-reduce-index.mdx b/docs/studio/database/indexes/create-map-reduce-index.mdx index 6ec048f5ac..b2827edee0 100644 --- a/docs/studio/database/indexes/create-map-reduce-index.mdx +++ b/docs/studio/database/indexes/create-map-reduce-index.mdx @@ -196,7 +196,7 @@ to edit them manually since any index results update would overwrite all manual * You can set up a [RavenDB ETL Task](../../../studio/database/tasks/ongoing-tasks/ravendb-etl-task.mdx) on the Artificial Documents collection to a dedicated database on a separate cluster for further processing, - as well as other ongoing tasks such as: [SQL ETL](../../../server/ongoing-tasks/etl/sql.mdx) and [Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx). + as well as other ongoing tasks such as: [SQL ETL](../../../server/ongoing-tasks/etl/sql.mdx) and [Subscriptions](../../../data-subscriptions/overview.mdx). @@ -217,7 +217,7 @@ to edit them manually since any index results update would overwrite all manual each node in the database group has its own (independent) copy of the index results. Therefore: - 1. It is recommended to use artificial documents with [Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) only on a _single_ node. + 1. It is recommended to use artificial documents with [Subscriptions](../../../data-subscriptions/overview.mdx) only on a _single_ node. A Subscription failover to another node may cause the subscription to send Artificial Documents that the subscription has already acknowledged. diff --git a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-11-data-subscription-closed-view.png b/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-11-data-subscription-closed-view.png deleted file mode 100644 index d87ccef8dd..0000000000 Binary files a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-11-data-subscription-closed-view.png and /dev/null differ diff --git a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-12-data-subscription-expanded-view.png b/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-12-data-subscription-expanded-view.png deleted file mode 100644 index d2a7e3636d..0000000000 Binary files a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-12-data-subscription-expanded-view.png and /dev/null differ diff --git a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-connection-initiated.png b/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-connection-initiated.png deleted file mode 100644 index 566829b9be..0000000000 Binary files a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-connection-initiated.png and /dev/null differ diff --git a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-forcibly-aborted-connection.png b/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-forcibly-aborted-connection.png deleted file mode 100644 index fb6f272b5a..0000000000 Binary files a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-forcibly-aborted-connection.png and /dev/null differ diff --git a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-pending-connection.png b/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-pending-connection.png deleted file mode 100644 index 32cd038e73..0000000000 Binary files a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-pending-connection.png and /dev/null differ diff --git a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-rejected-connection.png b/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-rejected-connection.png deleted file mode 100644 index 63379d7449..0000000000 Binary files a/docs/studio/database/stats/ongoing-tasks-stats/assets/stats-view-rejected-connection.png and /dev/null differ diff --git a/docs/studio/database/stats/ongoing-tasks-stats/subscription-stats.mdx b/docs/studio/database/stats/ongoing-tasks-stats/subscription-stats.mdx deleted file mode 100644 index bd22d9eef0..0000000000 --- a/docs/studio/database/stats/ongoing-tasks-stats/subscription-stats.mdx +++ /dev/null @@ -1,110 +0,0 @@ ---- -title: "Ongoing Tasks Stats: Subscription Stats" -sidebar_label: Subscription Stats -description: "Monitor RavenDB data subscription performance in Studio with connection tracking, batch statistics, and client processing metrics." -sidebar_position: 2 ---- - -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -# Ongoing Tasks Stats: Subscription Stats - - -* A **Subscription Task** is an ongoing task in which the server sends documents that match a query defined on - the task to a client. -* The server sends the documents in batches in an orderly manner, waiting for the client to acknowledge the - consumption of each batch before sending the next. -* Learn more about data subscription tasks [here](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx). - -* In this page: - * [Subscription Stats Closed View](../../../../studio/database/stats/ongoing-tasks-stats/subscription-stats.mdx#subscription-stats-closed-view) - * [Subscription Stats Expanded View](../../../../studio/database/stats/ongoing-tasks-stats/subscription-stats.mdx#subscription-stats-expanded-view) - * [Connection Event Indicators](../../../../studio/database/stats/ongoing-tasks-stats/subscription-stats.mdx#connection-event-indicators) - - -## Subscription Stats - -### Subscription Stats Closed View - -![Subscription Stats Closed View](./assets/stats-view-11-data-subscription-closed-view.png) - -1. **Task Type** - Click the arrow or the task type to toggle Closed/Expanded View. -2. **Task Name** -3. **Query** - Click to display the subscription query. -4. **Task Bar** - * Hover over the bar to display a tooltip with the task's information. - * Click the bar for the expanded view. - * Click and Drag the bar to slide the graph. - * Zoom in & out using the mouse wheel. -### Subscription Stats Expanded View - -![Subscription Stats Expanded View](./assets/stats-view-12-data-subscription-expanded-view.png) - -* **Client Connection** - * **Duration** - Overall connection time. - * **Client URL** - * **Strategy** - The [strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-interplay) used by the client. - * **Number of batches acknowledged** - The number of batches whose processing was completed & confirmed by the client. - * **Size of all batches** - The total size of all batches sent in this connection. -* **Documents Batch** - * **Total duration** - The amount of time it took to deliver this batch. - * **Documents sent in batch** - The number of documents delivered in this batch. - * **Documents size** - Overall size of Transferred documents. - * **Included Documents** - **Included Counters** - **Included Time Series entries** - Number of [included](../../../../client-api/data-subscriptions/creation/examples.mdx#create-subscription-with-include-statement) - documents, counters, and time series -* **Sending Documents** - The time it took the server to send the documents in this batch. -* **Waiting for ACK** - The time it took the client to acknowledge this batch. -### Connection Event Indicators - -Connection Event Indicators signify events related to the initiation, state, -or termination of a client connection. - -* Indicators are shown in both the [Closed](../../../../studio/database/stats/ongoing-tasks-stats/overview.mdx#task-rulers-closed-view) - and the [Expanded](../../../../studio/database/stats/ongoing-tasks-stats/overview.mdx#task-rulers-expanded-view) - task views. -* Hovering over a connection event indicator will popup information related to it, if available. -**Connection Initiated** -![Connnection Initiated](./assets/stats-view-connection-initiated.png) -**Connection Aborted** -![Forcibly-Aborted Connnection](./assets/stats-view-forcibly-aborted-connection.png) -The client has forcibly aborted the connection. -**Connection Rejected** -![Rejected Connnection](./assets/stats-view-rejected-connection.png) -A subscription task handles only a single client/connection at a time. -A client that uses the *OpenIfFree* [strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-interplay) -has attempted to open a connection to the subscription task but was rejected -because the task is already occupied by another client. -**Pending Connection** -![Pending Connnection](./assets/stats-view-pending-connection.png) - -1. Client **A** connects to the subscription task. -2. Client **B**, using the *WaitForFree* [strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#worker-interplay), - attempts to connect the task. - Discovering that the task is occupied, Client **B** is now pending, - waiting for Client **A** to disconnect. - The dotted yellow line shows the pending period for Client **B**. -3. Client **A** disconnects from the subscription task. -4. Client **B** starts a new connection to the subscription task. - - - - diff --git a/docs/studio/database/tasks/export-database.mdx b/docs/studio/database/tasks/export-database.mdx index a7012658e6..0a629bf542 100644 --- a/docs/studio/database/tasks/export-database.mdx +++ b/docs/studio/database/tasks/export-database.mdx @@ -51,7 +51,7 @@ and export it to a `.ravendbdump` file. You can remove [Analyzers](../../../indexes/using-analyzers.mdx) from exported indexes. * [Identities](../../../client-api/document-identifiers/working-with-document-identifiers.mdx) * [Compare Exchange](../../../compare-exchange/overview.mdx) - * [Subscriptions](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + * [Subscriptions](../../../data-subscriptions/overview.mdx) * [Configuration and Ongoing Tasks](../../../studio/database/tasks/import-data/import-from-ravendb.mdx#customize-configuration-and-ongoing-tasks) 4. **Include Artificial Documents** Toggle on to export [Artificial Documents](../../../studio/database/indexes/create-map-reduce-index.mdx#artificial-documents--vs--regular-documents). diff --git a/docs/studio/database/tasks/import-data/import-data-file.mdx b/docs/studio/database/tasks/import-data/import-data-file.mdx index ff8eea4057..93546a5db3 100644 --- a/docs/studio/database/tasks/import-data/import-data-file.mdx +++ b/docs/studio/database/tasks/import-data/import-data-file.mdx @@ -90,7 +90,7 @@ Here you can filter the data you want to import, select configuration and apply - [Remove Analyzers](../../../../indexes/using-analyzers.mdx) - [Include Identities](../../../../client-api/document-identifiers/working-with-document-identifiers.mdx) - [Include Compare Exchange](../../../../compare-exchange/overview.mdx) - - [Include Subscriptions](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + - [Include Subscriptions](../../../../data-subscriptions/overview.mdx) - [Include Configuration and OngoingTasks](../../../../studio/database/tasks/import-data/import-from-ravendb.mdx#customize-configuration-and-ongoing-tasks) diff --git a/docs/studio/database/tasks/import-data/import-from-ravendb.mdx b/docs/studio/database/tasks/import-data/import-from-ravendb.mdx index 63465e97f3..4ae8437ea5 100644 --- a/docs/studio/database/tasks/import-data/import-from-ravendb.mdx +++ b/docs/studio/database/tasks/import-data/import-from-ravendb.mdx @@ -147,7 +147,7 @@ import LanguageContent from "@site/src/components/LanguageContent"; 3. **Other** - [Include Identities](../../../../client-api/document-identifiers/working-with-document-identifiers.mdx) - [Include Compare Exchange](../../../../compare-exchange/overview.mdx) - - [Include Subscriptions](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + - [Include Subscriptions](../../../../data-subscriptions/overview.mdx) - [Include Configuration and OngoingTasks](../../../../studio/database/tasks/import-data/import-from-ravendb.mdx#customize-configuration-and-ongoing-tasks) diff --git a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-1.png b/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-1.png deleted file mode 100644 index 064f78df7d..0000000000 Binary files a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-1.png and /dev/null differ diff --git a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-2.png b/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-2.png deleted file mode 100644 index 8c6f5220cb..0000000000 Binary files a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-2.png and /dev/null differ diff --git a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-3.png b/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-3.png deleted file mode 100644 index e6e0c79fd3..0000000000 Binary files a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-3.png and /dev/null differ diff --git a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-4.png b/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-4.png deleted file mode 100644 index 989a3ec6d2..0000000000 Binary files a/docs/studio/database/tasks/ongoing-tasks/assets/subscriptions-4.png and /dev/null differ diff --git a/docs/studio/database/tasks/ongoing-tasks/external-replication-task.mdx b/docs/studio/database/tasks/ongoing-tasks/external-replication-task.mdx index a8427b7501..797d5d957f 100644 --- a/docs/studio/database/tasks/ongoing-tasks/external-replication-task.mdx +++ b/docs/studio/database/tasks/ongoing-tasks/external-replication-task.mdx @@ -58,7 +58,7 @@ Server and cluster level features. * [Indexes](../../../../indexes/creating-and-deploying.mdx) * [Conflict resolver definitions](../../../../server/clustering/replication/replication-conflicts.mdx#conflict-resolution-script) * [Compare-Exchange](../../../../compare-exchange/overview.mdx) -* [Subscriptions](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) +* [Subscriptions](../../../../data-subscriptions/overview.mdx) * [Identities](../../../../server/kb/document-identifier-generation.mdx#strategy--3) * Ongoing tasks * [ETL](../../../../server/ongoing-tasks/etl/basics.mdx) diff --git a/docs/studio/database/tasks/ongoing-tasks/general-info.mdx b/docs/studio/database/tasks/ongoing-tasks/general-info.mdx index dbf289abf7..77eb11f2a7 100644 --- a/docs/studio/database/tasks/ongoing-tasks/general-info.mdx +++ b/docs/studio/database/tasks/ongoing-tasks/general-info.mdx @@ -51,7 +51,7 @@ The available ongoing tasks are: * **[Backup](../../../../backup/create/periodic-tasks/database-backup.mdx)** Schedule a backup or a snapshot of the database at a specified point in time. -* **[Subscription](../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx)** +* **[Subscription](../../../../data-subscriptions/overview.mdx)** Send batches of documents that match a pre-defined query for processing on a client. **ETL (RavenDB => Target):** diff --git a/docs/studio/database/tasks/ongoing-tasks/hub-sink-replication/overview.mdx b/docs/studio/database/tasks/ongoing-tasks/hub-sink-replication/overview.mdx index 4fcc0b0112..fec099f25a 100644 --- a/docs/studio/database/tasks/ongoing-tasks/hub-sink-replication/overview.mdx +++ b/docs/studio/database/tasks/ongoing-tasks/hub-sink-replication/overview.mdx @@ -87,7 +87,7 @@ and Sink replication tasks. * [Indexes](../../../../../indexes/creating-and-deploying.mdx) * [Conflict resolver definitions](../../../../../server/clustering/replication/replication-conflicts.mdx#conflict-resolution-script) * [Compare-Exchange](../../../../../compare-exchange/overview.mdx) - * [Subscriptions](../../../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) + * [Subscriptions](../../../../../data-subscriptions/overview.mdx) * [Identities](../../../../../server/kb/document-identifier-generation.mdx#identity-id) * Ongoing tasks * [ETL](../../../../../server/ongoing-tasks/etl/basics.mdx) diff --git a/docs/studio/database/tasks/ongoing-tasks/subscription-task.mdx b/docs/studio/database/tasks/ongoing-tasks/subscription-task.mdx deleted file mode 100644 index 5320ef6f99..0000000000 --- a/docs/studio/database/tasks/ongoing-tasks/subscription-task.mdx +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: "Subscription Task" -sidebar_label: Subscription Task -description: "Create and manage data subscription tasks in RavenDB Studio with RQL filtering, client connection monitoring, and batch configuration." -sidebar_position: 3 ---- - -import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; - -# Subscription Task - - -* A **Data Subscription** is a task that automatically sends certain documents to a subscribed - client, called the "subscription worker". - -* This page explains how to create a subscription task on the server side using the Studio. - -* In this page: - * [Subscription Task Definition](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx#subscription-task-definition) - * [Testing Subscriptions](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx#testing) - * [Details in Task List View](../../../../studio/database/tasks/ongoing-tasks/subscription-task.mdx#details-in-task-list-view) - - -## Subscription Task Definition - -![Figure 1. Subscription Task Definition](./assets/subscriptions-1.png) - -1. The RQL query that selects which documents the subscription sends - to the client. - -2. Define the starting point from which to send the first batch: - * **Begining of Time** (default option) - All documents matching the RQL query will be sent, regardless of their creation time. - * **Latest Document** - Start from the first new document that will be created after the subscription is created. - * **Change Vector** - Specify a document change vector as the subscription starting point. - -3. Choose which node will perform this task by default. - -4. Test the subscription query. - - - -## Testing - -![Figure 2. Testing Subscription](./assets/subscriptions-2.png) - -Testing the subscription shows which documents match the specified query. - -1. Limit the number of results to retrieve for this test. - -2. Limit how long - in seconds - the test should continue before -stopping automatically. - -3. Run the test. - - - -## Details in Task List View - -![Figure 3. Task List View](./assets/subscriptions-3.png) - -Click the info button on a task in the task list view, to see -more detailed information. - -1. Task Details Panel - * Task status - - **Active**: status is `Active` while a subscription worker is connected to the task. - **Not active**: status is `Not-active` when no workers are connected. - * Mode - - **Single**: mode is `Single` if the - [subscription strategy](../../../../client-api/data-subscriptions/consumption/how-to-consume-data-subscription.mdx#determining-which-workers-a-subscription-will-serve) - allows only one worker to be connected at a time. - **Concurrent**: mode is `Concurrent` if the subscription strategy allows only workers - that use the `Concurrent` strategy to connect. - * Client URI - the identifier of the subscription worker - subscribed to this task. - * Connection strategy - determines the workers connection strategy. - * Change vector for next batch - the change vector of the last document - that was sent in the last batch. - * Last batch acknowledgement time - the last time a worker - responded that it has received a batch. - * Last client connection time - the last time a worker communicated - with or pinged the server. -2. Database Group Topology -### Concurrent Subscriptions - -When [Concurrent Subscribers](../../../../client-api/data-subscriptions/concurrent-subscriptions.mdx) -are connected to a subscription task, they are all listed on the task details panel. - -![Figure 4. Concurrent Subscribers](./assets/subscriptions-4.png) - -1. Each concurrent subscriber can be disconnected individually. -2. Refresh the view to see changes in workers connection states. - - - diff --git a/docs/studio/server/certificates/read-only-access-level.mdx b/docs/studio/server/certificates/read-only-access-level.mdx index 3ac8ff8d93..9144a74272 100644 --- a/docs/studio/server/certificates/read-only-access-level.mdx +++ b/docs/studio/server/certificates/read-only-access-level.mdx @@ -25,7 +25,7 @@ or [static indexes](../../../indexes/creating-and-deploying.mdx#static-indexes). will still create [auto-indexes](../../../indexes/creating-and-deploying.mdx#auto-indexes) in response to the clients' queries. -* Clients with Read Only access can still become [subscription workers](../../../client-api/data-subscriptions/what-are-data-subscriptions.mdx) +* Clients with Read Only access can still become [subscription workers](../../../data-subscriptions/overview.mdx) to consume data subscriptions. * When in 'Read Only mode', there are various slight differences in the appearance of the Studio diff --git a/guides/transactional-outbox.mdx b/guides/transactional-outbox.mdx index 0a12c9bcef..24f5ce16f7 100644 --- a/guides/transactional-outbox.mdx +++ b/guides/transactional-outbox.mdx @@ -14,10 +14,10 @@ see_also: link: "server/ongoing-tasks/etl/queue-etl/kafka" source: "docs" path: "Server > Ongoing Tasks > ETL > Queue ETL" - - title: "What are Data Subscriptions" - link: "client-api/data-subscriptions/what-are-data-subscriptions" + - title: "Data Subscriptions Overview" + link: "data-subscriptions/overview" source: "docs" - path: "Client API > Data Subscriptions" + path: "Data Subscriptions" - title: "Sample: The Library of Ravens" link: "/samples/the-ravens-library" source: "samples" diff --git a/scripts/redirects.json b/scripts/redirects.json index 47970e4e8e..68864d1677 100644 --- a/scripts/redirects.json +++ b/scripts/redirects.json @@ -313,6 +313,97 @@ "minimumVersion": "6.2" } }, + { + "key": "/client-api/data-subscriptions/what-are-data-subscriptions", + "value": { + "targetUrl": "/data-subscriptions/overview", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/concurrent-subscriptions", + "value": { + "targetUrl": "/data-subscriptions/concurrent-subscriptions", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/creation/how-to-create-data-subscription", + "value": { + "targetUrl": "/data-subscriptions/creating-subscription/creating-subscription_api", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/creation/api-overview", + "value": { + "targetUrl": "/data-subscriptions/creating-subscription/creating-subscription_api", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/creation/examples", + "value": { + "targetUrl": "/data-subscriptions/creating-subscription/creating-subscription_api", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/consumption/how-to-consume-data-subscription", + "value": { + "targetUrl": "/data-subscriptions/consuming-subscription", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/consumption/api-overview", + "value": { + "targetUrl": "/data-subscriptions/consuming-subscription", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/consumption/examples", + "value": { + "targetUrl": "/data-subscriptions/consuming-subscription", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/advanced-topics/maintenance-operations", + "value": { + "targetUrl": "/data-subscriptions/maintenance-operations", + "minimumVersion": "7.2" + } + }, + { + "key": "/client-api/data-subscriptions/advanced-topics/subscription-with-revisioning", + "value": { + "targetUrl": "/data-subscriptions/revisions-support", + "minimumVersion": "7.2" + } + }, + { + "key": "/studio/database/tasks/ongoing-tasks/subscription-task", + "value": { + "targetUrl": "/data-subscriptions/creating-subscription/creating-subscription_studio", + "minimumVersion": "7.2" + } + }, + { + "key": "/studio/database/stats/ongoing-tasks-stats/subscription-stats", + "value": { + "targetUrl": "/data-subscriptions/monitoring", + "minimumVersion": "7.2" + } + }, + { + "key": "/server/configuration/subscription-configuration", + "value": { + "targetUrl": "/data-subscriptions/configuration", + "minimumVersion": "7.2" + } + }, { "key": "/client-api/document-identifiers", "value": { diff --git a/sidebars.ts b/sidebars.ts index 7bdaba7bf7..d20eaf342f 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -77,6 +77,11 @@ const sidebars: SidebarsConfig = { label: "Server", items: [{ type: "autogenerated", dirName: "server" }], }, + { + type: "category", + label: "Data Subscriptions", + items: [{ type: "autogenerated", dirName: "data-subscriptions" }], + }, { type: "category", label: "Studio", diff --git a/src/components/Homepage/Features/FeaturesLists/OngoingTasksFeatures.tsx b/src/components/Homepage/Features/FeaturesLists/OngoingTasksFeatures.tsx index 86950bfc89..485c9e9f01 100644 --- a/src/components/Homepage/Features/FeaturesLists/OngoingTasksFeatures.tsx +++ b/src/components/Homepage/Features/FeaturesLists/OngoingTasksFeatures.tsx @@ -23,7 +23,10 @@ export default function OngoingTasksFeaturesGrid() { { title: "Subscriptions", icon: "subscriptions", - url: `/${activeVersion.label}/client-api/data-subscriptions/what-are-data-subscriptions`, + url: + activeVersion.label >= "7.2" + ? `/${activeVersion.label}/data-subscriptions/overview` + : `/${activeVersion.label}/client-api/data-subscriptions/what-are-data-subscriptions`, description: "Subscribe to defined documents, trigger your worker routines on field updates", minimumSupportedVersion: "3.0", },