From 42ce89adbfec78b0bc73d4bf8774a5738f3cbf6a Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 26 Jan 2026 14:58:47 -0800 Subject: [PATCH 1/8] add new topic --- modules/ROOT/nav.adoc | 1 + .../operations/pages/functions-transform.adoc | 167 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 modules/operations/pages/functions-transform.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 3abe4b68..155a7a7d 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -18,6 +18,7 @@ .Operations * xref:operations:auth.adoc[] * xref:operations:functions.adoc[] +* xref:operations:functions-transform.adoc[] * xref:operations:io-connectors.adoc[] * xref:operations:io-elastic-sink.adoc[] * xref:operations:scale-cluster.adoc[] diff --git a/modules/operations/pages/functions-transform.adoc b/modules/operations/pages/functions-transform.adoc new file mode 100644 index 00000000..c3f3bab4 --- /dev/null +++ b/modules/operations/pages/functions-transform.adoc @@ -0,0 +1,167 @@ += {product} {pulsar-short} transform functions configuration reference +:navtitle: Transform functions configuration reference + +include::common:streaming:partial$transform-functions/intro.adoc[] + +To deploy transform functions, see xref:operations:functions.adoc[Deploy and manage functions]. + +== Configure transform functions + +include::common:streaming:partial$transform-functions/config.adoc[] + +[#expression-language] +=== Expression language + +include::common:streaming:partial$transform-functions/expression-language.adoc[] + +[#conditional-steps] +=== Conditional steps + +include::common:streaming:partial$transform-functions/when.adoc[] + +[#cast] +== Cast + +include::common:streaming:partial$transform-functions/cast.adoc[] + +[#compute] +== Compute + +include::common:streaming:partial$transform-functions/compute.adoc[] + +[#drop] +== Drop + +include::common:streaming:partial$transform-functions/drop.adoc[] + +[#drop-fields] +== Drop fields + +include::common:streaming:partial$transform-functions/drop-fields.adoc[] + +[#flatten] +== Flatten + +include::common:streaming:partial$transform-functions/flatten.adoc[] + +[#merge-keyvalue] +== Merge KeyValue + +include::common:streaming:partial$transform-functions/merge-keyvalue.adoc[] + +[#unwrap-keyvalue] +== Unwrap KeyValue + +include::common:streaming:partial$transform-functions/unwrap-keyvalue.adoc[] + +//// + + +[#deploy-cli] +== Deploy with {pulsar-short} CLI + +https://github.com/datastax/pulsar[Luna Streaming 2.10+] is required to deploy custom functions in {pulsar-short}. + +The transform function `.nar` lives in the `/functions` directory of your {pulsar-short} deployment. + +[tabs] +====== +{pulsar-short} standalone:: ++ +-- +To deploy the built-in transform function locally in {pulsar-short} standalone, do the following: + +. Start {pulsar-short} standalone: ++ +[source,shell] +---- +./bin/pulsar standalone +---- + +. Create a transform function in `localrun` mode: ++ +[source,shell,subs="attributes+"] +---- +./bin/pulsar-admin functions localrun \ +--jar functions/pulsar-transformations-2.0.1.nar \ +--classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ +--inputs my-input-topic \ +--output my-output-topic \ +--user-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}, {"type": "cast", "schema-type": "STRING"}]}' +---- +-- + +{pulsar-short} cluster:: ++ +-- +To deploy a built-in transform function to a {pulsar-short} cluster, do the following: + +. Create a built-in transform function with the {pulsar-short} CLI: ++ +---- +./bin/pulsar-admin functions create \ +--tenant $TENANT \ +--namespace $NAMESPACE \ +--name transform-function \ +--inputs persistent://$TENANT/$NAMESPACE/$INPUT_TOPIC +--output persistent://$TENANT/$NAMESPACE/$OUTPUT_TOPIC \ +--classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ +--jar functions/pulsar-transformations-2.0.1.nar +---- ++ +.Result +[%collapsible] +==== +[source,console] +---- +Created successfully +---- +==== + +. Confirm your function has been created: ++ +[source,shell] +---- +./bin/pulsar-admin functions list --tenant $TENANT +---- ++ +.Result +[%collapsible] +==== +[source,console] +---- +cast-function +flatten-function +transform-function +transform-function-2 +---- +==== + +-- +====== + +== Deploy a transform function in a sink + +With modern Pulsar versions, transform functions can be deployed inside of a sink process. + +Before this update, functions transformed data either after it was written to a topic by a source connector, or before it was read from a topic by a sink connector. + +This required either an intermediate topic, with additional storage, IO, and latency, or a custom connector. + +Now, functions can be deployed at sink creation and apply preprocessing to sink topic writes. + + +Create sink function in pulsar-admin: + +https://github.com/datastax/pulsar[Luna Streaming 2.10+] is required to deploy custom functions in {pulsar-short}. + +Create a sink connector, and include the path to the transform function and configuration at creation: + +[source,shell] +---- +pulsar-admin sinks create \ +--sink-type elastic-sink \ +--inputs my-input-topic \ +--tenant public \ +--namespace default \ +--name my-sink \ +--transform-function "builtin://transforms" \ +--transform-function-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}' +---- +//// \ No newline at end of file From 16c29aea038af996bee739c4b9f09b1143563347 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 26 Jan 2026 14:59:55 -0800 Subject: [PATCH 2/8] edit main function page intro --- modules/operations/pages/functions.adoc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/operations/pages/functions.adoc b/modules/operations/pages/functions.adoc index 8433c082..4264b262 100644 --- a/modules/operations/pages/functions.adoc +++ b/modules/operations/pages/functions.adoc @@ -1,8 +1,14 @@ -= Luna Streaming Functions += Deploy and manage {product} {pulsar-short} functions +:navtitle: Deploy and manage {pulsar-short} functions -Functions are lightweight compute processes that enable you to process each message received on a topic or multiple topics. You can apply custom logic to that message, transforming or enriching it, and then output it to a different topic. +Functions are lightweight compute processes that you can run on each message a topic receives. +You can apply custom logic to a message, transforming or enriching it, and then output it to a different topic. -Functions run inside Luna Streaming and are therefore serverless. Write the code for your function in Java, Python, or Go, then upload the code to the {pulsar-short} cluster and deploy the function. The function will be automatically run for each message published to the specified input topic. See https://pulsar.apache.org/docs/en/functions-overview/[{pulsar-short} Functions overview] for more information about {pulsar-reg} functions. +Functions run inside {product}, which makes them serverless. +You write the code for your function in Java, Python, or Go, and then upload the code to the {pulsar-short} cluster and deploy the function. +The function automatically runs for each message published to the specified input topic. + +Functions are implemented using https://pulsar.apache.org/docs/en/functions-overview/[{pulsar-reg} functions]. == Manage functions using {pulsar-short} Admin CLI From e29a9d50cfd4c62928f6b2718e345f8bb82f762d Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 26 Jan 2026 15:32:59 -0800 Subject: [PATCH 3/8] combine content from stream learn --- .../operations/pages/functions-transform.adoc | 114 +---------- modules/operations/pages/functions.adoc | 190 ++++++++++++++---- 2 files changed, 153 insertions(+), 151 deletions(-) diff --git a/modules/operations/pages/functions-transform.adoc b/modules/operations/pages/functions-transform.adoc index c3f3bab4..e86dedbf 100644 --- a/modules/operations/pages/functions-transform.adoc +++ b/modules/operations/pages/functions-transform.adoc @@ -52,116 +52,4 @@ include::common:streaming:partial$transform-functions/merge-keyvalue.adoc[] [#unwrap-keyvalue] == Unwrap KeyValue -include::common:streaming:partial$transform-functions/unwrap-keyvalue.adoc[] - -//// - - -[#deploy-cli] -== Deploy with {pulsar-short} CLI - -https://github.com/datastax/pulsar[Luna Streaming 2.10+] is required to deploy custom functions in {pulsar-short}. - -The transform function `.nar` lives in the `/functions` directory of your {pulsar-short} deployment. - -[tabs] -====== -{pulsar-short} standalone:: -+ --- -To deploy the built-in transform function locally in {pulsar-short} standalone, do the following: - -. Start {pulsar-short} standalone: -+ -[source,shell] ----- -./bin/pulsar standalone ----- - -. Create a transform function in `localrun` mode: -+ -[source,shell,subs="attributes+"] ----- -./bin/pulsar-admin functions localrun \ ---jar functions/pulsar-transformations-2.0.1.nar \ ---classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ ---inputs my-input-topic \ ---output my-output-topic \ ---user-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}, {"type": "cast", "schema-type": "STRING"}]}' ----- --- - -{pulsar-short} cluster:: -+ --- -To deploy a built-in transform function to a {pulsar-short} cluster, do the following: - -. Create a built-in transform function with the {pulsar-short} CLI: -+ ----- -./bin/pulsar-admin functions create \ ---tenant $TENANT \ ---namespace $NAMESPACE \ ---name transform-function \ ---inputs persistent://$TENANT/$NAMESPACE/$INPUT_TOPIC ---output persistent://$TENANT/$NAMESPACE/$OUTPUT_TOPIC \ ---classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ ---jar functions/pulsar-transformations-2.0.1.nar ----- -+ -.Result -[%collapsible] -==== -[source,console] ----- -Created successfully ----- -==== - -. Confirm your function has been created: -+ -[source,shell] ----- -./bin/pulsar-admin functions list --tenant $TENANT ----- -+ -.Result -[%collapsible] -==== -[source,console] ----- -cast-function -flatten-function -transform-function -transform-function-2 ----- -==== - --- -====== - -== Deploy a transform function in a sink - -With modern Pulsar versions, transform functions can be deployed inside of a sink process. + -Before this update, functions transformed data either after it was written to a topic by a source connector, or before it was read from a topic by a sink connector. + -This required either an intermediate topic, with additional storage, IO, and latency, or a custom connector. + -Now, functions can be deployed at sink creation and apply preprocessing to sink topic writes. + - -Create sink function in pulsar-admin: - -https://github.com/datastax/pulsar[Luna Streaming 2.10+] is required to deploy custom functions in {pulsar-short}. - -Create a sink connector, and include the path to the transform function and configuration at creation: - -[source,shell] ----- -pulsar-admin sinks create \ ---sink-type elastic-sink \ ---inputs my-input-topic \ ---tenant public \ ---namespace default \ ---name my-sink \ ---transform-function "builtin://transforms" \ ---transform-function-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}' ----- -//// \ No newline at end of file +include::common:streaming:partial$transform-functions/unwrap-keyvalue.adoc[] \ No newline at end of file diff --git a/modules/operations/pages/functions.adoc b/modules/operations/pages/functions.adoc index 4264b262..587d50b8 100644 --- a/modules/operations/pages/functions.adoc +++ b/modules/operations/pages/functions.adoc @@ -10,12 +10,22 @@ The function automatically runs for each message published to the specified inpu Functions are implemented using https://pulsar.apache.org/docs/en/functions-overview/[{pulsar-reg} functions]. -== Manage functions using {pulsar-short} Admin CLI +https://github.com/datastax/pulsar[Luna Streaming 2.10 or later] is required to deploy custom functions in {pulsar-short}. -Add functions using the {pulsar-short} Admin CLI. Create a new Python function to consume a message from one topic, add an exclamation point, and publish the results to another topic. +== Manage functions using {pulsar-short} Admin CLI -. Create the following Python function in `function.py`: +Add functions using the {pulsar-short} Admin CLI. + +[tabs] +====== +Deploy custom function code:: ++ +-- +. Create a file that contains your function code. + +For example purposes, you can use the following sample function that consumes a message from one topic, adds an exclamation point, and then publishes the result to another topic. ++ +.function.py [source,python] ---- from pulsar import Function @@ -27,58 +37,161 @@ class ExclamationFunction(Function): def process(self, input, context): return input + '!' ---- -+ -. Deploy `function.py` to your {pulsar-short} cluster using the {pulsar-short} Admin CLI: + +. Deploy your function file to your {pulsar-short} cluster using the {pulsar-short} Admin CLI: + [source,bash] ---- -$ ./pulsar-admin functions create \ +./pulsar-admin functions create \ --py function.py \ --classname function.ExclamationFunction \ - --tenant \ - --namespace default \ + --tenant $TENANT \ + --namespace $NAMESPACE \ --name exclamation \ - --inputs persistent:///default/ - --output persistent:///default/ + --inputs persistent://$TENANT/$NAMESPACE/$INPUT_TOPIC \ + --output persistent://$TENANT/$NAMESPACE/$OUTPUT_TOPIC +---- ++ +If the function is set up and ready to accept messages, then the output is `Created Successfully`. + +. Confirm the function was created by listing the functions in the tenant: ++ +[source,bash] +---- +./pulsar-admin functions list --tenant $TENANT +---- + +-- + +Deploy {company} transform functions:: ++ +-- +Transform functions are built-in functions that perform common data transformation tasks, such as casting data types, dropping fields, flattening nested structures, and merging or unwrapping KeyValue pairs. +For more information, see the xref:operations:functions-transform.adoc[transform functions configuration reference]. + +You can find the transform function `.nar` in the `/functions` directory of your {pulsar-short} deployment. + +[tabs] +==== +{pulsar-short} standalone:: ++ +To deploy the built-in transform function locally in {pulsar-short} standalone, do the following: ++ +. Start {pulsar-short} standalone: ++ +[source,shell] +---- +./bin/pulsar standalone +---- ++ +. Create a transform function in `localrun` mode: ++ +[source,bash] +---- +./bin/pulsar-admin functions localrun \ +--jar functions/pulsar-transformations-2.0.1.nar \ +--classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ +--inputs $INPUT_TOPIC \ +--output $OUTPUT_TOPIC \ +--user-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}, {"type": "cast", "schema-type": "STRING"}]}' ---- + -If the function is set up and ready to accept messages, you should see "Created Successfully!" +For the `user-config` parameters, see the xref:operations:functions-transform.adoc[transform functions configuration reference]. -. Use `./pulsar-admin functions list --tenant ` to list the functions in your tenant and confirm your new function was created. +{pulsar-short} cluster:: ++ +To deploy a built-in transform function to a {pulsar-short} cluster, do the following: ++ +. Create a built-in transform function with the {pulsar-short} CLI: ++ +[source,bash] +---- +./bin/pulsar-admin functions create \ +--tenant $TENANT \ +--namespace $NAMESPACE \ +--name transform-function \ +--inputs persistent://$TENANT/$NAMESPACE/$INPUT_TOPIC \ +--output persistent://$TENANT/$NAMESPACE/$OUTPUT_TOPIC \ +--classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ +--jar functions/pulsar-transformations-2.0.1.nar +---- ++ +This option deployed the functions using the transform function `.nar`. ++ +You can also deploy individual functions by passing the function configuration in the `--user-config` parameter. +For more information, see the xref:operations:functions-transform.adoc[transform functions configuration reference]. ++ +. Confirm the function was created by listing the functions in the tenant: ++ +[source,bash] +---- +./pulsar-admin functions list --tenant $TENANT +---- + +{pulsar-short} sink:: +By default, functions modify data either after it is written to a topic by a source connector, or before it is read from a topic by a sink connector. +This requires either a custom connector or an intermediate topic (with additional storage, IO, and latency). ++ +Alternatively, you can deploy functions in a sink to apply preprocessing to sink topic writes (outgoing messages from the sink). ++ +To deploy a function in a sink, include the function path and configuration when you create the sink connector. +For example: ++ +[source,bash] +---- +pulsar-admin sinks create \ +--sink-type elastic-sink \ +--inputs my-input-topic \ +--tenant public \ +--namespace default \ +--name my-sink \ +--transform-function "builtin://transforms" \ +--transform-function-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}' +---- +==== + +-- +====== === Trigger with CLI -Triggering a function is a convenient way to test that the function is working. When you trigger a function, you are publishing a message on the function’s input topic, which triggers the function to run. +Triggering a function is a convenient way to test that the function is working. When you trigger a function, you are publishing a message on the function's input topic, which triggers the function to run. -To test a function with the {pulsar-short} CLI, send a test value with {pulsar-short} CLI's `trigger`. +To test a function with the {pulsar-short} CLI, send a test value with {pulsar-short} CLI's `functions trigger` command: -. Listen for messages on the output topic: +. Listen for messages on the output topic: + [source,bash] ---- -$ bin/pulsar-client consume persistent:///default/ \ +bin/pulsar-client consume persistent://$TENANT/$NAMESPACE/$TOPIC \ --subscription-name my-subscription --num-messages 0 # Listen indefinitely ---- + +. Test the function with the `functions trigger` command and a test message. + -. Test your exclamation function with `trigger`: +The following example triggers a function named `exclamation`: + [source,bash] ---- -$ ./pulsar-admin functions trigger \ +./pulsar-admin functions trigger \ --name exclamation \ - --tenant \ - --namespace default \ + --tenant $TENANT \ + --namespace $NAMESPACE \ --trigger-value "Hello world" ---- -+ -The trigger sends the string `Hello world` to your exclamation function. Your function should output `Hello world!` to your consumed output. + +. Make sure the output shows that the messages was processed as expected based on your function logic. +For example, the `exclamation` sample function adds an exclamation point to the input string. +Therefore, the test message `Hello world` should output `Hello world!`. == Add Functions using {pulsar-short} Admin Console -If the {pulsar-short} Admin Console is deployed, you can also add and manage the {pulsar-short} functions in the *Functions* tab of the Admin Console web UI. +If the {pulsar-short} Admin Console is deployed, you can also add and manage the {pulsar-short} functions in the *Functions* tab of the Admin Console web UI. -. Select *Choose File* to choose a local Function. In this example, we chose `exclamation_function.py`. Choose the file you want to pull the function from and which function you want to use within that file. +. Select *Choose File* to choose a local Function. +In this example, we chose `exclamation_function.py`. +Choose the file you want to pull the function from and which function you want to use within that file. + image:admin-console-add-function.png[Add Function in Admin Console] + @@ -86,32 +199,32 @@ To specify the class name, it depends on the programming language or runtime of + Python functions are added by loading a Python file (.py) or a zipped Python file (.zip). When adding Python files, the Class Name is specified as the name of the Python file without the extension plus the class you want to execute. For example, if the Python file is called `function.py` and the class is `exclamation`, then the class name would be `function.exclamation`. The file can contain multiple classes, but only one will be used. If there is no class in the Python file (when using a basic function, for example) just specify the filename without the extension (ex. `function`). + -Java functions are added by loading a Java jar file (.jar). When adding Java files, also specify the name of the class to execute as the function. +Java functions are added by loading a Java jar file (.jar). When adding Java files, also specify the name of the class to execute as the function. + Go functions are added by loading a Go file (.go). For more information on packaging a Go function, see https://pulsar.apache.org/docs/en/functions-package/#go[Packaging Functions in Go]. -[start=2] -. Choose your function name and namespace. +. Choose your function name and namespace. + -Your input topics, output topics, log topics, and processing guarantees will auto-populate. +Your input topics, output topics, log topics, and processing guarantees will auto-populate. . Provide a *Configuration Key* in the dropdown menu. + For a list of configuration keys, see the https://pulsar.apache.org/functions-rest-api/#operation/registerFunction[{pulsar-short} Functions API Docs]. -. Select *Add* to add your function. - -You have created a function for this namespace. You can confirm your function was created in the *Manage* tab. +. Select *Add* to add your function. ++ +You have created a function for this namespace. +You can confirm your function was created in the *Manage* tab. == Manage functions in Admin Console -You can start, stop, and restart your function by selecting *Manage* in the *Functions* dashboard. +You can start, stop, and restart your function by selecting *Manage* in the *Functions* dashboard. image::admin-console-function-added.png[Manage Functions] === Monitor your function -Functions produce logs to help you in debugging. To view your function's logs, select *Logs* in the *Manage* dashboard. +Functions produce logs to help you in debugging. To view your function's logs, select *Logs* in the *Manage* dashboard. image::admin-console-logs.png[Function Log] @@ -127,11 +240,11 @@ A function that is already running can be updated with a new configuration. The If you need to update any other setting of the function, delete and then re-add the function. -To update your function, select *Update* in the *Manage* dashboard. +To update your function, select *Update* in the *Manage* dashboard. === Delete your function -To delete a function, select *Delete* in the *Manage* dashboard. +To delete a function, select *Delete* in the *Manage* dashboard. A *Function-name Deleted Successfully!* flag will appear to let you know you've deleted your function. @@ -141,8 +254,9 @@ To trigger a function in the {pulsar-short} Admin Console, select *Trigger* in t image::admin-console-trigger-function.png[Trigger Function] -Enter your message in the *Message to Send* field, and select the output topic. In this case, the trigger sends the string `Hello world!` to your exclamation function with no output function. If the function has an output topic and the function returns data to the output topic, it will be displayed. +Enter your message in the *Message to Send* field, and select the output topic. In this case, the trigger sends the string `Hello world!` to your exclamation function with no output function. +If the function has an output topic and the function returns data to the output topic, it will be displayed. -== Next steps +== See also -For more about developing functions for Luna Streaming and {pulsar-short}, see https://pulsar.apache.org/docs/en/functions-develop/[here]. \ No newline at end of file +* https://pulsar.apache.org/docs/en/functions-develop/[{pulsar-short} documentation on developing functions] \ No newline at end of file From d5ed28db373068e0e3381c5cba6376015279ca70 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 26 Jan 2026 15:40:45 -0800 Subject: [PATCH 4/8] indentation --- modules/operations/pages/functions.adoc | 38 ++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/modules/operations/pages/functions.adoc b/modules/operations/pages/functions.adoc index 587d50b8..91e85655 100644 --- a/modules/operations/pages/functions.adoc +++ b/modules/operations/pages/functions.adoc @@ -89,11 +89,11 @@ To deploy the built-in transform function locally in {pulsar-short} standalone, [source,bash] ---- ./bin/pulsar-admin functions localrun \ ---jar functions/pulsar-transformations-2.0.1.nar \ ---classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ ---inputs $INPUT_TOPIC \ ---output $OUTPUT_TOPIC \ ---user-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}, {"type": "cast", "schema-type": "STRING"}]}' + --jar functions/pulsar-transformations-2.0.1.nar \ + --classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ + --inputs $INPUT_TOPIC \ + --output $OUTPUT_TOPIC \ + --user-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}, {"type": "cast", "schema-type": "STRING"}]}' ---- + For the `user-config` parameters, see the xref:operations:functions-transform.adoc[transform functions configuration reference]. @@ -107,13 +107,13 @@ To deploy a built-in transform function to a {pulsar-short} cluster, do the foll [source,bash] ---- ./bin/pulsar-admin functions create \ ---tenant $TENANT \ ---namespace $NAMESPACE \ ---name transform-function \ ---inputs persistent://$TENANT/$NAMESPACE/$INPUT_TOPIC \ ---output persistent://$TENANT/$NAMESPACE/$OUTPUT_TOPIC \ ---classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ ---jar functions/pulsar-transformations-2.0.1.nar + --tenant $TENANT \ + --namespace $NAMESPACE \ + --name transform-function \ + --inputs persistent://$TENANT/$NAMESPACE/$INPUT_TOPIC \ + --output persistent://$TENANT/$NAMESPACE/$OUTPUT_TOPIC \ + --classname com.datastax.oss.pulsar.functions.transforms.TransformFunction \ + --jar functions/pulsar-transformations-2.0.1.nar ---- + This option deployed the functions using the transform function `.nar`. @@ -140,13 +140,13 @@ For example: [source,bash] ---- pulsar-admin sinks create \ ---sink-type elastic-sink \ ---inputs my-input-topic \ ---tenant public \ ---namespace default \ ---name my-sink \ ---transform-function "builtin://transforms" \ ---transform-function-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}' + --sink-type elastic-sink \ + --inputs my-input-topic \ + --tenant public \ + --namespace default \ + --name my-sink \ + --transform-function "builtin://transforms" \ + --transform-function-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}' ---- ==== From 14f347c46e1c1c27f5f4603993d0ca530e8a3f2f Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:01:37 -0800 Subject: [PATCH 5/8] remove embedded videos and redundant minitocs --- .../components/pages/admin-console-tutorial.adoc | 15 --------------- modules/components/pages/admin-console-vm.adoc | 10 ---------- .../pages/quickstart-helm-installs.adoc | 7 ------- modules/operations/pages/troubleshooting.adoc | 6 ------ 4 files changed, 38 deletions(-) diff --git a/modules/components/pages/admin-console-tutorial.adoc b/modules/components/pages/admin-console-tutorial.adoc index 5fd75dce..85fbcb0c 100644 --- a/modules/components/pages/admin-console-tutorial.adoc +++ b/modules/components/pages/admin-console-tutorial.adoc @@ -2,14 +2,6 @@ The *{company} Admin Console for {pulsar-reg}* is a web-based UI from {company} that administers topics, namespaces, sources, sinks, and various aspects of {pulsar} features. -* xref:components:admin-console-tutorial.adoc#getting-started[] -* xref:components:admin-console-tutorial.adoc#features[] -* xref:components:admin-console-tutorial.adoc#send-receive[] -* xref:components:admin-console-tutorial.adoc#create-topics[] -* xref:components:admin-console-tutorial.adoc#code-samples[] -* xref:components:admin-console-tutorial.adoc#connect-to-pulsar[] -* xref:components:admin-console-tutorial.adoc#video[] - [#getting-started] == Getting Started in {pulsar-short} Admin Console @@ -147,13 +139,6 @@ You can get the token from the {pulsar-short} Admin Console's *Credentials* page Alternatively, you can save the URL authentication parameters in your `client.conf` file. -[#video] -== Admin console video - -You can also follow along with this video from our *Five Minutes About {pulsar-short}* series to get started with the admin console. - -video::1IwblLfPiPQ[youtube, list=PL2g2h-wyI4SqeKH16czlcQ5x4Q_z-X7_m] - == Next steps For more on building and running a standalone {pulsar-short} Admin console, see the xref:admin-console-vm.adoc[Admin Console on Server/VM] or the {pulsar-short} Admin console repo https://github.com/datastax/pulsar-admin-console#dev[readme]. \ No newline at end of file diff --git a/modules/components/pages/admin-console-vm.adoc b/modules/components/pages/admin-console-vm.adoc index a0241a48..d99448b3 100644 --- a/modules/components/pages/admin-console-vm.adoc +++ b/modules/components/pages/admin-console-vm.adoc @@ -4,16 +4,6 @@ The Admin Console is a VueJS application that runs in a browser. It also includes a web server that serves up the files for the Admin Console as well as providing configuration and authentication services. -This document covers: - -* <> - -* <> - -* <> - -* <> - [#install] == Install {pulsar-short} Admin Console diff --git a/modules/install-upgrade/pages/quickstart-helm-installs.adoc b/modules/install-upgrade/pages/quickstart-helm-installs.adoc index 4ca4041c..82e1060f 100644 --- a/modules/install-upgrade/pages/quickstart-helm-installs.adoc +++ b/modules/install-upgrade/pages/quickstart-helm-installs.adoc @@ -474,13 +474,6 @@ Once you have created the secrets that store the certificate info (or specified `enableTls: yes` -[#video] -== Getting started with Kubernetes video - -Follow along with this video from our *Five Minutes About {pulsar-short}* series to get started with a Helm installation. - -video::hEBP_IVQqQM[youtube, list=PL2g2h-wyI4SqeKH16czlcQ5x4Q_z-X7_m] - == Next steps To learn about installing Luna Streaming for Bare Metal/VM, see xref:install-upgrade:quickstart-server-installs.adoc[Quickstart for Bare Metal/VM Installs]. \ No newline at end of file diff --git a/modules/operations/pages/troubleshooting.adoc b/modules/operations/pages/troubleshooting.adoc index d068ba57..a800d5ac 100644 --- a/modules/operations/pages/troubleshooting.adoc +++ b/modules/operations/pages/troubleshooting.adoc @@ -1,11 +1,5 @@ = Troubleshooting -To troubleshoot Luna Streaming, look at these potential issues: - -* xref:troubleshooting.adoc#checkquotas[**Insufficient Quota**] -* xref:troubleshooting.adoc#containerspending[**Containers in Pending State**] -* xref:troubleshooting.adoc#bastionpod[**Using the Bastion Pod**] - [#checkquotas] == Check quotas From 950f09aed7578568cc003573892aff6f4a6aab1b Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:02:59 -0800 Subject: [PATCH 6/8] remove collapsible --- modules/operations/pages/auth.adoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/operations/pages/auth.adoc b/modules/operations/pages/auth.adoc index d81c6ddd..9bab6726 100644 --- a/modules/operations/pages/auth.adoc +++ b/modules/operations/pages/auth.adoc @@ -121,8 +121,6 @@ pulsar@pulsar-bastion-85c9b777f6-gt9ct:/pulsar$ curl -d "client_id=test-client" ---- + .Results -[%collapsible] -==== [source,json] ---- { @@ -134,7 +132,6 @@ pulsar@pulsar-bastion-85c9b777f6-gt9ct:/pulsar$ curl -d "client_id=test-client" "scope":"email profile" } ---- -==== . Copy the `access_token` contents and use it in the `pulsar-admin` command's `--auth-params` option: + From ba6309a03e5b340f7830eb272b0714be0aeae549 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:13:11 -0800 Subject: [PATCH 7/8] remove tabs --- .../pages/quickstart-helm-installs.adoc | 17 ++----- modules/operations/pages/functions.adoc | 48 +++++++------------ 2 files changed, 22 insertions(+), 43 deletions(-) diff --git a/modules/install-upgrade/pages/quickstart-helm-installs.adoc b/modules/install-upgrade/pages/quickstart-helm-installs.adoc index 82e1060f..2bb11873 100644 --- a/modules/install-upgrade/pages/quickstart-helm-installs.adoc +++ b/modules/install-upgrade/pages/quickstart-helm-installs.adoc @@ -69,13 +69,10 @@ For a component like BookKeeper, which requires stateful storage, we need to ove There are two ways to override `default_storage`: -* Leave `existingStorageClassName` blank and specify the storage class parameters below. +* Leave `existingStorageClassName` blank and specify the storage class parameters for your cloud provider: + -[tabs] -==== -GKE:: +** GKE: + --- [source,yaml] ---- volumes: @@ -90,11 +87,9 @@ GKE:: extraParams: replication-type: none ---- --- -EKS:: +** EKS: + --- [source,yaml] ---- volumes: @@ -109,11 +104,9 @@ EKS:: extraParams: iopsPerGB: "10" ---- --- -AKS:: +** AKS: + --- [source,yaml] ---- volumes: @@ -128,8 +121,6 @@ AKS:: extraParams: replication-type: none ---- --- -==== * Create a custom storage configuration as a `yaml` file (https://github.com/datastax/pulsar-helm-chart/blob/master/helm-chart-sources/pulsar/templates/bookkeeper/bookkeeper-storageclass.yaml[like the {company} example]) and tell the Helm chart to use that storage configuration when it creates the BookKeeper PVCs. + diff --git a/modules/operations/pages/functions.adoc b/modules/operations/pages/functions.adoc index 91e85655..26e152aa 100644 --- a/modules/operations/pages/functions.adoc +++ b/modules/operations/pages/functions.adoc @@ -14,13 +14,10 @@ https://github.com/datastax/pulsar[Luna Streaming 2.10 or later] is required to == Manage functions using {pulsar-short} Admin CLI -Add functions using the {pulsar-short} Admin CLI. +There are multiple ways to add functions using the {pulsar-short} Admin CLI. + +=== Deploy custom function code -[tabs] -====== -Deploy custom function code:: -+ --- . Create a file that contains your function code. + For example purposes, you can use the following sample function that consumes a message from one topic, adds an exclamation point, and then publishes the result to another topic. @@ -61,29 +58,24 @@ If the function is set up and ready to accept messages, then the output is `Crea ./pulsar-admin functions list --tenant $TENANT ---- --- +=== Deploy {company} transform functions -Deploy {company} transform functions:: -+ --- Transform functions are built-in functions that perform common data transformation tasks, such as casting data types, dropping fields, flattening nested structures, and merging or unwrapping KeyValue pairs. For more information, see the xref:operations:functions-transform.adoc[transform functions configuration reference]. You can find the transform function `.nar` in the `/functions` directory of your {pulsar-short} deployment. -[tabs] -==== -{pulsar-short} standalone:: -+ +==== Deploy transform functions in {pulsar-short} standalone + To deploy the built-in transform function locally in {pulsar-short} standalone, do the following: -+ + . Start {pulsar-short} standalone: + [source,shell] ---- ./bin/pulsar standalone ---- -+ + . Create a transform function in `localrun` mode: + [source,bash] @@ -98,10 +90,10 @@ To deploy the built-in transform function locally in {pulsar-short} standalone, + For the `user-config` parameters, see the xref:operations:functions-transform.adoc[transform functions configuration reference]. -{pulsar-short} cluster:: -+ +==== Deploy transform functions in a {pulsar-short} cluster + To deploy a built-in transform function to a {pulsar-short} cluster, do the following: -+ + . Create a built-in transform function with the {pulsar-short} CLI: + [source,bash] @@ -117,10 +109,9 @@ To deploy a built-in transform function to a {pulsar-short} cluster, do the foll ---- + This option deployed the functions using the transform function `.nar`. -+ You can also deploy individual functions by passing the function configuration in the `--user-config` parameter. For more information, see the xref:operations:functions-transform.adoc[transform functions configuration reference]. -+ + . Confirm the function was created by listing the functions in the tenant: + [source,bash] @@ -128,15 +119,16 @@ For more information, see the xref:operations:functions-transform.adoc[transform ./pulsar-admin functions list --tenant $TENANT ---- -{pulsar-short} sink:: +=== Deploy a function in a {pulsar-short} sink + By default, functions modify data either after it is written to a topic by a source connector, or before it is read from a topic by a sink connector. This requires either a custom connector or an intermediate topic (with additional storage, IO, and latency). -+ + Alternatively, you can deploy functions in a sink to apply preprocessing to sink topic writes (outgoing messages from the sink). -+ + To deploy a function in a sink, include the function path and configuration when you create the sink connector. For example: -+ + [source,bash] ---- pulsar-admin sinks create \ @@ -148,12 +140,8 @@ pulsar-admin sinks create \ --transform-function "builtin://transforms" \ --transform-function-config '{"steps": [{"type": "drop-fields", "fields": "password"}, {"type": "merge-key-value"}, {"type": "unwrap-key-value"}' ---- -==== - --- -====== -=== Trigger with CLI +== Trigger functions with the {pulsar-short} CLI Triggering a function is a convenient way to test that the function is working. When you trigger a function, you are publishing a message on the function's input topic, which triggers the function to run. From 61c1d0e3b592457e7da4520c36657595e276d6cf Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Thu, 29 Jan 2026 04:50:56 -0800 Subject: [PATCH 8/8] missing attribute --- antora.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/antora.yml b/antora.yml index f4a6c756..9e77f227 100644 --- a/antora.yml +++ b/antora.yml @@ -8,6 +8,7 @@ nav: asciidoc: attributes: + product: 'Luna Streaming' luna-version: '2.10' pulsar-version: '2.10' admin-console-version: '2.0.4'