From 64022c9deb2741689aa25f7f745f09b8dccb887b Mon Sep 17 00:00:00 2001 From: Marek Schmidt Date: Tue, 21 Jul 2026 11:23:33 +0200 Subject: [PATCH 1/3] initial attempts at radosgw setup --- rook-kafka-connection-setup.sh | 253 +++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100755 rook-kafka-connection-setup.sh diff --git a/rook-kafka-connection-setup.sh b/rook-kafka-connection-setup.sh new file mode 100755 index 0000000..8f32ea2 --- /dev/null +++ b/rook-kafka-connection-setup.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash + +# Ceph/Rook Kafka Connection setup with defaults +# +# Prerequisites: +# - OpenShift Data Foundation installed in `openshift-storage` +# - Strimzi Kafka cluster `my-cluster` running in `kafka` namespace +# +# Before running this script, install the CRD and the objectbucket-notifications-adapter from this repo: +# make install +# make deploy-kafka IMG=/objectbucket-notifications-adapter:tag + +set -Eeuxo pipefail + +rgw_adapter_topic="objectbucket-notifications-adapter-notifications-rgw" +rgw_adapter_namespace="objectbucket-notifications-adapter-system" +kafka_namespace="kafka" +kafka_cluster="my-cluster" +notification_name="objectbucket-notifications-adapter" +notification_kafka_user="rgw-objectbucket-notifications-adapter" + +# --- Strimzi KafkaTopic and KafkaUsers --- + +cat <= 4.0.0 because of https://github.com/confluentinc/librdkafka/pull/4895 , rhel9 image has librdkafka-1.6.1-102.el9.x86_64 , which doesn't have the fix +#cat <Kafka connection secret --- +# strimzi_crt=$(oc -n "${kafka_namespace}" get secret my-cluster-cluster-ca-cert --template='{{index .data "ca.crt"}}' | base64 --decode ) + +#rgw_kafka_cacrt=$(oc -n "${kafka_namespace}" get secret "${notification_kafka_user}" --template='{{index .data "ca.crt"}}' | base64 --decode ) +#rgw_kafka_usercrt=$(oc -n "${kafka_namespace}" get secret "${notification_kafka_user}" --template='{{index .data "user.crt"}}' | base64 --decode ) +#rgw_kafka_userkey=$(oc -n "${kafka_namespace}" get secret "${notification_kafka_user}" --template='{{index .data "user.key"}}' | base64 --decode ) + +#rgw_kafka_username="${notification_kafka_user}" +#rgw_kafka_password=$(oc -n "${kafka_namespace}" get secret "${notification_kafka_user}" --template='{{index .data "password"}}' | base64 --decode ) + +# CA secret +#oc delete secret --namespace openshift-storage "${notification_kafka_user}-kafka" 2>/dev/null || true +#oc create secret --namespace openshift-storage generic "${notification_kafka_user}-kafka" \ +# --from-literal=ca.crt="${rgw_kafka_cacrt}" \ +# --from-literal=user.crt="${rgw_kafka_usercrt}" \ +# --from-literal=user.key="${rgw_kafka_userkey}" + +# Patch the cephObjectStores reconcileStrategy to Init so that it allows us adding the additionalVolumeMounts +oc patch storagecluster ocs-storagecluster -n openshift-storage \ + --type=merge \ + -p '{"spec":{"managedResources":{"cephObjectStores":{"reconcileStrategy":"init"}}}}' + +echo "Sleeping for 10s to let the StorageCluster update the reconcileStrategy" +sleep 10 + +# Mount the Kafka Secret +cat </dev/null || true +#oc create secret --namespace openshift-storage generic "${notification_kafka_user}" \ +# --from-literal=user="${notification_kafka_user}" \ +# --from-literal=password="${rgw_kafka_password}" + +# Create a CephObjectStoreUser so we have credentials for the CreateTopic command +cat </dev/null) +ROOK_SECRET_KEY=$(oc extract secret/$topicAdminSecretName -n openshift-storage --keys=SecretKey --to=- 2>/dev/null) + +S3_ENDPOINT=https://$(oc get route ocs-storagecluster-cephobjectstore-secure -n openshift-storage -o json | jq -r ".spec.host") +aws_alias() { + AWS_ACCESS_KEY_ID=$ROOK_ACCESS_KEY AWS_SECRET_ACCESS_KEY=$ROOK_SECRET_KEY aws --endpoint "$S3_ENDPOINT" --no-verify-ssl "$@" +} + +# TODO: cannot use SCRAM-SHA-512 because of https://github.com/confluentinc/librdkafka/pull/4895 , rhel9 image has librdkafka-1.6.1-102.el9.x86_64 , which doesn't have the fix +#aws_alias sns create-topic \ +# --region default \ +# --name "${rgw_adapter_topic}" \ +# --attributes '{ +# "push-endpoint": "kafka://'${notification_kafka_user}:${rgw_kafka_password}@${kafka_cluster}'-kafka-bootstrap.'${kafka_namespace}.svc:9094'", +# "use-ssl": "true", +# "ca-location": "/var/rgw/kafka-ca/ca.crt", +# "mechanism": "SCRAM-SHA-512", +# "kafka-ack-level": "broker" +#}' + +# TODO: Cannot use tls either, as ceph v20.1.0 does not yet implement mTLS (user cert/key) +#aws_alias sns create-topic \ +# --region default \ +# --name "${rgw_adapter_topic}" \ +# --attributes '{ +# "push-endpoint": "kafka://'${kafka_cluster}'-kafka-bootstrap.'${kafka_namespace}.svc:9093'", +# "use-ssl": "true", +# "ca-location": "/var/rgw/'${notification_kafka_user}-kafka'/ca.crt", +# "kafka-ack-level": "broker" +#}' + +# TODO: So instead relying on an anonymous admin user +aws_alias sns create-topic \ + --region default \ + --name "${rgw_adapter_topic}" \ + --attributes '{ + "push-endpoint": "kafka://'${kafka_cluster}'-kafka-bootstrap.'${kafka_namespace}.svc:9092'", + "use-ssl": "false", + "kafka-ack-level": "broker" +}' + +#aws_alias sns create-topic \ +# --region default \ +# --name "${rgw_adapter_topic}" \ +# --attributes '{ +# "push-endpoint": "kafka://'${kafka_cluster}'-kafka-bootstrap.'${kafka_namespace}.svc:9092'", +# "use-ssl": "false", +# "kafka-ack-level": "broker" +#}' + +#cat </dev/null || true + +adapter_password=$(oc get secret -n "${kafka_namespace}" objectbucket-notifications-adapter-user \ + -o jsonpath='{.data.password}' | base64 --decode) + +oc delete secret -n "${rgw_adapter_namespace}" objectbucket-notifications-adapter-user 2>/dev/null || true +oc create secret generic objectbucket-notifications-adapter-user -n "${rgw_adapter_namespace}" \ + --from-literal=protocol=SASL_PLAINTEXT \ + --from-literal=sasl.mechanism=SCRAM-SHA-512 \ + --from-literal=user=objectbucket-notifications-adapter-user \ + --from-literal=password="${adapter_password}" + + From af935a33426a566194a0d550608246363a7769c4 Mon Sep 17 00:00:00 2001 From: Marek Schmidt Date: Tue, 21 Jul 2026 12:49:06 +0200 Subject: [PATCH 2/3] adding support for ceph/radosgw --- PLAN | 51 ++++++++----- README.md | 44 +++++++---- cmd/main.go | 62 +++++++++++---- config/kafka/kafka_env_patch.yaml | 4 +- config/manager/manager.yaml | 16 +++- .../objectbucketsource_controller.go | 76 +++++++++++++++++-- internal/notificationserver/server.go | 6 +- noobaa-connection-setup.sh | 6 +- noobaa-kafka-connection-setup.sh | 4 +- rook-kafka-connection-setup.sh | 2 +- 10 files changed, 202 insertions(+), 69 deletions(-) diff --git a/PLAN b/PLAN index 0bb7d63..6b64131 100644 --- a/PLAN +++ b/PLAN @@ -1,8 +1,8 @@ ObjectBucket Notifications Adapter ================================== -The adapter will be an HTTP server and/or Kafka consumer running in a pod inside a k8s cluster, configured in noobaa as "bucketNotifications" connection (either as HTTP notifications, or Kafka notifications) -It will distribute the notifications received from NooBaa to individual triggers, according to the configuration specified by ObjectBucketSource custom resources. +The adapter will be an HTTP server and/or Kafka consumer running in a pod inside a k8s cluster, configured in noobaa/radosgw as "bucketNotifications" connection (either as HTTP notifications, or Kafka notifications) +It will distribute the notifications received from NooBaa or radosgw to individual triggers, according to the configuration specified by ObjectBucketSource custom resources. The notifications to individual triggers will be sent as cloudevents via HTTP or Kafka It will be implemented in go, using k8s client libraries, the cloudevents k8s libraries, aws s3 client library, https://github.com/IBM/sarama, and the operator sdk (https://sdk.operatorframework.io) @@ -10,8 +10,8 @@ NooBaa HTTP connection setup (manual, one-time) ----------------------------------------------- Before the adapter can receive bucket notifications via HTTP, it must be registered as an -HTTP notification connection in NooBaa. This is a manual, one-time setup step -performed by a cluster administrator. +HTTP notification connection in NooBaa/radosgw. This is a manual, one-time setup step +performed by a cluster administrator. Example for Noobaa: 1. Create a Secret in the openshift-storage namespace containing the connection details for the adapter. The Secret holds a JSON file describing the HTTP @@ -53,7 +53,7 @@ oc patch noobaa noobaa --type='merge' -n openshift-storage -p '{ ``` The adapter's deployment should be configured with environment variables that -identify its connection name (ADAPTER_ID and ADAPTER_TOPIC). When the operator +identify its connection name (NOOBAA_ADAPTER_ID and NOOBAA_ADAPTER_TOPIC_ARN). When the operator calls put-bucket-notification for a bucket, it uses these values so that NooBaa knows where to deliver notifications (HTTP endpoint or Kafka topic, depending on NOTIFICATIONS_MODE). @@ -181,15 +181,15 @@ and use those connection details and keys to issue the request to noobaa using S ``` aws s3api put-bucket-notification --bucket "foo-bucket" --notification-configuration '{ "TopicConfiguration": { - "Id": "$ADAPTER_ID", + "Id": "$NOOBAA_ADAPTER_ID", "Events": ["s3:ObjectCreated:*"], - "Topic": "$ADAPTER_TOPIC" + "Topic": "$NOOBAA_ADAPTER_TOPIC_ARN" } }' ``` The Topic in the S3 notification configuration is always a reference to the -NooBaa connection secret (ADAPTER_TOPIC, e.g. "mcg-adapter-connection/connect.json"). +NooBaa connection secret (NOOBAA_ADAPTER_TOPIC_ARN, e.g. "mcg-adapter-connection/connect.json"). It is not the Kafka topic itself — for Kafka mode, the actual Kafka topic is the "topic" field inside the connect.json (see the NooBaa Kafka connection setup section). For HTTP mode, the connect.json contains the HTTP endpoint details @@ -417,7 +417,7 @@ oc patch noobaa noobaa --type='merge' -n openshift-storage -p '{ ``` 4. When the operator calls put-bucket-notification for a bucket, it uses - ADAPTER_ID and ADAPTER_TOPIC, e.g.: + NOOBAA_ADAPTER_ID and NOOBAA_ADAPTER_TOPIC_ARN, e.g.: ``` aws s3api put-bucket-notification --bucket "foo-bucket" \ @@ -431,33 +431,44 @@ aws s3api put-bucket-notification --bucket "foo-bucket" \ ``` -Kafka configuration +Configuration ------------------- -Kafka configuration is defined via env on the adapter Deployment: +Configuration is defined via env on the adapter Deployment: - ADAPTER_ID: identifier for bucket notification configuration + NOOBAA_ADAPTER_ID: identifier for NooBaa bucket notification configuration (default: "mcg-adapter") - ADAPTER_TOPIC: NooBaa connection secret reference used as TopicArn in + NOOBAA_ADAPTER_TOPIC_ARN: NooBaa connection secret reference used as TopicArn in put-bucket-notification calls (default: "mcg-adapter-connection/connect.json") + NOOBAA_ADAPTER_STORAGECLASS_PATTERN: A regular expression on OBC + .spec.storageClassName to classify the OBC as a NooBaa-managed OBC + (default: ".*noobaa\.io$") + RADOSGW_ADAPTER_ID: identifier for RadosGW bucket notification configuration + (default: "rgw-adapter") + RADOSGW_ADAPTER_TOPIC_ARN: RadosGW TopicArn in put-bucket-notification calls + (default: "arn:aws:sns:ocs-storagecluster-cephobjectstore::rgw-adapter-notifications") + RADOSGW_ADAPTER_STORAGECLASS_PATTERN: A regular expression on OBC + .spec.storageClassName to classify the OBC as a ceph/radosgw-managed OBC + (default: ".*ceph-rgw$") NOTIFICATIONS_MODE: "http" or "kafka" — selects whether the adapter - receives NooBaa bucket notifications via its HTTP server or by + receives NooBaa/RadosGW bucket notifications via its HTTP server or by consuming from a Kafka topic (default: "http") KAFKA_BROKERS: comma-separated list of host:port (required in both modes — even in HTTP mode the adapter needs Kafka to produce cloudevents to trigger Kafka topics) KAFKA_SECRET: (required in both modes, same reason as KAFKA_BROKERS) - KAFKA_NOTIFICATIONS_TOPIC: the Kafka topic to consume NooBaa bucket - notifications from (e.g. "mcg-adapter-notifications"). Only - required when NOTIFICATIONS_MODE=kafka. + KAFKA_NOTIFICATIONS_TOPIC: comma-separated list of the Kafka topics to + consume NooBaa/RadosGW bucket notifications from + (e.g. "mcg-adapter-notifications,rgw-adapter-notifications"). + Only required when NOTIFICATIONS_MODE=kafka. KAFKA_NOTIFICATIONS_GROUP_ID: the consumer group ID for consuming - NooBaa notifications (e.g. "mcg-adapter-notifications"). Only - required when NOTIFICATIONS_MODE=kafka. + NooBaa/RadosGW notifications (e.g. "objectbucket-notifications-adapter"). + Only required when NOTIFICATIONS_MODE=kafka. The same Kafka credentials (KAFKA_SECRET) are used for both consuming -notifications from NooBaa and producing cloudevents to trigger Kafka topics. +notifications from NooBaa/RadosGW and producing cloudevents to trigger Kafka topics. The format of the kafka secret is defined in ./kafka-secret-format.md diff --git a/README.md b/README.md index 99a711f..457eadc 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # ObjectBucket Notifications Adapter -A Kubernetes operator that receives S3 bucket notifications from NooBaa (Multicloud Object Gateway) and dispatches them as CloudEvents to configured sink endpoints. +A Kubernetes operator that receives S3 bucket notifications from NooBaa (Multicloud Object Gateway) and/or Ceph RadosGW and dispatches them as CloudEvents to configured sink endpoints. ## Description -The ObjectBucket Notifications Adapter runs inside a Kubernetes cluster, registered as a `bucketNotifications` connection in NooBaa. It can receive notifications via HTTP (default) or by consuming from a Kafka topic. It watches `ObjectBucketSource` custom resources to determine which S3 events from which buckets should be forwarded to which endpoints. When NooBaa delivers a notification, the adapter matches the event against all configured sources and dispatches CloudEvents via HTTP or Kafka to the matching sinks. +The ObjectBucket Notifications Adapter runs inside a Kubernetes cluster, registered as a `bucketNotifications` connection in NooBaa and/or as an SNS topic in RadosGW. It can receive notifications via HTTP (default) or by consuming from Kafka topics. It watches `ObjectBucketSource` custom resources to determine which S3 events from which buckets should be forwarded to which endpoints. When a notification arrives, the adapter matches the event against all configured sources and dispatches CloudEvents via HTTP or Kafka to the matching sinks. + +The adapter determines whether an OBC is managed by NooBaa or RadosGW by matching the OBC's `spec.storageClassName` against configurable regex patterns, and uses the corresponding adapter ID and topic ARN when setting up bucket notifications. ## Custom Resource: ObjectBucketSource @@ -50,26 +52,30 @@ The adapter is configured via environment variables: | Variable | Default | Description | |---|---|---| -| `ADAPTER_ID` | `objectbucket-notifications-adapter` | Identifier used in the S3 bucket notification configuration | -| `ADAPTER_TOPIC` | `objectbucket-notifications-adapter-connection/connect.json` | NooBaa connection secret reference used as TopicArn in put-bucket-notification calls | +| `NOOBAA_ADAPTER_ID` | `mcg-adapter` | Identifier used in the S3 bucket notification configuration for NooBaa-managed OBCs | +| `NOOBAA_ADAPTER_TOPIC_ARN` | `mcg-adapter-connection/connect.json` | NooBaa connection secret reference used as TopicArn in put-bucket-notification calls | +| `NOOBAA_ADAPTER_STORAGECLASS_PATTERN` | `.*noobaa\.io$` | Regex matched against OBC `spec.storageClassName` to classify as NooBaa-managed | +| `RADOSGW_ADAPTER_ID` | `rgw-adapter` | Identifier used in the S3 bucket notification configuration for RadosGW-managed OBCs | +| `RADOSGW_ADAPTER_TOPIC_ARN` | `arn:aws:sns:ocs-storagecluster-cephobjectstore::rgw-adapter-notifications` | RadosGW SNS TopicArn used in put-bucket-notification calls | +| `RADOSGW_ADAPTER_STORAGECLASS_PATTERN` | `.*ceph-rgw$` | Regex matched against OBC `spec.storageClassName` to classify as RadosGW-managed | | `ADAPTER_PORT` | `8888` | Port the notification HTTP server listens on (HTTP mode only) | -| `NOTIFICATIONS_MODE` | `http` | `http` or `kafka` — selects how the adapter receives NooBaa notifications | +| `NOTIFICATIONS_MODE` | `http` | `http` or `kafka` — selects how the adapter receives NooBaa/RadosGW notifications | | `KAFKA_BROKERS` | _(none)_ | Comma-separated list of Kafka broker addresses. Required for Kafka sinks and when `NOTIFICATIONS_MODE=kafka`. | | `KAFKA_SECRET` | _(none)_ | Name of a Kubernetes Secret (in the adapter's namespace) containing Kafka credentials. See `kafka-secret-format.md`. | -| `KAFKA_NOTIFICATIONS_TOPIC` | _(none)_ | Kafka topic to consume NooBaa notifications from. Required when `NOTIFICATIONS_MODE=kafka`. | -| `KAFKA_NOTIFICATIONS_GROUP_ID` | _(none)_ | Consumer group ID for consuming NooBaa notifications. Required when `NOTIFICATIONS_MODE=kafka`. | +| `KAFKA_NOTIFICATIONS_TOPIC` | _(none)_ | Comma-separated list of Kafka topics to consume NooBaa/RadosGW notifications from (e.g. `mcg-adapter-notifications,rgw-adapter-notifications`). Required when `NOTIFICATIONS_MODE=kafka`. | +| `KAFKA_NOTIFICATIONS_GROUP_ID` | _(none)_ | Consumer group ID for consuming NooBaa/RadosGW notifications. Required when `NOTIFICATIONS_MODE=kafka`. | ### NooBaa Connection Setup (HTTP mode) -Before the adapter can receive notifications via HTTP, register it as an HTTP connection in NooBaa (one-time setup): +Before the adapter can receive notifications via HTTP, register it as an HTTP connection in NooBaa (one-time setup). A helper script `noobaa-connection-setup.sh` is provided. Manual steps: 1. Create the connection secret: ```sh -oc create secret generic objectbucket-notifications-adapter-connection \ +oc create secret generic mcg-adapter-connection \ --from-file=connect.json=/dev/stdin -n openshift-storage <..svc.cluster.local", @@ -86,7 +92,7 @@ existing_connections=$(oc get noobaa noobaa -n openshift-storage -o json \ | jq -c '.spec.bucketNotifications.connections // []') updated_connections=$(echo "$existing_connections" | jq -c \ - --arg name "objectbucket-notifications-adapter-connection" \ + --arg name "mcg-adapter-connection" \ '[.[] | select(.name != $name)] + [{"name": $name, "namespace": "openshift-storage"}]') oc patch noobaa noobaa --type='merge' -n openshift-storage -p '{ @@ -101,15 +107,25 @@ oc patch noobaa noobaa --type='merge' -n openshift-storage -p '{ ### NooBaa Connection Setup (Kafka mode) -As an alternative to HTTP, the adapter can consume NooBaa notifications from a Kafka topic. This requires a Strimzi-managed Kafka cluster. See the `PLAN` file for detailed setup steps, including creating the KafkaTopic, KafkaUsers, and the NooBaa Kafka connection secret. +As an alternative to HTTP, the adapter can consume NooBaa notifications from a Kafka topic. This requires a Strimzi-managed Kafka cluster. A helper script `noobaa-kafka-connection-setup.sh` is provided. See the `PLAN` file for detailed steps. Summary of the one-time setup: -1. Create a `KafkaTopic` (e.g. `objectbucket-notifications-adapter-notifications`) and `KafkaUser` resources in Strimzi -2. Create a NooBaa connection secret with `notification_protocol: kafka` pointing to the Kafka bootstrap and topic +1. Create a `KafkaTopic` (e.g. `mcg-adapter-notifications`) and `KafkaUser` resources in Strimzi +2. Create a NooBaa connection secret (`mcg-adapter-connection`) with `notification_protocol: kafka` pointing to the Kafka bootstrap and topic 3. Patch the NooBaa CR to register the Kafka connection 4. Create a Kubernetes Secret in the adapter's namespace with the adapter's Kafka credentials (see `kafka-secret-format.md`) +### RadosGW Connection Setup (Kafka mode) + +For Ceph RadosGW, bucket notifications are delivered via Kafka using SNS topics. A helper script `rook-kafka-connection-setup.sh` is provided. See the `PLAN` file for detailed steps. + +Summary of the one-time setup: + +1. Create a `KafkaTopic` (e.g. `rgw-adapter-notifications`) and `KafkaUser` resources in Strimzi +2. Create an SNS topic in RadosGW pointing to the Kafka broker and topic +3. Create a Kubernetes Secret in the adapter's namespace with the adapter's Kafka credentials (see `kafka-secret-format.md`) + ### Deploy ```sh diff --git a/cmd/main.go b/cmd/main.go index 3dd826c..14a27f2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strconv" "strings" @@ -212,14 +213,34 @@ func main() { os.Exit(1) } - adapterID := os.Getenv("ADAPTER_ID") - if adapterID == "" { - adapterID = "objectbucket-notifications-adapter" - } - adapterTopic := os.Getenv("ADAPTER_TOPIC") - if adapterTopic == "" { - adapterTopic = "objectbucket-notifications-adapter-connection/connect.json" + noobaaAdapterID := envOrDefault("NOOBAA_ADAPTER_ID", "mcg-adapter") + noobaaAdapterTopic := envOrDefault("NOOBAA_ADAPTER_TOPIC_ARN", "mcg-adapter-connection/connect.json") + noobaaStorageClassPattern := envOrDefault("NOOBAA_ADAPTER_STORAGECLASS_PATTERN", `.*noobaa\.io$`) + + radosgwAdapterID := envOrDefault("RADOSGW_ADAPTER_ID", "rgw-adapter") + radosgwAdapterTopic := envOrDefault("RADOSGW_ADAPTER_TOPIC_ARN", + "arn:aws:sns:ocs-storagecluster-cephobjectstore::rgw-adapter-notifications") + radosgwStorageClassPattern := envOrDefault("RADOSGW_ADAPTER_STORAGECLASS_PATTERN", `.*ceph-rgw$`) + + var adapterConfigs []controller.AdapterConfig + for _, cfg := range []struct { + id, topic, pattern string + }{ + {noobaaAdapterID, noobaaAdapterTopic, noobaaStorageClassPattern}, + {radosgwAdapterID, radosgwAdapterTopic, radosgwStorageClassPattern}, + } { + re, err := regexp.Compile(cfg.pattern) + if err != nil { + setupLog.Error(err, "invalid storageclass pattern", "pattern", cfg.pattern) + os.Exit(1) + } + adapterConfigs = append(adapterConfigs, controller.AdapterConfig{ + ID: cfg.id, + Topic: cfg.topic, + StorageClassPattern: re, + }) } + adapterPort := 8888 if portStr := os.Getenv("ADAPTER_PORT"); portStr != "" { var err error @@ -238,7 +259,14 @@ func main() { os.Exit(1) } - kafkaNotificationsTopic := os.Getenv("KAFKA_NOTIFICATIONS_TOPIC") + var kafkaNotificationsTopics []string + if topicsStr := os.Getenv("KAFKA_NOTIFICATIONS_TOPIC"); topicsStr != "" { + for _, t := range strings.Split(topicsStr, ",") { + if trimmed := strings.TrimSpace(t); trimmed != "" { + kafkaNotificationsTopics = append(kafkaNotificationsTopics, trimmed) + } + } + } kafkaNotificationsGroupID := os.Getenv("KAFKA_NOTIFICATIONS_GROUP_ID") var kafkaBrokers []string @@ -283,10 +311,9 @@ func main() { } if err := (&controller.ObjectBucketSourceReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - AdapterID: adapterID, - AdapterTopic: adapterTopic, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + AdapterConfigs: adapterConfigs, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ObjectBucketSource") os.Exit(1) @@ -294,7 +321,7 @@ func main() { // +kubebuilder:scaffold:builder if notificationsMode == "kafka" { - if kafkaNotificationsTopic == "" { + if len(kafkaNotificationsTopics) == 0 { setupLog.Error(fmt.Errorf("KAFKA_NOTIFICATIONS_TOPIC is required when NOTIFICATIONS_MODE=kafka"), "missing env") os.Exit(1) } @@ -314,7 +341,7 @@ func main() { KafkaBrokers: kafkaBrokers, KafkaConfig: kafkaCfg, NotificationsMode: notificationsMode, - KafkaNotificationsTopic: kafkaNotificationsTopic, + KafkaNotificationsTopics: kafkaNotificationsTopics, KafkaNotificationsGroupID: kafkaNotificationsGroupID, } if err := mgr.Add(notifServer); err != nil { @@ -353,3 +380,10 @@ func main() { os.Exit(1) } } + +func envOrDefault(key, defaultValue string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultValue +} diff --git a/config/kafka/kafka_env_patch.yaml b/config/kafka/kafka_env_patch.yaml index 201e8a2..e7b8063 100644 --- a/config/kafka/kafka_env_patch.yaml +++ b/config/kafka/kafka_env_patch.yaml @@ -15,6 +15,6 @@ spec: - name: KAFKA_SECRET value: "objectbucket-notifications-adapter-user" - name: KAFKA_NOTIFICATIONS_TOPIC - value: "objectbucket-notifications-adapter-notifications" + value: "mcg-adapter-notifications,rgw-adapter-notifications" - name: KAFKA_NOTIFICATIONS_GROUP_ID - value: "objectbucket-notifications-adapter-notifications" + value: "objectbucket-notifications-adapter" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 86e8d99..e094859 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -66,10 +66,18 @@ spec: image: controller:latest name: manager env: - - name: ADAPTER_ID - value: "objectbucket-notifications-adapter" - - name: ADAPTER_TOPIC - value: "objectbucket-notifications-adapter-connection/connect.json" + - name: NOOBAA_ADAPTER_ID + value: "mcg-adapter" + - name: NOOBAA_ADAPTER_TOPIC_ARN + value: "mcg-adapter-connection/connect.json" + - name: NOOBAA_ADAPTER_STORAGECLASS_PATTERN + value: ".*noobaa\\.io$" + - name: RADOSGW_ADAPTER_ID + value: "rgw-adapter" + - name: RADOSGW_ADAPTER_TOPIC_ARN + value: "arn:aws:sns:ocs-storagecluster-cephobjectstore::rgw-adapter-notifications" + - name: RADOSGW_ADAPTER_STORAGECLASS_PATTERN + value: ".*ceph-rgw$" - name: ADAPTER_PORT value: "8888" - name: NOTIFICATIONS_MODE diff --git a/internal/controller/objectbucketsource_controller.go b/internal/controller/objectbucketsource_controller.go index 1f997ae..6842dde 100644 --- a/internal/controller/objectbucketsource_controller.go +++ b/internal/controller/objectbucketsource_controller.go @@ -19,13 +19,16 @@ package controller import ( "context" "fmt" + "regexp" "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,12 +39,24 @@ import ( "github.com/functions-dev/objectbucket-notifications-adapter/internal/s3client" ) +// AdapterConfig holds the notification configuration for a specific storage backend. +type AdapterConfig struct { + ID string + Topic string + StorageClassPattern *regexp.Regexp +} + +var obcGVR = schema.GroupVersionResource{ + Group: "objectbucket.io", + Version: "v1alpha1", + Resource: "objectbucketclaims", +} + // ObjectBucketSourceReconciler reconciles an ObjectBucketSource object type ObjectBucketSourceReconciler struct { client.Client - Scheme *runtime.Scheme - AdapterID string - AdapterTopic string + Scheme *runtime.Scheme + AdapterConfigs []AdapterConfig } // +kubebuilder:rbac:groups=sources.functions.dev,resources=objectbucketsources,verbs=get;list;watch;create;update;patch;delete @@ -49,6 +64,7 @@ type ObjectBucketSourceReconciler struct { // +kubebuilder:rbac:groups=sources.functions.dev,resources=objectbucketsources/finalizers,verbs=update // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +// +kubebuilder:rbac:groups=objectbucket.io,resources=objectbucketclaims,verbs=get;list;watch func (r *ObjectBucketSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := logf.FromContext(ctx) @@ -76,6 +92,13 @@ func (r *ObjectBucketSourceReconciler) Reconcile(ctx context.Context, req ctrl.R obcName := source.Spec.ObjectBucketClaim.Name ns := source.Namespace + adapterCfg, err := r.resolveAdapterConfig(ctx, ns, obcName) + if err != nil { + log.Info("cannot resolve adapter config for OBC, requeuing", "obc", obcName, "error", err) + r.setCondition(ctx, &source, sourcesv1alpha1.ConditionOBCCredentialsAvailable, metav1.ConditionFalse, "OBCNotReady", err.Error()) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + bucketHost, bucketName, bucketPort, err := r.readOBCConfigMap(ctx, ns, obcName) if err != nil { log.Info("OBC ConfigMap not available, requeuing", "obc", obcName, "error", err) @@ -100,13 +123,13 @@ func (r *ObjectBucketSourceReconciler) Reconcile(ctx context.Context, req ctrl.R endpoint := fmt.Sprintf("https://%s:%s", bucketHost, bucketPort) s3c := s3client.NewS3Client(endpoint, accessKey, secretKey) - if err := s3client.PutBucketNotification(ctx, s3c, bucketName, r.AdapterID, r.AdapterTopic, mergedEvents); err != nil { + if err := s3client.PutBucketNotification(ctx, s3c, bucketName, adapterCfg.ID, adapterCfg.Topic, mergedEvents); err != nil { log.Error(err, "failed to set bucket notification", "bucket", bucketName) r.setCondition(ctx, &source, sourcesv1alpha1.ConditionBucketNotificationSet, metav1.ConditionFalse, "PutNotificationFailed", err.Error()) return ctrl.Result{}, err } - log.Info("bucket notification set", "bucket", bucketName, "events", mergedEvents) + log.Info("bucket notification set", "bucket", bucketName, "events", mergedEvents, "adapterID", adapterCfg.ID) r.setCondition(ctx, &source, sourcesv1alpha1.ConditionBucketNotificationSet, metav1.ConditionTrue, "NotificationConfigured", "Bucket notification configured successfully") return ctrl.Result{}, nil @@ -122,9 +145,16 @@ func (r *ObjectBucketSourceReconciler) reconcileDelete(ctx context.Context, sour obcName := source.Spec.ObjectBucketClaim.Name ns := source.Namespace + adapterCfg, cfgErr := r.resolveAdapterConfig(ctx, ns, obcName) + if cfgErr != nil { + log.Info("cannot resolve adapter config during deletion, removing finalizer anyway", "obc", obcName, "error", cfgErr) + } + bucketHost, bucketName, bucketPort, err := r.readOBCConfigMap(ctx, ns, obcName) if err != nil { log.Info("OBC ConfigMap not available during deletion, removing finalizer anyway", "obc", obcName) + } else if cfgErr != nil { + log.Info("adapter config not available during deletion, removing finalizer anyway", "obc", obcName) } else { accessKey, secretKey, secretErr := r.readOBCSecret(ctx, ns, obcName) if secretErr != nil { @@ -144,7 +174,7 @@ func (r *ObjectBucketSourceReconciler) reconcileDelete(ctx context.Context, sour log.Info("removed bucket notification", "bucket", bucketName) } } else { - if err := s3client.PutBucketNotification(ctx, s3c, bucketName, r.AdapterID, r.AdapterTopic, mergedEvents); err != nil { + if err := s3client.PutBucketNotification(ctx, s3c, bucketName, adapterCfg.ID, adapterCfg.Topic, mergedEvents); err != nil { log.Error(err, "updating bucket notification during deletion", "bucket", bucketName) } else { log.Info("updated bucket notification after deletion", "bucket", bucketName, "events", mergedEvents) @@ -226,6 +256,40 @@ func (r *ObjectBucketSourceReconciler) computeMergedEventsExcluding(ctx context. return events, nil } +func (r *ObjectBucketSourceReconciler) readOBCStorageClassName(ctx context.Context, namespace, name string) (string, error) { + var obc unstructured.Unstructured + obc.SetGroupVersionKind(schema.GroupVersionKind{ + Group: obcGVR.Group, + Version: obcGVR.Version, + Kind: "ObjectBucketClaim", + }) + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &obc); err != nil { + return "", fmt.Errorf("getting ObjectBucketClaim %s/%s: %w", namespace, name, err) + } + sc, _, _ := unstructured.NestedString(obc.Object, "spec", "storageClassName") + return sc, nil +} + +func (r *ObjectBucketSourceReconciler) resolveAdapterConfig(ctx context.Context, namespace, obcName string) (*AdapterConfig, error) { + if len(r.AdapterConfigs) == 0 { + return nil, fmt.Errorf("no adapter configs defined") + } + if len(r.AdapterConfigs) == 1 { + return &r.AdapterConfigs[0], nil + } + storageClass, err := r.readOBCStorageClassName(ctx, namespace, obcName) + if err != nil { + return nil, err + } + for i := range r.AdapterConfigs { + cfg := &r.AdapterConfigs[i] + if cfg.StorageClassPattern != nil && cfg.StorageClassPattern.MatchString(storageClass) { + return cfg, nil + } + } + return nil, fmt.Errorf("no adapter config matches storageClassName %q for OBC %s/%s", storageClass, namespace, obcName) +} + func (r *ObjectBucketSourceReconciler) setCondition(ctx context.Context, source *sourcesv1alpha1.ObjectBucketSource, condType string, status metav1.ConditionStatus, reason, message string) { meta.SetStatusCondition(&source.Status.Conditions, metav1.Condition{ Type: condType, diff --git a/internal/notificationserver/server.go b/internal/notificationserver/server.go index 8cd8a4d..a2656f2 100644 --- a/internal/notificationserver/server.go +++ b/internal/notificationserver/server.go @@ -21,7 +21,7 @@ type NotificationServer struct { KafkaBrokers []string KafkaConfig *sarama.Config NotificationsMode string - KafkaNotificationsTopic string + KafkaNotificationsTopics []string KafkaNotificationsGroupID string } @@ -83,7 +83,7 @@ func (s *NotificationServer) startKafkaConsumer(ctx context.Context, handler *no defer func() { _ = consumerGroup.Close() }() log.Info("starting kafka notification consumer", - "topic", s.KafkaNotificationsTopic, + "topics", s.KafkaNotificationsTopics, "group", s.KafkaNotificationsGroupID, "brokers", s.KafkaBrokers) @@ -96,7 +96,7 @@ func (s *NotificationServer) startKafkaConsumer(ctx context.Context, handler *no cgHandler := &consumerGroupHandler{handler: handler} for { - if err := consumerGroup.Consume(ctx, []string{s.KafkaNotificationsTopic}, cgHandler); err != nil { + if err := consumerGroup.Consume(ctx, s.KafkaNotificationsTopics, cgHandler); err != nil { if ctx.Err() != nil { return nil } diff --git a/noobaa-connection-setup.sh b/noobaa-connection-setup.sh index 9a77875..0cfc55e 100755 --- a/noobaa-connection-setup.sh +++ b/noobaa-connection-setup.sh @@ -8,10 +8,10 @@ # make install # make deploy IMG=/objectbucket-notifications-adapter:tag -oc create secret generic objectbucket-notifications-adapter-connection \ +oc create secret generic mcg-adapter-connection \ --from-file=connect.json=/dev/stdin -n openshift-storage < Date: Tue, 21 Jul 2026 13:00:14 +0200 Subject: [PATCH 3/3] lint --- cmd/main.go | 2 +- internal/controller/objectbucketsource_controller.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 14a27f2..05cd073 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -222,7 +222,7 @@ func main() { "arn:aws:sns:ocs-storagecluster-cephobjectstore::rgw-adapter-notifications") radosgwStorageClassPattern := envOrDefault("RADOSGW_ADAPTER_STORAGECLASS_PATTERN", `.*ceph-rgw$`) - var adapterConfigs []controller.AdapterConfig + adapterConfigs := make([]controller.AdapterConfig, 0, 2) for _, cfg := range []struct { id, topic, pattern string }{ diff --git a/internal/controller/objectbucketsource_controller.go b/internal/controller/objectbucketsource_controller.go index 6842dde..5db0327 100644 --- a/internal/controller/objectbucketsource_controller.go +++ b/internal/controller/objectbucketsource_controller.go @@ -41,9 +41,9 @@ import ( // AdapterConfig holds the notification configuration for a specific storage backend. type AdapterConfig struct { - ID string - Topic string - StorageClassPattern *regexp.Regexp + ID string + Topic string + StorageClassPattern *regexp.Regexp } var obcGVR = schema.GroupVersionResource{