You need to enable JavaScript to run this app.
Message Queue for Kafka

Message Queue for Kafka

Copy page
Download PDF
Java client SDK
SASL_SSL access point sending and receiving messages using SCRAM mechanism
Copy page
Download PDF
SASL_SSL access point sending and receiving messages using SCRAM mechanism
This topic uses the Java client as an example to describe how to connect to Message Queue for Kafka using the SCRAM mechanism through SASL_SSL endpoints in VPC or public network environments, and send and receive messages.​
Prerequisites​
Preparatory work has been completed. For detailed explanations, refer to Preparations.​
1. Install Java dependencies​
Add the relevant dependencies in the Java project's pom.xml file. Here is an example of Kafka version 2.2.2.​
​
<dependency>​
<groupId>org.apache.kafka</groupId>​
<artifactId>kafka-clients</artifactId>​
<version>2.2.2</version>​
</dependency>​
<dependency>​
<groupId>org.slf4j</groupId>​
<artifactId>slf4j-log4j12</artifactId>​
<version>1.7.6</version>​
</dependency>​
​
2 Add configuration file​
  1. Create the Message Queue for Kafka configuration file config.properties.​
  • For detailed descriptions of the configuration file fields, refer to configuration file.​
tip​
Under SCRAM mechanism, use a SCRAM user with corresponding Topic access rights for SASL authentication. For how to obtain the username and password, refer to 2 Collect Connection Information.​
  • When connecting using the SCRAM mechanism through a SASL_SSL access point, an example configuration file is as follows.​
  • ​
    bootstrap.servers=xxxxx​
    security.protocol=SASL_SSL​
    topic=my-topic​
    consumer.group.id=test​
    consumer.auto.offset.reset=earliest​
    consumer.enable.auto.commit=false​
    client.dns.lookup=use_all_dns_ips​
    sasl.mechanism=SCRAM-SHA-256​
    sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="xxxx" password="xxxxx";​
    ​
  1. Create the configuration file loaderKafkaConfigurer.java.​
  • ​
    package com.volcengine.openservice.kafka;​
    ​
    import java.io.FileInputStream;​
    import java.io.InputStream;​
    import java.util.Properties;​
    ​
    public class KafkaConfigurer {​
    ​
    private static Properties properties;​
    ​
    public synchronized static Properties getKafkaProperties(String path) {​
    if (null != properties) {​
    return properties;​
    }​
    Properties kafkaProperties = new Properties();​
    try {​
    FileInputStream conf = new FileInputStream(path);​
    kafkaProperties.load(conf);​
    } catch (Exception e) {​
    e.printStackTrace();​
    }​
    properties = kafkaProperties;​
    return kafkaProperties;​
    }​
    }​
    ​
3 Produce messages​
implementation method​
  1. Create message production program ProducerDemo.java.​
  1. Compile and run ProducerDemo.java to produce messages.​
  1. check the running results.​
  • the running result example is as follows.​
  • ​
    Produce ok:sasl-0@0​
    Produce ok:sasl-0@1​
    Produce ok:sasl-0@2​
    Produce ok:sasl-0@3​
    ​
tip​
Message Queue for Kafka provides an example project Demo to help you quickly integrate. After downloading and extracting the demo, you can directly execute the following commands to send messages.​
​
java -cp kafka-demo.jar com.volcengine.openservice.kafka.ProducerDemo ./ssl.config.properties​
​
Example code​
The following is example code for producing messages using the SCRAM mechanism via a SASL_SSL endpoint. You can also refer to the sample file in the demo {DemoPath}/src/main/java/com/volcengine/openservice/kafka/ProducerDemo.java to implement related business logic.​
​
package com.volcengine.openservice.kafka;​
​
import java.util.ArrayList;​
import java.util.List;​
import java.util.Properties;​
import java.util.concurrent.Future;​
​
import java.util.concurrent.TimeUnit;​
import org.apache.kafka.clients.CommonClientConfigs;​
import org.apache.kafka.clients.consumer.ConsumerConfig;​
import org.apache.kafka.clients.consumer.ConsumerRecord;​
import org.apache.kafka.clients.consumer.ConsumerRecords;​
import org.apache.kafka.clients.consumer.KafkaConsumer;​
import org.apache.kafka.common.config.SaslConfigs;​
import org.apache.kafka.clients.producer.KafkaProducer;​
import org.apache.kafka.clients.producer.ProducerConfig;​
import org.apache.kafka.clients.producer.ProducerRecord;​
import org.apache.kafka.clients.producer.RecordMetadata;​
import org.apache.kafka.common.config.SslConfigs;​
​
class Producer {​
private static String topic;​
​
private static Properties props = new Properties();​
​
private static KafkaProducer<String, String> producer;​
​
public Producer(Properties kafkaProperties) {​
setProps(kafkaProperties);​
setSasl(kafkaProperties);​
setTopic(kafkaProperties);​
newProducer();​
}​
​
private void setTopic(Properties kafkaProperties) {​
topic = kafkaProperties.getProperty("topic");​
}​
​
private void setSasl(Properties kafkaProperties) {​
String protocol = kafkaProperties.getProperty("security.protocol");​
props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, protocol);​
​
if (protocol.equals("SASL_SSL")) {​
props.put(SaslConfigs.SASL_MECHANISM, kafkaProperties.getProperty("sasl.mechanism"));​
props.put(SaslConfigs.SASL_JAAS_CONFIG, kafkaProperties.getProperty("sasl.jaas.config"));​
return;​
}​
​
throw new IllegalArgumentException("security.protocol is not correct");​
}​
​
private void setProps(Properties kafkaProperties) {​
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getProperty("bootstrap.servers"));​
​
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");​
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");​
​
props.put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, "");​
​
props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30 * 1000);​
props.put(ProducerConfig.RETRIES_CONFIG, 5);​
props.put(ProducerConfig.RECONNECT_BACKOFF_MS_CONFIG, 3000);​
props.put(ProducerConfig.CLIENT_DNS_LOOKUP_CONFIG, kafkaProperties.getProperty("client.dns.lookup"));​
}​
​
private void newProducer() {​
producer = new KafkaProducer<String, String>(props);​
}​
​
public void Produce() {​
String value = "this is demo message "; ​
​
List<Future<RecordMetadata>> futures = new ArrayList<Future<RecordMetadata>>(128);​
for (int i =0; i < 100; i++) {​
ProducerRecord<String, String> kafkaMessage = new ProducerRecord<String, String>(topic, value + ": " + i);​
Future<RecordMetadata> metadataFuture = producer.send(kafkaMessage);​
futures.add(metadataFuture);​
}​
producer.flush();​
for (Future<RecordMetadata> future: futures) {​
try {​
RecordMetadata recordMetadata = future.get();​
System.out.println("Produce ok:" + recordMetadata.toString());​
} catch (Throwable t) {​
t.printStackTrace();​
}​
}​
}​
}​
​
public class ProducerDemo {​
​
public static void main(String args[]) {​
if (args.length < 1) {​
System.out.println("please input config path");​
return;​
}​
Producer producer = new Producer(KafkaConfigurer.getKafkaProperties(args[0]));​
​
try {​
producer.Produce();​
} catch (Exception e) {​
System.out.println("error occurred");​
e.printStackTrace();​
}​
}​
​
​
}​
​
4 Consume messages​
implementation method​
  1. Create a consumer subscription program ConsumerDemo.java.​
  1. Compile and run ConsumerDemo.java to consume messages.​
  1. check the running results.​
  • the running result example is as follows.​
​
Consume: ConsumerRecord(topic = sasl, patition = 6, leaderEpoch=0, offset=0, CreateTime = 1637207092476, serialized key size = -1, serialized key size = 24, headers = ReacordHeaders(headers = [], isReadOnly = false), key = null, value = this is demo, message : 0)​
Consume: ConsumerRecord(topic = sasl, patition = 6, leaderEpoch=0, offset=0, CreateTime = 1637207092476, serialized key size = -1, serialized key size = 24, headers = ReacordHeaders(headers = [], isReadOnly = false), key = null, value = this is demo, message : 1)​
​
tip​
Message Queue for Kafka provides an example project Demo to help you quickly integrate. After downloading and extracting the sample project, you can directly run the following commands to consume messages.​
​
java -cp kafka-demo.jar com.volcengine.openservice.kafka.ConsumerDemo ./ssl.config.properties​
​
Sample code​
The following sample code demonstrates how to consume messages using the SCRAM mechanism via a SASL_SSL endpoint. You can also refer to the sample file in the demo {DemoPath}/src/main/java/com/volcengine/openservice/kafka/ConsumerDemo.java to implement related business logic.​
​
package com.volcengine.openservice.kafka;​
​
import java.util.ArrayList;​
import java.util.List;​
import java.util.Properties;​
​
import org.apache.kafka.clients.CommonClientConfigs;​
import org.apache.kafka.clients.consumer.ConsumerConfig;​
import org.apache.kafka.clients.consumer.ConsumerRecord;​
import org.apache.kafka.clients.consumer.ConsumerRecords;​
import org.apache.kafka.clients.consumer.KafkaConsumer;​
import org.apache.kafka.clients.producer.ProducerConfig;​
import org.apache.kafka.common.config.SaslConfigs;​
import org.apache.kafka.common.config.SslConfigs;​
​
class Consumer {​
private static Properties props = new Properties();​
​
private KafkaConsumer<String, String> consumer;​
​
public Consumer(Properties kafkaProperties) {​
setProps(kafkaProperties);​
setSasl(kafkaProperties);​
newConsumer();​
subscribed(kafkaProperties);​
}​
​
private void setSasl(Properties kafkaProperties) {​
String protocol = kafkaProperties.getProperty("security.protocol");​
props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, protocol);​
​
if (protocol.equals("SASL_SSL")) {​
props.put(SaslConfigs.SASL_MECHANISM, kafkaProperties.getProperty("sasl.mechanism"));​
props.put(SaslConfigs.SASL_JAAS_CONFIG, kafkaProperties.getProperty("sasl.jaas.config"));​
return;​
}​
​
throw new IllegalArgumentException("security.protocol is not correct");​
}​
​
private void setProps(Properties kafkaProperties) {​
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getProperty("bootstrap.servers"));​
​
props.put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, "");​
​
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 30000);​
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 30);​
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");​
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");​
props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProperties.getProperty("consumer.group.id"));​
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaProperties.getProperty("consumer.auto.offset.reset"));​
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaProperties.getProperty("consumer.enable.auto.commit"));​
props.put(ConsumerConfig.CLIENT_DNS_LOOKUP_CONFIG, kafkaProperties.getProperty("client.dns.lookup"));​
}​
​
private void newConsumer() {​
consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<String, String>(props);​
}​
​
private void subscribed(Properties kafkaProperties) {​
List<String> subscribedTopic = new ArrayList<String>();​
subscribedTopic.add(kafkaProperties.getProperty("topic"));​
consumer.subscribe(subscribedTopic);​
}​
​
public void poll() {​
ConsumerRecords<String, String> records = consumer.poll(1000);​
for (ConsumerRecord<String, String> record : records) {​
System.out.printf("Consume: %s%n", record.toString());​
}​
}​
}​
​
public class ConsumerDemo {​
public static void main(String args[]) {​
if (args.length < 1) {​
System.out.println("please input config path");​
return;​
}​
​
Consumer consumer = new Consumer(KafkaConfigurer.getKafkaProperties(args[0]));​
​
while (true){​
try {​
consumer.poll();​
} catch (Exception e) {​
try {​
Thread.sleep(1000);​
} catch (Throwable ignore) {​
​
}​
e.printStackTrace();​
}​
}​
}​
}​
​
​
Last updated: 2025.09.18 17:51:06