- FlinkKafkaConsumer011.setStartFromEarliest()
/**
*Specifies that consumer reads from the earliest offset of all partitions.
*This method causes consumer to ignore any submitted offset s in Zookeeper / Kafka brokers.
*
*When consumer is restored from checkpoint or savepoint, this method does not work. In this case, the offset in the restored state will be used.
*
*@ return the consumer object that allows chain method calls
*/
/**
* Specifies the consumer to start reading from the earliest offset for all partitions.
* This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers.
*
* <p>This method does not affect where partitions are read from when the consumer is restored
* from a checkpoint or savepoint. When the consumer is restored from a checkpoint or
* savepoint, only the offsets in the restored state will be used.
*
* @return The consumer object, to allow function chaining.
*/
public FlinkKafkaConsumerBase<T> setStartFromEarliest() {
this.startupMode = StartupMode.EARLIEST;
this.startupOffsetsTimestamp = null;
this.specificStartupOffsets = null;
return this;
}
- FlinkKafkaConsumer011.setStartFromLatest()
/**
*Specifies that consumer reads from the closest offset of all partitions.
*This method causes consumer to ignore any offsets submitted in Zookeeper / Kafka brokers.
*
*When consumer is restored from checkpoint or savepoint, this method does not work. In this case, the offset in the restored state will be used.
*
*@ return the consumer object that allows chain method calls
*/
/**
* Specifies the consumer to start reading from the latest offset for all partitions.
* This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers.
*
* <p>This method does not affect where partitions are read from when the consumer is restored
* from a checkpoint or savepoint. When the consumer is restored from a checkpoint or
* savepoint, only the offsets in the restored state will be used.
*
* @return The consumer object, to allow function chaining.
*/
public FlinkKafkaConsumerBase<T> setStartFromLatest() {
this.startupMode = StartupMode.LATEST;
this.startupOffsetsTimestamp = null;
this.specificStartupOffsets = null;
return this;
}
- FlinkKafkaConsumer011.setStartFromTimestamp(long startupOffsetsTimestamp)
/**
*Specifies that consumer reads the partition from the specified timestamp.
*The specified timestamp must be earlier than the current timestamp.
*This method causes consumer to ignore any submitted offset s in Zookeeper / Kafka brokers.
*
*Consumer will find the earliest offset of the specified timestamp from Kafka, if not, consumer will use the latest offset to read data from Kafka.
*
*When consumer is restored from checkpoint or savepoint, this method does not work. In this case, the offset in the restored state will be used.
*
*@ param startupOffsetsTimestamp the timestamp of the offset starting from the era.
*
*@ return the consumer object that allows chain method calls
*/
/**
* Specifies the consumer to start reading partitions from a specified timestamp.
* The specified timestamp must be before the current timestamp.
* This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers.
*
* <p>The consumer will look up the earliest offset whose timestamp is greater than or equal
* to the specific timestamp from Kafka. If there's no such offset, the consumer will use the
* latest offset to read data from kafka.
*
* <p>This method does not affect where partitions are read from when the consumer is restored
* from a checkpoint or savepoint. When the consumer is restored from a checkpoint or
* savepoint, only the offsets in the restored state will be used.
*
* @param startupOffsetsTimestamp timestamp for the startup offsets, as milliseconds from epoch.
*
* @return The consumer object, to allow function chaining.
*/
// NOTE -
// This method is implemented in the base class because this is where the startup logging and verifications live.
// However, it is not publicly exposed since only newer Kafka versions support the functionality.
// Version-specific subclasses which can expose the functionality should override and allow public access.
protected FlinkKafkaConsumerBase<T> setStartFromTimestamp(long startupOffsetsTimestamp) {
checkArgument(startupOffsetsTimestamp >= 0, "The provided value for the startup offsets timestamp is invalid.");
long currentTimestamp = System.currentTimeMillis();
checkArgument(startupOffsetsTimestamp <= currentTimestamp,
"Startup time[%s] must be before current time[%s].", startupOffsetsTimestamp, currentTimestamp);
this.startupMode = StartupMode.TIMESTAMP;
this.startupOffsetsTimestamp = startupOffsetsTimestamp;
this.specificStartupOffsets = null;
return this;
}
- FlinkKafkaConsumer011.setStartFromGroupOffsets()
/**
*Specifies that consumer reads from committed offsets in Zookeeper / Kafka brokers. 'group.id' must be configured in the configuration property. If the offset of a partition is not found, the partition will use the
*"auto.offset.reset" of "
*
*When consumer is restored from checkpoint or savepoint, this method does not work. In this case, the offset in the restored state will be used.
*
*@ return the consumer object that allows chain method calls
*/
/**
* Specifies the consumer to start reading from any committed group offsets found
* in Zookeeper / Kafka brokers. The "group.id" property must be set in the configuration
* properties. If no offset can be found for a partition, the behaviour in "auto.offset.reset"
* set in the configuration properties will be used for the partition.
*
* <p>This method does not affect where partitions are read from when the consumer is restored
* from a checkpoint or savepoint. When the consumer is restored from a checkpoint or
* savepoint, only the offsets in the restored state will be used.
*
* @return The consumer object, to allow function chaining.
*/
public FlinkKafkaConsumerBase<T> setStartFromGroupOffsets() {
this.startupMode = StartupMode.GROUP_OFFSETS;
this.startupOffsetsTimestamp = null;
this.specificStartupOffsets = null;
return this;
}
- FlinkKafkaConsumer011.setStartFromSpecificOffsets()
/**
*Specifies that consumer reads the partition from the specified offset. It needs to be set up separately for each partition.
*The specified offset should be the offset of the next record to be read from the partition.
*This method causes consumer to ignore any submitted offset s in Zookeeper / Kafka brokers.
*
*If the offset map provided contains an entry that consumer does not subscribe to, the entry will be ignored. If the consumer subscribe s to a partition that does not exist in the provided offset map, the consumer will
*Use the default offset for this partition (see {@ linkflinkkafka consumerbase ා set start from groupoffset()}).
*
*If the offset specified by a partition is invalid, and the default offset of the partition cannot be found, the partition will use "auto.offset.reset" in the configuration property.
*
*When consumer is restored from checkpoint or savepoint, this method does not work. In this case, the offset in the restored state will be used.
*
*@ return the consumer object that allows chain method calls
*/
/**
* Specifies the consumer to start reading partitions from specific offsets, set independently for each partition.
* The specified offset should be the offset of the next record that will be read from partitions.
* This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers.
*
* <p>If the provided map of offsets contains entries whose {@link KafkaTopicPartition} is not subscribed by the
* consumer, the entry will be ignored. If the consumer subscribes to a partition that does not exist in the provided
* map of offsets, the consumer will fallback to the default group offset behaviour (see
* {@link FlinkKafkaConsumerBase#setStartFromGroupOffsets()}) for that particular partition.
*
* <p>If the specified offset for a partition is invalid, or the behaviour for that partition is defaulted to group
* offsets but still no group offset could be found for it, then the "auto.offset.reset" behaviour set in the
* configuration properties will be used for the partition
*
* <p>This method does not affect where partitions are read from when the consumer is restored
* from a checkpoint or savepoint. When the consumer is restored from a checkpoint or
* savepoint, only the offsets in the restored state will be used.
*
* @return The consumer object, to allow function chaining.
*/
public FlinkKafkaConsumerBase<T> setStartFromSpecificOffsets(Map<KafkaTopicPartition, Long> specificStartupOffsets) {
this.startupMode = StartupMode.SPECIFIC_OFFSETS;
this.startupOffsetsTimestamp = null;
this.specificStartupOffsets = checkNotNull(specificStartupOffsets);
return this;
}