org.elasticsearch.spark.sql.streaming

Class EsStreamQueryWriter

java.lang.Object

<any>

org.elasticsearch.spark.sql.streaming.EsStreamQueryWriter

public class EsStreamQueryWriter
extends <any>

Takes in iterator of InternalRow objects from a partition of data, writes it to Elasticsearch, and manages the streaming commit protocol.

Constructor Summary

Constructors

Constructor and Description
EsStreamQueryWriter(java.lang.String serializedSettings, org.apache.spark.sql.types.StructType schema, EsCommitProtocol commitProtocol)

Method Summary

Modifier and Type	Method and Description
protected java.lang.Class<? extends org.elasticsearch.hadoop.serialization.BytesConverter>	bytesConverter()
protected java.lang.Class<? extends org.elasticsearch.hadoop.serialization.field.FieldExtractor>	fieldExtractor()
protected java.lang.Object	processData(scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)
TaskCommit	run(org.apache.spark.TaskContext taskContext, scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)
protected java.lang.Class<? extends org.elasticsearch.hadoop.serialization.builder.ValueWriter<?>>	valueWriter()
void	write(org.apache.spark.TaskContext taskContext, scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

EsStreamQueryWriter

public EsStreamQueryWriter(java.lang.String serializedSettings,
                           org.apache.spark.sql.types.StructType schema,
                           EsCommitProtocol commitProtocol)

Method Detail

bytesConverter

protected java.lang.Class<? extends org.elasticsearch.hadoop.serialization.BytesConverter> bytesConverter()

fieldExtractor

protected java.lang.Class<? extends org.elasticsearch.hadoop.serialization.field.FieldExtractor> fieldExtractor()

processData

protected java.lang.Object processData(scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)

run

public TaskCommit run(org.apache.spark.TaskContext taskContext,
                      scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)

valueWriter

protected java.lang.Class<? extends org.elasticsearch.hadoop.serialization.builder.ValueWriter<?>> valueWriter()

write

public void write(org.apache.spark.TaskContext taskContext,
                  scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)