The Avro Builder provides the following functionality
Avro Builder | |
---|---|
Factory Name | com.searchtechnologies.aspire:app-publish-to-avro |
subType | |
Inputs | AspireObject from a connector's job with metadata and content extracted from a specific file |
Outputs | Avro representation of AspireObject sent to an Avro file. |
Element | Type | Default | Description |
---|---|---|---|
avroSchema | string | Avro schema path | |
outputNonBatchFile | string | Output file path | |
nonBatchFileMaxSize | int | 600 | Max file size (MB) |
outputBatchDirName | string | Output directory path for internal batch files | |
maxInternalBatchSize | int | 1000 | Batch size |
timeRolloverThreshold | long | 86400 | Interval after which rollover should happen in sec |
rolloverFileLocation | string | The directory where Avro files should be moved | |
debug | boolean | false | If true it will log debug information from the component |
Code Block |
---|
<application config="com.searchtechnologies.aspire:app-publish-to-avro" name="/Aspire_Publish_To_Avro_Application"> |
<properties>
<property <properties> <property name="useSizeLimited" |
>
false
<>false</property> |
<property <property name="debug" |
>
false
<>false</property> |
<property <property name="maxInternalBatchSize" |
>
2
<>5</property> |
<property <property name="outputBatchDirName" |
>c:\temp\avro</property>
<property >dirName</property> <property name="useInternalBatch" |
>
true
<>true</property> |
<property <property name="avroSchema">c:\ |
tmp\ |
<property schema.avsc</property> <property name="timeRolloverThreshold">5</property> <property name="rolloverFileLocation></property> <property name="OutputType">internalBatches</property> |
</properties> |
</application> |
Code Block |
---|
<application config="com.searchtechnologies.aspire:app-publish-to-avro" name="/Aspire_Publish_To_Avro_Application"> |
<properties>
<property <properties> <property name="useSizeLimited" |
>
true
<>true</property> |
<property <property name="debug" |
>
false
<>false</property> |
<property <property name="nonBatchFileMaxSize" |
>
50
<>50</property> |
<property <property name="outputNonBatchFile">c:\ |
avro\avro.snappy</property> |
<property <property name="useInternalBatch" |
>
false
<>false</property> |
<property <property name="avroSchema" |
>c>\ |
tmp\ |
<property schema.avsc</property> <property name="timeRolloverThreshold">5</property> <property name="rolloverFileLocation></property> <property name="OutputType">sizeLimited</property> |
</properties> |
</application> |