Skip to main content
Version: 1.7.0

Pulsar2Hive Example

Prepare to get module archive

Module archive is in the directory:inlong-sort-standalone/sort-standalone-dist/target/, the archive file is apache-inlong-sort-standalone-${project.version}-bin.tar.gz.

Prepare to modify configuration file

At first, decompress the archive file, copy three files in the directory "conf/hive/" to the directory "conf/".

  • conf/common.properties, common configuration of all components.
  • conf/SortClusterConfig.conf, sink configuration of all sort tasks.
  • conf/sid_hive_inlong6th_v3.conf, data source configuration example of a sort task, the file name is same with sort task name in SortClusterConfig.conf.

Example: conf/common.properties

clusterId=hivev3-sz-sz1
nodeId=nodeId
metricDomains=Sort
metricDomains.Sort.domainListeners=org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener
metricDomains.Sort.snapshotInterval=60000
sortChannel.type=org.apache.inlong.sort.standalone.channel.BufferQueueChannel
sortSink.type=org.apache.inlong.sort.standalone.sink.hive.HiveSink
sortSource.type=org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource

sortClusterConfig.type=file
sortClusterConfig.file=SortClusterConfig.conf
sortSourceConfig.QueryConsumeConfigType=file

# manager config example
#sortClusterConfig.type=manager
#sortSourceConfig.QueryConsumeConfigType=manager
#managerUrlLoaderType=org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader
#sortClusterConfig.managerUrl=http://${manager_ip:port}/api/inlong/manager/openapi/sort/getClusterConfig
#sortSourceConfig.managerUrl=http://${manager_ip:port}/api/inlong/manager/openapi/sort/getSortSource

Example: conf/SortClusterConfig.conf

{
"data":{
"clusterName":"hivev3-sz-sz1",
"sortTasks":[
{
"idParams":[
{
"inlongGroupId":"0fc00000046",
"inlongStreamId":"",
"separator":"|",
"partitionIntervalMs":3600000,
"idRootPath":"/user/hive/warehouse/t_inlong_v1_0fc00000046",
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
"hiveTableName":"t_inlong_v1_0fc00000046",
"partitionFieldName":"dt",
"partitionFieldPattern":"yyyyMMddHH",
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
"maxPartitionOpenDelayHour":8
},
{
"inlongGroupId":"03600000045",
"inlongStreamId":"",
"separator":"|",
"partitionIntervalMs":3600000,
"idRootPath":"/user/hive/warehouse/t_inlong_v1_03600000045",
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
"hiveTableName":"t_inlong_v1_03600000045",
"partitionFieldName":"dt",
"partitionFieldPattern":"yyyyMMddHH",
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
"maxPartitionOpenDelayHour":8
},
{
"inlongGroupId":"05100054990",
"inlongStreamId":"",
"separator":"|",
"partitionIntervalMs":3600000,
"idRootPath":"/user/hive/warehouse/t_inlong_v1_05100054990",
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
"hiveTableName":"t_inlong_v1_05100054990",
"partitionFieldName":"dt",
"partitionFieldPattern":"yyyyMMddHH",
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
"maxPartitionOpenDelayHour":8
},
{
"inlongGroupId":"09c00014434",
"inlongStreamId":"",
"separator":"|",
"partitionIntervalMs":3600000,
"idRootPath":"/user/hive/warehouse/t_inlong_v1_09c00014434",
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
"hiveTableName":"t_inlong_v1_09c00014434",
"partitionFieldName":"dt",
"partitionFieldPattern":"yyyyMMddHH",
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
"maxPartitionOpenDelayHour":8
},
{
"inlongGroupId":"0c900035509",
"inlongStreamId":"",
"separator":"|",
"partitionIntervalMs":3600000,
"idRootPath":"/user/hive/warehouse/t_inlong_v1_0c900035509",
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
"hiveTableName":"t_inlong_v1_0c900035509",
"partitionFieldName":"dt",
"partitionFieldPattern":"yyyyMMddHH",
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
"maxPartitionOpenDelayHour":8
}
],
"name":"sid_hive_inlong6th_v3",
"sinkParams":{
"hdfsPath":"hdfs://127.0.0.1:9000",
"maxFileOpenDelayMinute":"5",
"tokenOvertimeMinute":"60",
"maxOutputFileSizeGb":"2",
"hiveJdbcUrl":"jdbc:hive2://127.0.0.2:10000",
"hiveDatabase":"default",
"hiveUsername":"hive",
"hivePassword":"hive"
},
"type":"HIVE"
}
]
},
"errCode":0,
"md5":"md5",
"result":true
}

Example: conf/sid_hive_inlong6th_v3.conf

{
"sortClusterName": "hivev3-sz-sz1",
"sortTaskId": "sid_hive_inlong6th_v3",
"cacheZones": {
"pc_inlong6th_sz1": {
"zoneName": "${PULSAR_CLUSTER_NAME}",
"serviceUrl": "http://${PULSAR_IP}:${PULSAR_PORT}",
"authentication": "${PULSAR_AUTH}",
"topics": [{
"topic": "${TENANT/NAMESPACE/TOPIC}",
"partitionCnt": 10,
"topicProperties": {}
}],
"cacheZoneProperties": {},
"zoneType": "pulsar"
}
}
}