forked from docs/doc-exports
Su, Xiaomeng
04d4597cf3
Reviewed-by: Pruthi, Vineet <vineet.pruthi@t-systems.com> Co-authored-by: Su, Xiaomeng <suxiaomeng1@huawei.com> Co-committed-by: Su, Xiaomeng <suxiaomeng1@huawei.com>
143 lines
16 KiB
HTML
143 lines
16 KiB
HTML
<a name="dli_08_0309"></a><a name="dli_08_0309"></a>
|
|
|
|
<h1 class="topictitle1">Upsert Kafka Result Table</h1>
|
|
<div id="body8662426"><div class="section" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_section77915506517"><h4 class="sectiontitle">Function</h4><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p661816320442">DLI exports the output data of the Flink job to Kafka in upsert mode.</p>
|
|
<p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p2618163211448">Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provides data replicas and fault tolerance. Apache Kafka is applicable to scenarios of handling massive messages.</p>
|
|
</div>
|
|
<div class="section" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_dli_08_0256_en-us_topic_0132788972_section2579142713429"><h4 class="sectiontitle">Prerequisites</h4><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_dli_08_0237_p1038754112542">Kafka is an offline cluster. You have built an enhanced datasource connection to connect Flink jobs to Kafka. You have set security group rules as required.</p>
|
|
</div>
|
|
<div class="section" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_section13191959213"><h4 class="sectiontitle">Precautions</h4><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p247314912217">SASL_SSL cannot be enabled for the interconnected Kafka cluster.</p>
|
|
</div>
|
|
<div class="section" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_section892013201167"><h4 class="sectiontitle">Syntax</h4><div class="codecoloring" codetype="Sql" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_dli_08_0256_screen1461215294716"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal"> 1</span>
|
|
<span class="normal"> 2</span>
|
|
<span class="normal"> 3</span>
|
|
<span class="normal"> 4</span>
|
|
<span class="normal"> 5</span>
|
|
<span class="normal"> 6</span>
|
|
<span class="normal"> 7</span>
|
|
<span class="normal"> 8</span>
|
|
<span class="normal"> 9</span>
|
|
<span class="normal">10</span>
|
|
<span class="normal">11</span>
|
|
<span class="normal">12</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">create</span><span class="w"> </span><span class="k">table</span><span class="w"> </span><span class="n">kafkaSource</span><span class="p">(</span>
|
|
<span class="w"> </span><span class="n">attr_name</span><span class="w"> </span><span class="n">attr_type</span><span class="w"> </span>
|
|
<span class="w"> </span><span class="p">(</span><span class="s1">','</span><span class="w"> </span><span class="n">attr_name</span><span class="w"> </span><span class="n">attr_type</span><span class="p">)</span><span class="o">*</span><span class="w"> </span>
|
|
<span class="w"> </span><span class="p">(</span><span class="s1">','</span><span class="k">PRIMARY</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="n">attr_name</span><span class="p">,</span><span class="w"> </span><span class="p">...)</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="n">ENFORCED</span><span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="k">with</span><span class="w"> </span><span class="p">(</span>
|
|
<span class="w"> </span><span class="s1">'connector.type'</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">'upsert-kafka'</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="s1">'connector.version'</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">''</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="s1">'connector.topic'</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">''</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="s1">'connector.properties.bootstrap.servers'</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">''</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="s1">'format.type'</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">''</span>
|
|
<span class="p">);</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="section" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_section893519312815"><h4 class="sectiontitle">Parameters</h4>
|
|
<div class="tablenoborder"><table cellpadding="4" cellspacing="0" summary="" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_table82421231587" frame="border" border="1" rules="all"><caption><b>Table 1 </b>Parameter description</caption><thead align="left"><tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row13242831486"><th align="left" class="cellrowborder" valign="top" width="16.491649164916492%" id="mcps1.3.5.2.2.4.1.1"><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p524213118810">Parameter</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="7.81078107810781%" id="mcps1.3.5.2.2.4.1.2"><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p1124218311813">Mandatory</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="75.6975697569757%" id="mcps1.3.5.2.2.4.1.3"><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p15242163119815">Description</p>
|
|
</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody><tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row15242113115811"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p02421431888">connector.type</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p112425311817">Yes</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p11242431287">Connector type. Set this parameter to <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b1367516597526">upsert-kafka</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row524211316813"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p112421313810">connector.version</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p1824263112812">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p624283111815">Kafka version. The value can only be <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b1679272214536">0.11</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row424216311788"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p8242731087">format.type</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p12426311387">Yes</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p18242203112815">Data serialization format. The value can be <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b065352725310">csv</strong>, <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b14654192711535">json</strong>, or <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b06541827135313">avro</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row82421531587"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p2024317318813">connector.topic</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p12431831982">Yes</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p12432311483">Kafka topic name</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row724317317820"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p924303118810">connector.properties.bootstrap.servers</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p32435314815">Yes</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p024319311282">Kafka broker addresses. Use commas (,) to separated them.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row17243203117811"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p112431031983">connector.sink-partitioner</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p124317311385">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p3243173117812">Partitioner type. The value can be <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b18640746185318">fixed</strong>, <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b16461846135318">round-robin</strong>, or <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b16647104625317">custom</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row424319311581"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p4243731185">connector.sink-partitioner-class</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p42431315820">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p724316311987">Custom partitioner. This parameter is mandatory when <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b716718914914">sink-partitioner</strong> is <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b6167692917">custom</strong>, for example, <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b6167791699">org.mycompany.MyPartitioner</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row182431231682"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p1524319314816">connector.sink.ignore-retraction</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p924317311283">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p1324333114810">Whether to ignore the retraction message. The default value is <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b1337641513114">false</strong>, indicating that the retraction message is written to Kafka as <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b93393532111">null</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row1124315311883"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p22431931481">update-mode</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p18243123114819">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p624319311982">Data update mode. Three write modes are supported: <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b189116331210">append</strong>, <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b69119312127">retract</strong>, and <strong id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_b7921737124">upsert</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_row1624317311884"><td class="cellrowborder" valign="top" width="16.491649164916492%" headers="mcps1.3.5.2.2.4.1.1 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p124303110813">connector.properties.*</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="7.81078107810781%" headers="mcps1.3.5.2.2.4.1.2 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p924317312813">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="75.6975697569757%" headers="mcps1.3.5.2.2.4.1.3 "><p id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_p1243143120814">Native properties of Kafka</p>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_section12707752101211"><h4 class="sectiontitle">Example</h4><pre class="screen" id="dli_08_0309__en-us_topic_0000001165951919_en-us_topic_0000001131854005_screen118035569129">create table upsertKafkaSink(
|
|
car_id STRING,
|
|
car_owner STRING,
|
|
car_brand STRING,
|
|
car_speed INT,
|
|
primary key (car_id) not enforced
|
|
)
|
|
with (
|
|
'connector.type' = 'upsert-kafka',
|
|
'connector.version' = '0.11',
|
|
'connector.topic' = 'test-topic',
|
|
'connector.properties.bootstrap.servers' = 'xx.xx.xx.xx:9092',
|
|
'format.type' = 'csv'
|
|
);</pre>
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<div class="familylinks">
|
|
<div class="parentlink"><strong>Parent topic:</strong> <a href="dli_08_0307.html">Creating a Result Table</a></div>
|
|
</div>
|
|
</div>
|
|
|