forked from docs/doc-exports
Su, Xiaomeng
04d4597cf3
Reviewed-by: Pruthi, Vineet <vineet.pruthi@t-systems.com> Co-authored-by: Su, Xiaomeng <suxiaomeng1@huawei.com> Co-committed-by: Su, Xiaomeng <suxiaomeng1@huawei.com>
187 lines
22 KiB
HTML
187 lines
22 KiB
HTML
<a name="dli_08_0107"></a><a name="dli_08_0107"></a>
|
|
|
|
<h1 class="topictitle1">Configuring Time Models</h1>
|
|
<div id="body1574406512570"><p id="dli_08_0107__p14215542394">Flink provides two time models: processing time and event time.</p>
|
|
<p id="dli_08_0107__p142155443913">DLI allows you to specify the time model during creation of the source stream and temporary stream.</p>
|
|
<div class="section" id="dli_08_0107__section1945370714"><h4 class="sectiontitle">Configuring Processing Time</h4><p id="dli_08_0107__p166311278112">Processing time refers to the system time, which is irrelevant to the data timestamp.</p>
|
|
<p id="dli_08_0107__p87912280324"><strong id="dli_08_0107__b17897117133111">Syntax</strong></p>
|
|
<div class="codecoloring" codetype="Sql" id="dli_08_0107__screen173441343134018"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal">1</span>
|
|
<span class="normal">2</span>
|
|
<span class="normal">3</span>
|
|
<span class="normal">4</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">SOURCE</span><span class="w"> </span><span class="n">STREAM</span><span class="w"> </span><span class="n">stream_name</span><span class="p">(...)</span><span class="w"> </span><span class="k">WITH</span><span class="w"> </span><span class="p">(...)</span>
|
|
<span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">proctime</span><span class="p">.</span><span class="n">proctime</span><span class="p">;</span>
|
|
<span class="k">CREATE</span><span class="w"> </span><span class="n">TEMP</span><span class="w"> </span><span class="n">STREAM</span><span class="w"> </span><span class="n">stream_name</span><span class="p">(...)</span>
|
|
<span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">proctime</span><span class="p">.</span><span class="n">proctime</span><span class="p">;</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
<p id="dli_08_0107__p115412815328"><strong id="dli_08_0107__b57783216312">Description</strong></p>
|
|
<p id="dli_08_0107__a8f7319db59ba400fa474251351564e30">To set the processing time, you only need to add proctime.proctime following TIMESTAMP BY. You can directly use the proctime field later.</p>
|
|
<p id="dli_08_0107__p729152893210"><strong id="dli_08_0107__b19560827163114">Precautions</strong></p>
|
|
<p id="dli_08_0107__p16890249120">None</p>
|
|
<p id="dli_08_0107__p10518285322"><strong id="dli_08_0107__b359532233212">Example</strong></p>
|
|
<div class="codecoloring" codetype="Sql" id="dli_08_0107__sa600481ddf364d9db2627755b36a1cac"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal"> 1</span>
|
|
<span class="normal"> 2</span>
|
|
<span class="normal"> 3</span>
|
|
<span class="normal"> 4</span>
|
|
<span class="normal"> 5</span>
|
|
<span class="normal"> 6</span>
|
|
<span class="normal"> 7</span>
|
|
<span class="normal"> 8</span>
|
|
<span class="normal"> 9</span>
|
|
<span class="normal">10</span>
|
|
<span class="normal">11</span>
|
|
<span class="normal">12</span>
|
|
<span class="normal">13</span>
|
|
<span class="normal">14</span>
|
|
<span class="normal">15</span>
|
|
<span class="normal">16</span>
|
|
<span class="normal">17</span>
|
|
<span class="normal">18</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">SOURCE</span><span class="w"> </span><span class="n">STREAM</span><span class="w"> </span><span class="n">student_scores</span><span class="w"> </span><span class="p">(</span>
|
|
<span class="w"> </span><span class="n">student_number</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Student ID */</span>
|
|
<span class="w"> </span><span class="n">student_name</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Name */</span>
|
|
<span class="w"> </span><span class="n">subject</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Subject */</span>
|
|
<span class="w"> </span><span class="n">score</span><span class="w"> </span><span class="nb">INT</span><span class="w"> </span><span class="cm">/* Score */</span>
|
|
<span class="p">)</span>
|
|
<span class="k">WITH</span><span class="w"> </span><span class="p">(</span>
|
|
<span class="w"> </span><span class="k">type</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"dis"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">region</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">""</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">channel</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"dliinput"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">partition_count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"1"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">encode</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"csv"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">field_delimiter</span><span class="o">=</span><span class="ss">","</span>
|
|
<span class="p">)</span><span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">proctime</span><span class="p">.</span><span class="n">proctime</span><span class="p">;</span>
|
|
|
|
<span class="k">INSERT</span><span class="w"> </span><span class="k">INTO</span><span class="w"> </span><span class="n">score_greate_90</span>
|
|
<span class="k">SELECT</span><span class="w"> </span><span class="n">student_name</span><span class="p">,</span><span class="w"> </span><span class="k">sum</span><span class="p">(</span><span class="n">score</span><span class="p">)</span><span class="w"> </span><span class="n">over</span><span class="w"> </span><span class="p">(</span><span class="k">order</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="n">proctime</span><span class="w"> </span><span class="n">RANGE</span><span class="w"> </span><span class="n">UNBOUNDED</span><span class="w"> </span><span class="n">PRECEDING</span><span class="p">)</span><span class="w"> </span>
|
|
<span class="k">FROM</span><span class="w"> </span><span class="n">student_scores</span><span class="p">;</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="section" id="dli_08_0107__section1084044811311"><h4 class="sectiontitle">Configuring Event Time</h4><p id="dli_08_0107__p41051851631">Event Time refers to the time when an event is generated, that is, the timestamp generated during data generation.</p>
|
|
<p id="dli_08_0107__p5616159945"><strong id="dli_08_0107__b1427520368326">Syntax</strong></p>
|
|
<div class="codecoloring" codetype="Sql" id="dli_08_0107__screen764019168411"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal">1</span>
|
|
<span class="normal">2</span>
|
|
<span class="normal">3</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">SOURCE</span><span class="w"> </span><span class="n">STREAM</span><span class="w"> </span><span class="n">stream_name</span><span class="p">(...)</span><span class="w"> </span><span class="k">WITH</span><span class="w"> </span><span class="p">(...)</span>
|
|
<span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="err">{</span><span class="n">attr_name</span><span class="err">}</span><span class="p">.</span><span class="n">rowtime</span>
|
|
<span class="k">SET</span><span class="w"> </span><span class="n">WATERMARK</span><span class="w"> </span><span class="p">(</span><span class="n">RANGE</span><span class="w"> </span><span class="err">{</span><span class="n">time_interval</span><span class="err">}</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="k">ROWS</span><span class="w"> </span><span class="err">{</span><span class="n">literal</span><span class="err">}</span><span class="p">,</span><span class="w"> </span><span class="err">{</span><span class="n">time_interval</span><span class="err">}</span><span class="p">);</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
<p id="dli_08_0107__p11867433649"><strong id="dli_08_0107__b128283819323">Description</strong></p>
|
|
<p id="dli_08_0107__p4506203043211">To set the event time, you need to select a certain attribute in the stream as the timestamp and set the watermark policy.</p>
|
|
<p id="dli_08_0107__p183231427101716">Out-of-order events or late events may occur due to network faults. The watermark must be configured to trigger the window for calculation after waiting for a certain period of time. Watermarks are mainly used to process out-of-order data before generated events are sent to DLI during stream processing.</p>
|
|
<p id="dli_08_0107__p7656114518310">The following two watermark policies are available:</p>
|
|
<ul id="dli_08_0107__ul2648125614314"><li id="dli_08_0107__li964875683111">By time interval<div class="codecoloring" codetype="Sql" id="dli_08_0107__screen8359153416414"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">SET</span><span class="w"> </span><span class="n">WATERMARK</span><span class="p">(</span><span class="n">range</span><span class="w"> </span><span class="nb">interval</span><span class="w"> </span><span class="err">{</span><span class="n">time_unit</span><span class="err">}</span><span class="p">,</span><span class="w"> </span><span class="nb">interval</span><span class="w"> </span><span class="err">{</span><span class="n">time_unit</span><span class="err">}</span><span class="p">)</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
</li><li id="dli_08_0107__li21071317163216">By event quantity<div class="codecoloring" codetype="Sql" id="dli_08_0107__screen15125174734118"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">SET</span><span class="w"> </span><span class="n">WATERMARK</span><span class="p">(</span><span class="k">rows</span><span class="w"> </span><span class="n">literal</span><span class="p">,</span><span class="w"> </span><span class="nb">interval</span><span class="w"> </span><span class="err">{</span><span class="n">time_unit</span><span class="err">}</span><span class="p">)</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
</li></ul>
|
|
<div class="note" id="dli_08_0107__note6487036133915"><img src="public_sys-resources/note_3.0-en-us.png"><span class="notetitle"> </span><div class="notebody"><p id="dli_08_0107__p262874711399">Parameters are separated by commas (,). The first parameter indicates the watermark sending interval and the second indicates the maximum event delay.</p>
|
|
</div></div>
|
|
<p id="dli_08_0107__p250815017513"><strong id="dli_08_0107__b1864445819323">Precautions</strong></p>
|
|
<p id="dli_08_0107__p161541814151">None</p>
|
|
<p id="dli_08_0107__p186031520953"><strong id="dli_08_0107__b382413010338">Example</strong></p>
|
|
<ul id="dli_08_0107__ul1319252104111"><li id="dli_08_0107__li181965224118">Send a watermark every 10s the <strong id="dli_08_0107__b109832612223">time2</strong> event is generated. The maximum event latency is 20s.<div class="codecoloring" codetype="Sql" id="dli_08_0107__screen11260142344210"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal"> 1</span>
|
|
<span class="normal"> 2</span>
|
|
<span class="normal"> 3</span>
|
|
<span class="normal"> 4</span>
|
|
<span class="normal"> 5</span>
|
|
<span class="normal"> 6</span>
|
|
<span class="normal"> 7</span>
|
|
<span class="normal"> 8</span>
|
|
<span class="normal"> 9</span>
|
|
<span class="normal">10</span>
|
|
<span class="normal">11</span>
|
|
<span class="normal">12</span>
|
|
<span class="normal">13</span>
|
|
<span class="normal">14</span>
|
|
<span class="normal">15</span>
|
|
<span class="normal">16</span>
|
|
<span class="normal">17</span>
|
|
<span class="normal">18</span>
|
|
<span class="normal">19</span>
|
|
<span class="normal">20</span>
|
|
<span class="normal">21</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">SOURCE</span><span class="w"> </span><span class="n">STREAM</span><span class="w"> </span><span class="n">student_scores</span><span class="w"> </span><span class="p">(</span>
|
|
<span class="w"> </span><span class="n">student_number</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Student ID */</span>
|
|
<span class="w"> </span><span class="n">student_name</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Name */</span>
|
|
<span class="w"> </span><span class="n">subject</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Subject */</span>
|
|
<span class="w"> </span><span class="n">score</span><span class="w"> </span><span class="nb">INT</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Score */</span>
|
|
<span class="w"> </span><span class="n">time2</span><span class="w"> </span><span class="k">TIMESTAMP</span>
|
|
<span class="p">)</span>
|
|
<span class="k">WITH</span><span class="w"> </span><span class="p">(</span>
|
|
<span class="w"> </span><span class="k">type</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"dis"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">region</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">""</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">channel</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"dliinput"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">partition_count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"1"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">encode</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"csv"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">field_delimiter</span><span class="o">=</span><span class="ss">","</span>
|
|
<span class="p">)</span>
|
|
<span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">time2</span><span class="p">.</span><span class="n">rowtime</span>
|
|
<span class="k">SET</span><span class="w"> </span><span class="n">WATERMARK</span><span class="w"> </span><span class="p">(</span><span class="n">RANGE</span><span class="w"> </span><span class="nb">interval</span><span class="w"> </span><span class="mi">10</span><span class="w"> </span><span class="k">second</span><span class="p">,</span><span class="w"> </span><span class="nb">interval</span><span class="w"> </span><span class="mi">20</span><span class="w"> </span><span class="k">second</span><span class="p">);</span>
|
|
|
|
<span class="k">INSERT</span><span class="w"> </span><span class="k">INTO</span><span class="w"> </span><span class="n">score_greate_90</span>
|
|
<span class="k">SELECT</span><span class="w"> </span><span class="n">student_name</span><span class="p">,</span><span class="w"> </span><span class="k">sum</span><span class="p">(</span><span class="n">score</span><span class="p">)</span><span class="w"> </span><span class="n">over</span><span class="w"> </span><span class="p">(</span><span class="k">order</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="n">time2</span><span class="w"> </span><span class="n">RANGE</span><span class="w"> </span><span class="n">UNBOUNDED</span><span class="w"> </span><span class="n">PRECEDING</span><span class="p">)</span><span class="w"> </span>
|
|
<span class="k">FROM</span><span class="w"> </span><span class="n">student_scores</span><span class="p">;</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
</li><li id="dli_08_0107__li4973041154215">Send the watermark every time when 10 pieces of data are received, and the maximum event latency is 20s.<div class="codecoloring" codetype="Sql" id="dli_08_0107__screen1845619496439"><div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal"> 1</span>
|
|
<span class="normal"> 2</span>
|
|
<span class="normal"> 3</span>
|
|
<span class="normal"> 4</span>
|
|
<span class="normal"> 5</span>
|
|
<span class="normal"> 6</span>
|
|
<span class="normal"> 7</span>
|
|
<span class="normal"> 8</span>
|
|
<span class="normal"> 9</span>
|
|
<span class="normal">10</span>
|
|
<span class="normal">11</span>
|
|
<span class="normal">12</span>
|
|
<span class="normal">13</span>
|
|
<span class="normal">14</span>
|
|
<span class="normal">15</span>
|
|
<span class="normal">16</span>
|
|
<span class="normal">17</span>
|
|
<span class="normal">18</span>
|
|
<span class="normal">19</span>
|
|
<span class="normal">20</span>
|
|
<span class="normal">21</span></pre></div></td><td class="code"><div><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">SOURCE</span><span class="w"> </span><span class="n">STREAM</span><span class="w"> </span><span class="n">student_scores</span><span class="w"> </span><span class="p">(</span>
|
|
<span class="w"> </span><span class="n">student_number</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Student ID */</span>
|
|
<span class="w"> </span><span class="n">student_name</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Name */</span>
|
|
<span class="w"> </span><span class="n">subject</span><span class="w"> </span><span class="n">STRING</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Subject */</span>
|
|
<span class="w"> </span><span class="n">score</span><span class="w"> </span><span class="nb">INT</span><span class="p">,</span><span class="w"> </span><span class="cm">/* Score */</span>
|
|
<span class="w"> </span><span class="n">time2</span><span class="w"> </span><span class="k">TIMESTAMP</span>
|
|
<span class="p">)</span>
|
|
<span class="k">WITH</span><span class="w"> </span><span class="p">(</span>
|
|
<span class="w"> </span><span class="k">type</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"dis"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">region</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">""</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">channel</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"dliinput"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">partition_count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"1"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">encode</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="ss">"csv"</span><span class="p">,</span>
|
|
<span class="w"> </span><span class="n">field_delimiter</span><span class="o">=</span><span class="ss">","</span>
|
|
<span class="p">)</span>
|
|
<span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">BY</span><span class="w"> </span><span class="n">time2</span><span class="p">.</span><span class="n">rowtime</span>
|
|
<span class="k">SET</span><span class="w"> </span><span class="n">WATERMARK</span><span class="w"> </span><span class="p">(</span><span class="k">ROWS</span><span class="w"> </span><span class="mi">10</span><span class="p">,</span><span class="w"> </span><span class="nb">interval</span><span class="w"> </span><span class="mi">20</span><span class="w"> </span><span class="k">second</span><span class="p">);</span>
|
|
|
|
<span class="k">INSERT</span><span class="w"> </span><span class="k">INTO</span><span class="w"> </span><span class="n">score_greate_90</span>
|
|
<span class="k">SELECT</span><span class="w"> </span><span class="n">student_name</span><span class="p">,</span><span class="w"> </span><span class="k">sum</span><span class="p">(</span><span class="n">score</span><span class="p">)</span><span class="w"> </span><span class="n">over</span><span class="w"> </span><span class="p">(</span><span class="k">order</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="n">time2</span><span class="w"> </span><span class="n">RANGE</span><span class="w"> </span><span class="n">UNBOUNDED</span><span class="w"> </span><span class="n">PRECEDING</span><span class="p">)</span><span class="w"> </span>
|
|
<span class="k">FROM</span><span class="w"> </span><span class="n">student_scores</span><span class="p">;</span>
|
|
</pre></div></td></tr></table></div>
|
|
|
|
</div>
|
|
</li></ul>
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<div class="familylinks">
|
|
<div class="parentlink"><strong>Parent topic:</strong> <a href="dli_08_0233.html">Flink SQL Syntax</a></div>
|
|
</div>
|
|
</div>
|
|
|