forked from docs/doc-exports
Xiong, Chen Xiao
14a6d65e8c
Reviewed-by: Pruthi, Vineet <vineet.pruthi@t-systems.com> Co-authored-by: Xiong, Chen Xiao <chenxiaoxiong@huawei.com> Co-committed-by: Xiong, Chen Xiao <chenxiaoxiong@huawei.com>
192 lines
17 KiB
HTML
192 lines
17 KiB
HTML
<a name="dataartsstudio_02_0266"></a><a name="dataartsstudio_02_0266"></a>
|
|
|
|
<h1 class="topictitle1">Link to HDFS</h1>
|
|
<div id="body8662426"><div class="section" id="dataartsstudio_02_0266__en-us_topic_0108272817_section621837"><h4 class="sectiontitle">Description</h4><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p1011207717515">By creating an HDFS link, you can extract files from or load files to MRS, FusionInsight HD, or Apache Hadoop. Files in CSV, Parquet, and binary formats are supported.</p>
|
|
</div>
|
|
<div class="section" id="dataartsstudio_02_0266__en-us_topic_0108272817_section6163607716523"><h4 class="sectiontitle">Sample Link</h4><pre class="screen" id="dataartsstudio_02_0266__en-us_topic_0108272817_screen421977021901">{
|
|
"links": [
|
|
{
|
|
"link-config-values": {
|
|
"configs": [
|
|
{
|
|
"inputs": [
|
|
{
|
|
"name": "linkConfig.hadoopType",
|
|
"value": "FusionInsight HD"
|
|
},
|
|
{
|
|
"name": "linkConfig.host",
|
|
"value": "10.120.205.143"
|
|
},
|
|
{
|
|
"name": "linkConfig.casPort",
|
|
"value": "20009"
|
|
},
|
|
{
|
|
"name": "linkConfig.port",
|
|
"value": "28443"
|
|
},
|
|
{
|
|
"name": "linkConfig.authType",
|
|
"value": "KERBEROS"
|
|
},
|
|
{
|
|
"name": "linkConfig.user",
|
|
"value": "<em id="dataartsstudio_02_0266__en-us_topic_0108272817_i18566989192658">admin</em>"
|
|
},
|
|
{
|
|
"name": "linkConfig.password",
|
|
"value": "<em id="dataartsstudio_02_0266__en-us_topic_0108272817_i24826227192652">Add password here</em>"
|
|
},
|
|
{
|
|
"name": "linkConfig.runMode",
|
|
"value": "STANDALONE"
|
|
}
|
|
],
|
|
"name": "linkConfig"
|
|
}
|
|
]
|
|
},
|
|
"name": "hdfslink",
|
|
"connector-name": "hdfs-connector"
|
|
}
|
|
]
|
|
}</pre>
|
|
</div>
|
|
<div class="section" id="dataartsstudio_02_0266__en-us_topic_0108272817_section5035508012043"><h4 class="sectiontitle">Link Parameters</h4>
|
|
<div class="tablenoborder"><table cellpadding="4" cellspacing="0" summary="" id="dataartsstudio_02_0266__en-us_topic_0108272817_table13922888141527" frame="border" border="1" rules="all"><thead align="left"><tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row229143141527"><th align="left" class="cellrowborder" valign="top" width="20.96%" id="mcps1.3.3.2.1.5.1.1"><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p66756185141527">Parameter</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="19.59%" id="mcps1.3.3.2.1.5.1.2"><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p38541938141527">Mandatory</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="18.93%" id="mcps1.3.3.2.1.5.1.3"><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p34889279141527">Type</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="40.52%" id="mcps1.3.3.2.1.5.1.4"><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p7459369141527">Description</p>
|
|
</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody><tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row2725489141730"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p31327482175124">linkConfig.hadoopType</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p19211512175124">Yes</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p19590909175124">Enumeration</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p40849601113150">Hadoop type. The options are as follows:</p>
|
|
<ul id="dataartsstudio_02_0266__en-us_topic_0108272817_ul49344411321"><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li2477177611321"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b842352706165340">MRS</strong>: link to HDFS of MRS</li><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li4555547819736"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b842352706165634">FusionInsight HD</strong>: link to HDFS of FusionInsight HD</li><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li5938342114118"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b842352706165653">Apache Hadoop</strong>: link to HDFS of Apache Hadoop</li></ul>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row302832141730"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p30068829175124">linkConfig.uri</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p379375175124">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p46443378175124">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p50220406175124">NameNode URI required for the link to Apache Hadoop. The format is <em id="dataartsstudio_02_0266__en-us_topic_0108272817_i842352697165918">ip:port</em>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row50442567191550"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p59316094191550">linkConfig.host</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p782000191843">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p63342026191843">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p25133216191550">IP address of Manager required for the link to MRS or FusionInsight HD</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row23965116817"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p51105455191612">linkConfig.port</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p56079990191848">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p46185322191848">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p3913811183515">Port number of Manager required for the link to FusionInsight HD</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row44965011191611"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p18287275191611">linkConfig.casPort</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p42854722191846">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p48680453191846">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p36596319191611">Port number of CAS Server that connects to FusionInsight HD required for the link to FusionInsight HD</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row12723067191613"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p23935469191613">linkConfig.user</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p44715260191850">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p65166273191850">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p15481183410599">Username for logging in to Manager. This parameter is not required if the cluster configuration is used.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row27957086191613"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p49931487191613">linkConfig.password</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p7575218191852">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p9612954191852">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p120393117135">Password for logging in to Manager. This parameter is not required if the cluster configuration is used.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row57133381612"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p10494739175124">linkConfig.authType</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p43871655175124">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p32281241175124">Enumeration</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><div class="p" id="dataartsstudio_02_0266__en-us_topic_0108272817_p31363547113238">Authentication method. The options are as follows:<ul id="dataartsstudio_02_0266__en-us_topic_0108272817_ul320611711334"><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li5952630511334"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b178538233168">Simple</strong>: for non-security mode</li><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li4561262711334"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b842352706183921_3">Kerberos</strong>: for security mode</li></ul>
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row5821133914163"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p176731282356">linkConfig.principal</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p198011521353">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p1298065218357">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p1450092103611">Account <strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b710353121715">principal</strong> required for Kerveros authentication. You can contact the administrator to obtain the account. Before using the cluster configuration, you must set this parameter in cluster configuration management.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row468115404169"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p12940816193514">linkConfig.keytab</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p2098025217352">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p8980145203519">FileContent</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p69402160358">Local absolute path of the <strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b178678045514">keytab</strong> file required for Kerveros authentication. You can also contact the administrator to obtain the file. Before using the cluster configuration, you must set this parameter in cluster configuration management.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row769418475561"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p7694174735610">linkConfig.runMode</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p8694164755614">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p3694174725618">Enumeration</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><div class="p" id="dataartsstudio_02_0266__en-us_topic_0108272817_p8372165910575">Running mode of the HDFS link. The options are as follows:<ul id="dataartsstudio_02_0266__en-us_topic_0108272817_ul1747017597574"><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li6848185317587"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b1296351794113">EMBEDDED</strong>: The link instance runs with CDM. This mode delivers better performance.</li><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li1261464125915"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b711818136424">STANDALONE</strong>: The link instance runs in an independent process. If CDM needs to connect to multiple Hadoop data sources (MRS, Hadoop, or CloudTable) with both Kerberos and Simple authentication methods, <strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b176468388434">STANDALONE</strong> is used.<p id="dataartsstudio_02_0266__en-us_topic_0108272817_p9697135219222">If <strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b175692711556">STANDALONE</strong> is selected, CDM can migrate data between HDFSs of multiple MRS clusters.</p>
|
|
</li><li id="dataartsstudio_02_0266__en-us_topic_0108272817_li2018234920212"><strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b1650862985514">Agent</strong>: The link instance runs on an agent.<p id="dataartsstudio_02_0266__en-us_topic_0108272817_p95031050162114">If <strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b35611033205518">Agent</strong> is not used, and the CDM cluster connects to two or more clusters with Kerberos authentication enabled and the same realm, only one cluster can be connected in <strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b1656673311557">EMBEDDED</strong> mode, and the other clusters must be in <strong id="dataartsstudio_02_0266__en-us_topic_0108272817_b165661233135511">STANDALONE</strong> mode.</p>
|
|
</li></ul>
|
|
</div>
|
|
<p id="dataartsstudio_02_0266__en-us_topic_0108272817_p5211105873913"></p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dataartsstudio_02_0266__en-us_topic_0108272817_row8246444122517"><td class="cellrowborder" valign="top" width="20.96%" headers="mcps1.3.3.2.1.5.1.1 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p843281310380">linkConfig.properties</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="19.59%" headers="mcps1.3.3.2.1.5.1.2 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p1196925110380">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="18.93%" headers="mcps1.3.3.2.1.5.1.3 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p2998524510380">Map</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="40.52%" headers="mcps1.3.3.2.1.5.1.4 "><p id="dataartsstudio_02_0266__en-us_topic_0108272817_p1288576310380">Properties. You can add configuration properties of the client. Each property must contain a name and a value.</p>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<div class="familylinks">
|
|
<div class="parentlink"><strong>Parent topic:</strong> <a href="link_parameter.html">Link Parameter Description</a></div>
|
|
</div>
|
|
</div>
|
|
|