diff --git a/docs/dli/umn/ALL_META.TXT.json b/docs/dli/umn/ALL_META.TXT.json new file mode 100644 index 00000000..70359215 --- /dev/null +++ b/docs/dli/umn/ALL_META.TXT.json @@ -0,0 +1,2422 @@ +[ + { + "uri":"dli_01_0538.html", + "product_code":"dli", + "code":"1", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Service Overview", + "title":"Service Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0378.html", + "product_code":"dli", + "code":"2", + "des":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing an", + "doc_type":"usermanual", + "kw":"DLI Introduction,Service Overview,User Guide", + "title":"DLI Introduction", + "githuburl":"" + }, + { + "uri":"dli_07_0007.html", + "product_code":"dli", + "code":"3", + "des":"You do not need a background in big data to use DLI for data analysis. You only need to know SQL, and you are good to go. The SQL syntax is fully compatible with the stan", + "doc_type":"usermanual", + "kw":"Advantages,Service Overview,User Guide", + "title":"Advantages", + "githuburl":"" + }, + { + "uri":"dli_07_0002.html", + "product_code":"dli", + "code":"4", + "des":"DLI is applicable to large-scale log analysis, federated analysis of heterogeneous data sources, and big data ETL processing.Gaming operation data analysisDifferent depar", + "doc_type":"usermanual", + "kw":"Application Scenarios,Service Overview,User Guide", + "title":"Application Scenarios", + "githuburl":"" + }, + { + "uri":"dli_07_0005.html", + "product_code":"dli", + "code":"5", + "des":"When using DLI, you may need to make adjustments according to the following restrictions:Recommended browsers for logging in to DLI:Google Chrome 43.0 or laterMozilla Fir", + "doc_type":"usermanual", + "kw":"Constraints and Limitations on Using DLI,Service Overview,User Guide", + "title":"Constraints and Limitations on Using DLI", + "githuburl":"" + }, + { + "uri":"dli_07_0006.html", + "product_code":"dli", + "code":"6", + "des":"If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IA", + "doc_type":"usermanual", + "kw":"Permissions Management,Service Overview,User Guide", + "title":"Permissions Management", + "githuburl":"" + }, + { + "uri":"dli_07_0003.html", + "product_code":"dli", + "code":"7", + "des":"DLI allows multiple organizations, departments, or applications to share resources. A logical entity, also called a tenant, is provided to use diverse resources and servi", + "doc_type":"usermanual", + "kw":"Basic Concepts,Service Overview,User Guide", + "title":"Basic Concepts", + "githuburl":"" + }, + { + "uri":"dli_01_0220.html", + "product_code":"dli", + "code":"8", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Getting Started", + "title":"Getting Started", + "githuburl":"" + }, + { + "uri":"dli_01_0002.html", + "product_code":"dli", + "code":"9", + "des":"You can use DLI to submit a Spark SQL job to query data. The general procedure is as follows:Step 1: Logging in to the Cloud PlatformStep 2: Uploading Data to OBSStep 3: ", + "doc_type":"usermanual", + "kw":"Creating and Submitting a Spark SQL Job,Getting Started,User Guide", + "title":"Creating and Submitting a Spark SQL Job", + "githuburl":"" + }, + { + "uri":"dli_01_0512.html", + "product_code":"dli", + "code":"10", + "des":"To facilitate SQL operations, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not need to wr", + "doc_type":"usermanual", + "kw":"Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template,Getting Started,User Guide", + "title":"Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template", + "githuburl":"" + }, + { + "uri":"dli_01_0375.html", + "product_code":"dli", + "code":"11", + "des":"You can use DLI to submit Spark jobs for real-time computing. The general procedure is as follows:Step 1: Logging in to the CloudStep 2: Uploading Data to OBSStep 3: Logg", + "doc_type":"usermanual", + "kw":"Creating and Submitting a Spark Jar Job,Getting Started,User Guide", + "title":"Creating and Submitting a Spark Jar Job", + "githuburl":"" + }, + { + "uri":"dli_01_0531.html", + "product_code":"dli", + "code":"12", + "des":"This section describes how to submit a Flink OpenSource SQL job on the DLI console for real-time computing. The general procedure is as follows:Step 1: Log In to the Clou", + "doc_type":"usermanual", + "kw":"Creating and Submitting a Flink OpenSource SQL Job,Getting Started,User Guide", + "title":"Creating and Submitting a Flink OpenSource SQL Job", + "githuburl":"" + }, + { + "uri":"dli_01_0377.html", + "product_code":"dli", + "code":"13", + "des":"Log in to DLI and go to the Overview page.The following table describes the functional areas of the Overview page.", + "doc_type":"usermanual", + "kw":"DLI Console Overview,User Guide", + "title":"DLI Console Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0320.html", + "product_code":"dli", + "code":"14", + "des":"You can use SQL statements in the SQL job editor to execute data query. DLI supports SQL 2003 and complies with Spark SQL.On the Overview page, click SQL Editor in the na", + "doc_type":"usermanual", + "kw":"SQL Editor,User Guide", + "title":"SQL Editor", + "githuburl":"" + }, + { + "uri":"dli_01_0001.html", + "product_code":"dli", + "code":"15", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Job Management", + "title":"Job Management", + "githuburl":"" + }, + { + "uri":"dli_01_0017.html", + "product_code":"dli", + "code":"16", + "des":"SQL jobs allow you to execute SQL statements entered in the SQL job editing window, import data, and export data.SQL job management provides the following functions:Searc", + "doc_type":"usermanual", + "kw":"SQL Job Management,Job Management,User Guide", + "title":"SQL Job Management", + "githuburl":"" + }, + { + "uri":"dli_01_0389.html", + "product_code":"dli", + "code":"17", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Flink Job Management", + "title":"Flink Job Management", + "githuburl":"" + }, + { + "uri":"dli_01_0403.html", + "product_code":"dli", + "code":"18", + "des":"On the Job Management page of Flink jobs, you can submit a Flink job. Currently, the following job types are supported:Flink SQL uses SQL statements to define jobs and ca", + "doc_type":"usermanual", + "kw":"Overview,Flink Job Management,User Guide", + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0479.html", + "product_code":"dli", + "code":"19", + "des":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", + "doc_type":"usermanual", + "kw":"Managing Flink Job Permissions,Flink Job Management,User Guide", + "title":"Managing Flink Job Permissions", + "githuburl":"" + }, + { + "uri":"dli_01_0454.html", + "product_code":"dli", + "code":"20", + "des":"To create a Flink job, you need to enter the data source and data output channel, that is, source and sink. To use another service as the source or sink stream, you need ", + "doc_type":"usermanual", + "kw":"Preparing Flink Job Data,Flink Job Management,User Guide", + "title":"Preparing Flink Job Data", + "githuburl":"" + }, + { + "uri":"dli_01_0498.html", + "product_code":"dli", + "code":"21", + "des":"This section describes how to create a Flink OpenSource SQL job. DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the", + "doc_type":"usermanual", + "kw":"(Recommended) Creating a Flink OpenSource SQL Job,Flink Job Management,User Guide", + "title":"(Recommended) Creating a Flink OpenSource SQL Job", + "githuburl":"" + }, + { + "uri":"dli_01_0455.html", + "product_code":"dli", + "code":"22", + "des":"This section describes how to create a Flink SQL job. You can use Flink SQLs to develop jobs to meet your service requirements. Using SQL statements simplifies logic impl", + "doc_type":"usermanual", + "kw":"Creating a Flink SQL job,Flink Job Management,User Guide", + "title":"Creating a Flink SQL job", + "githuburl":"" + }, + { + "uri":"dli_01_0457.html", + "product_code":"dli", + "code":"23", + "des":"This section describes how to create a Flink Jar job. You can perform secondary development based on Flink APIs, build your own JAR file, and submit the JAR file to DLI q", + "doc_type":"usermanual", + "kw":"Creating a Flink Jar Job,Flink Job Management,User Guide", + "title":"Creating a Flink Jar Job", + "githuburl":"" + }, + { + "uri":"dli_01_0458.html", + "product_code":"dli", + "code":"24", + "des":"The job debugging function helps you check the logic correctness of your compiled SQL statements before running a job.Currently, only Flink SQL jobs support this function", + "doc_type":"usermanual", + "kw":"Debugging a Flink Job,Flink Job Management,User Guide", + "title":"Debugging a Flink Job", + "githuburl":"" + }, + { + "uri":"dli_01_0461.html", + "product_code":"dli", + "code":"25", + "des":"After a job is created, you can perform operations on the job as required.Editing a JobStarting a JobStopping a JobDeleting a JobExporting a JobImporting a JobModifying N", + "doc_type":"usermanual", + "kw":"Performing Operations on a Flink Job,Flink Job Management,User Guide", + "title":"Performing Operations on a Flink Job", + "githuburl":"" + }, + { + "uri":"dli_01_0462.html", + "product_code":"dli", + "code":"26", + "des":"After creating a job, you can view the job details to learn about the following information:Viewing Job DetailsChecking the Job Monitoring InformationViewing the Task Lis", + "doc_type":"usermanual", + "kw":"Flink Job Details,Flink Job Management,User Guide", + "title":"Flink Job Details", + "githuburl":"" + }, + { + "uri":"dli_01_0465.html", + "product_code":"dli", + "code":"27", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Spark Job Management", + "title":"Spark Job Management", + "githuburl":"" + }, + { + "uri":"dli_01_0385.html", + "product_code":"dli", + "code":"28", + "des":"Based on the open-source Spark, DLI optimizes performance and reconstructs services to be compatible with the Apache Spark ecosystem and interfaces, and executes batch pr", + "doc_type":"usermanual", + "kw":"Spark Job Management,Spark Job Management,User Guide", + "title":"Spark Job Management", + "githuburl":"" + }, + { + "uri":"dli_01_0384.html", + "product_code":"dli", + "code":"29", + "des":"DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.On the Overview page, click Create Job in the upper right corner of the Spark Jo", + "doc_type":"usermanual", + "kw":"Creating a Spark Job,Spark Job Management,User Guide", + "title":"Creating a Spark Job", + "githuburl":"" + }, + { + "uri":"dli_01_0012.html", + "product_code":"dli", + "code":"30", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Queue Management", + "title":"Queue Management", + "githuburl":"" + }, + { + "uri":"dli_01_0402.html", + "product_code":"dli", + "code":"31", + "des":"Queues in DLI are computing resources, which are the basis for using DLI. All executed jobs require computing resources.Currently, DLI provides two types of queues, For S", + "doc_type":"usermanual", + "kw":"Overview,Queue Management,User Guide", + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0015.html", + "product_code":"dli", + "code":"32", + "des":"You can isolate queues allocated to different users by setting permissions to ensure data query performance.The administrator and queue owner have all permissions, which ", + "doc_type":"usermanual", + "kw":"Queue Permission Management,Queue Management,User Guide", + "title":"Queue Permission Management", + "githuburl":"" + }, + { + "uri":"dli_01_0363.html", + "product_code":"dli", + "code":"33", + "des":"Before executing a job, you need to create a queue.If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main acco", + "doc_type":"usermanual", + "kw":"Creating a Queue,Queue Management,User Guide", + "title":"Creating a Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0016.html", + "product_code":"dli", + "code":"34", + "des":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", + "doc_type":"usermanual", + "kw":"Deleting a Queue,Queue Management,User Guide", + "title":"Deleting a Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0443.html", + "product_code":"dli", + "code":"35", + "des":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", + "doc_type":"usermanual", + "kw":"Modifying the CIDR Block,Queue Management,User Guide", + "title":"Modifying the CIDR Block", + "githuburl":"" + }, + { + "uri":"dli_01_0487.html", + "product_code":"dli", + "code":"36", + "des":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.If Status of queue xxx is assigning, which is not available is ", + "doc_type":"usermanual", + "kw":"Elastic Scaling,Queue Management,User Guide", + "title":"Elastic Scaling", + "githuburl":"" + }, + { + "uri":"dli_01_0488.html", + "product_code":"dli", + "code":"37", + "des":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", + "doc_type":"usermanual", + "kw":"Scheduling CU Changes,Queue Management,User Guide", + "title":"Scheduling CU Changes", + "githuburl":"" + }, + { + "uri":"dli_01_0489.html", + "product_code":"dli", + "code":"38", + "des":"It can be used to test the connectivity between the DLI queue and the peer IP address specified by the user in common scenarios, or the connectivity between the DLI queue", + "doc_type":"usermanual", + "kw":"Testing Address Connectivity,Queue Management,User Guide", + "title":"Testing Address Connectivity", + "githuburl":"" + }, + { + "uri":"dli_01_0421.html", + "product_code":"dli", + "code":"39", + "des":"Once you have created a message notification topic, you can Add subscription of the topic on the Topic Management page of the Simple Message Notification service. You can", + "doc_type":"usermanual", + "kw":"Creating a Message Notification Topic,Queue Management,User Guide", + "title":"Creating a Message Notification Topic", + "githuburl":"" + }, + { + "uri":"dli_01_0004.html", + "product_code":"dli", + "code":"40", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Data Management", + "title":"Data Management", + "githuburl":"" + }, + { + "uri":"dli_01_0390.html", + "product_code":"dli", + "code":"41", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Databases and Tables", + "title":"Databases and Tables", + "githuburl":"" + }, + { + "uri":"dli_01_0228.html", + "product_code":"dli", + "code":"42", + "des":"DLI database and table management provide the following functions:Database Permission ManagementTable Permission ManagementCreating a Database or a TableDeleting a Databa", + "doc_type":"usermanual", + "kw":"Overview,Databases and Tables,User Guide", + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0447.html", + "product_code":"dli", + "code":"43", + "des":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "doc_type":"usermanual", + "kw":"Database Permission Management,Databases and Tables,User Guide", + "title":"Database Permission Management", + "githuburl":"" + }, + { + "uri":"dli_01_0448.html", + "product_code":"dli", + "code":"44", + "des":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "doc_type":"usermanual", + "kw":"Table Permission Management,Databases and Tables,User Guide", + "title":"Table Permission Management", + "githuburl":"" + }, + { + "uri":"dli_01_0005.html", + "product_code":"dli", + "code":"45", + "des":"A database, built on the computer storage device, is a data warehouse where data is organized, stored, and managed based on its structure.The table is an important part o", + "doc_type":"usermanual", + "kw":"Creating a Database or a Table,Databases and Tables,User Guide", + "title":"Creating a Database or a Table", + "githuburl":"" + }, + { + "uri":"dli_01_0011.html", + "product_code":"dli", + "code":"46", + "des":"You can delete unnecessary databases and tables based on actual conditions.You are not allowed to delete databases or tables that are being used for running jobs.The admi", + "doc_type":"usermanual", + "kw":"Deleting a Database or a Table,Databases and Tables,User Guide", + "title":"Deleting a Database or a Table", + "githuburl":"" + }, + { + "uri":"dli_01_0376.html", + "product_code":"dli", + "code":"47", + "des":"During actual use, developers create databases and tables and submit them to test personnel for testing. After the test is complete, the databases and tables are transfer", + "doc_type":"usermanual", + "kw":"Modifying the Owners of Databases and Tables,Databases and Tables,User Guide", + "title":"Modifying the Owners of Databases and Tables", + "githuburl":"" + }, + { + "uri":"dli_01_0253.html", + "product_code":"dli", + "code":"48", + "des":"You can import data from OBS to a table created in DLI.Only one path can be specified during data import. The path cannot contain commas (,).To import data in CSV format ", + "doc_type":"usermanual", + "kw":"Importing Data to the Table,Databases and Tables,User Guide", + "title":"Importing Data to the Table", + "githuburl":"" + }, + { + "uri":"dli_01_0010.html", + "product_code":"dli", + "code":"49", + "des":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", + "doc_type":"usermanual", + "kw":"Exporting Data from DLI to OBS,Databases and Tables,User Guide", + "title":"Exporting Data from DLI to OBS", + "githuburl":"" + }, + { + "uri":"dli_01_0008.html", + "product_code":"dli", + "code":"50", + "des":"Metadata is used to define data types. It describes information about the data, including the source, size, format, and other data features. In database fields, metadata ", + "doc_type":"usermanual", + "kw":"Viewing Metadata,Databases and Tables,User Guide", + "title":"Viewing Metadata", + "githuburl":"" + }, + { + "uri":"dli_01_0007.html", + "product_code":"dli", + "code":"51", + "des":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", + "doc_type":"usermanual", + "kw":"Previewing Data,Databases and Tables,User Guide", + "title":"Previewing Data", + "githuburl":"" + }, + { + "uri":"dli_01_0366.html", + "product_code":"dli", + "code":"52", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Package Management", + "title":"Package Management", + "githuburl":"" + }, + { + "uri":"dli_01_0407.html", + "product_code":"dli", + "code":"53", + "des":"Package management provides the following functions:Managing Package PermissionsCreating a PackageDeleting a PackageYou can delete program packages in batches.You can del", + "doc_type":"usermanual", + "kw":"Overview,Package Management,User Guide", + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0477.html", + "product_code":"dli", + "code":"54", + "des":"You can isolate package groups or packages allocated to different users by setting permissions to ensure data query performance.The administrator and the owner of a packa", + "doc_type":"usermanual", + "kw":"Managing Permissions on Packages and Package Groups,Package Management,User Guide", + "title":"Managing Permissions on Packages and Package Groups", + "githuburl":"" + }, + { + "uri":"dli_01_0367.html", + "product_code":"dli", + "code":"55", + "des":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", + "doc_type":"usermanual", + "kw":"Creating a Package,Package Management,User Guide", + "title":"Creating a Package", + "githuburl":"" + }, + { + "uri":"dli_01_0369.html", + "product_code":"dli", + "code":"56", + "des":"You can delete a package based on actual conditions.On the left of the management console, choose Data Management > Package Management.Click Delete in the Operation colum", + "doc_type":"usermanual", + "kw":"Deleting a Package,Package Management,User Guide", + "title":"Deleting a Package", + "githuburl":"" + }, + { + "uri":"dli_01_0478.html", + "product_code":"dli", + "code":"57", + "des":"To change the owner of a package, click More > Modify Owner in the Operation column of a package on the Package Management page.If the package has been grouped, you can m", + "doc_type":"usermanual", + "kw":"Modifying the Owner,Package Management,User Guide", + "title":"Modifying the Owner", + "githuburl":"" + }, + { + "uri":"dli_01_0397.html", + "product_code":"dli", + "code":"58", + "des":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", + "doc_type":"usermanual", + "kw":"Built-in Dependencies,Package Management,User Guide", + "title":"Built-in Dependencies", + "githuburl":"" + }, + { + "uri":"dli_01_0379.html", + "product_code":"dli", + "code":"59", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Job Templates", + "title":"Job Templates", + "githuburl":"" + }, + { + "uri":"dli_01_0021.html", + "product_code":"dli", + "code":"60", + "des":"To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not n", + "doc_type":"usermanual", + "kw":"SQL Template Management,Job Templates,User Guide", + "title":"SQL Template Management", + "githuburl":"" + }, + { + "uri":"dli_01_0464.html", + "product_code":"dli", + "code":"61", + "des":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", + "doc_type":"usermanual", + "kw":"Flink Template Management,Job Templates,User Guide", + "title":"Flink Template Management", + "githuburl":"" + }, + { + "uri":"dli_01_05110.html", + "product_code":"dli", + "code":"62", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Appendix", + "title":"Appendix", + "githuburl":"" + }, + { + "uri":"dli_01_05111.html", + "product_code":"dli", + "code":"63", + "des":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", + "doc_type":"usermanual", + "kw":"TPC-H Sample Data in the SQL Template,Appendix,User Guide", + "title":"TPC-H Sample Data in the SQL Template", + "githuburl":"" + }, + { + "uri":"dli_01_0422.html", + "product_code":"dli", + "code":"64", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Datasource Connections", + "title":"Datasource Connections", + "githuburl":"" + }, + { + "uri":"dli_01_0410.html", + "product_code":"dli", + "code":"65", + "des":"DLI supports the datasource capability of the native Spark and extends it. With DLI datasource connection, you can access other data storage services through SQL statemen", + "doc_type":"usermanual", + "kw":"Datasource Connection and Cross-Source Analysis,Datasource Connections,User Guide", + "title":"Datasource Connection and Cross-Source Analysis", + "githuburl":"" + }, + { + "uri":"dli_01_0426.html", + "product_code":"dli", + "code":"66", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Enhanced Datasource Connections", + "title":"Enhanced Datasource Connections", + "githuburl":"" + }, + { + "uri":"dli_01_0003.html", + "product_code":"dli", + "code":"67", + "des":"The enhanced datasource connection uses VPC peering at the bottom layer to directly connect the VPC network between the DLI cluster and the destination datasource. Data i", + "doc_type":"usermanual", + "kw":"Overview,Enhanced Datasource Connections,User Guide", + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0006.html", + "product_code":"dli", + "code":"68", + "des":"The following describes how to create a datasource HBase connection for MRS.Only enhanced datasource connection to MRS HBase is supported.Apply for a cluster in MRS.If a ", + "doc_type":"usermanual", + "kw":"Creating, Querying, and Deleting an Enhanced Datasource Connection,Enhanced Datasource Connections,U", + "title":"Creating, Querying, and Deleting an Enhanced Datasource Connection", + "githuburl":"" + }, + { + "uri":"dli_01_0009.html", + "product_code":"dli", + "code":"69", + "des":"The CIDR block of the DLI queue that is bound with a datasource connection cannot overlap with that of the data source.The default queue cannot be bound with a connection", + "doc_type":"usermanual", + "kw":"Binding and Unbinding a Queue,Enhanced Datasource Connections,User Guide", + "title":"Binding and Unbinding a Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0013.html", + "product_code":"dli", + "code":"70", + "des":"Method 1: Copy hosts information in /etc/hosts of an MRS node.cat /etc/hostsMethod 2: Log in to FusionInsight Manager to obtain MRS hosts information.The host information", + "doc_type":"usermanual", + "kw":"Modifying Host Information,Enhanced Datasource Connections,User Guide", + "title":"Modifying Host Information", + "githuburl":"" + }, + { + "uri":"dli_01_0014.html", + "product_code":"dli", + "code":"71", + "des":"After an enhanced datasource connection is created and bound to a queue, the system automatically configures route information. You can also add a custom route for the qu", + "doc_type":"usermanual", + "kw":"Custom Route Information,Enhanced Datasource Connections,User Guide", + "title":"Custom Route Information", + "githuburl":"" + }, + { + "uri":"dli_01_0018.html", + "product_code":"dli", + "code":"72", + "des":"You can grant or revoke permissions for enhanced datasource connections of other projects.AuthorizationLog in to the DLI management console, choose Datasource Connections", + "doc_type":"usermanual", + "kw":"Enhanced Datasource Connection Permission Management,Enhanced Datasource Connections,User Guide", + "title":"Enhanced Datasource Connection Permission Management", + "githuburl":"" + }, + { + "uri":"dli_01_0480.html", + "product_code":"dli", + "code":"73", + "des":"You can isolate datasource connections allocated to different users by setting permissions to ensure data query performance.The administrator and datasource connection ow", + "doc_type":"usermanual", + "kw":"Managing Datasource Connection Permissions,Datasource Connections,User Guide", + "title":"Managing Datasource Connection Permissions", + "githuburl":"" + }, + { + "uri":"dli_01_0427.html", + "product_code":"dli", + "code":"74", + "des":"Datasource authentication is used to manage authentication information for accessing CSS and MRS security environments and encrypt passwords for accessing DWS, RDS, DDS, ", + "doc_type":"usermanual", + "kw":"Creating and Managing Datasource Authentication,Datasource Connections,User Guide", + "title":"Creating and Managing Datasource Authentication", + "githuburl":"" + }, + { + "uri":"dli_01_0485.html", + "product_code":"dli", + "code":"75", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Global Configuration", + "title":"Global Configuration", + "githuburl":"" + }, + { + "uri":"dli_01_0476.html", + "product_code":"dli", + "code":"76", + "des":"Global variables can be used to simplify complex parameters. For example, long and difficult variables can be replaced to improve the readability of SQL statements.In the", + "doc_type":"usermanual", + "kw":"Global Variables,Global Configuration,User Guide", + "title":"Global Variables", + "githuburl":"" + }, + { + "uri":"dli_01_0486.html", + "product_code":"dli", + "code":"77", + "des":"Only the tenant account or a subaccount of user group admin can authorize access.After entering the DLI management console, you are advised to set agency permissions to e", + "doc_type":"usermanual", + "kw":"Service Authorization,Global Configuration,User Guide", + "title":"Service Authorization", + "githuburl":"" + }, + { + "uri":"dli_01_0408.html", + "product_code":"dli", + "code":"78", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Permissions Management", + "title":"Permissions Management", + "githuburl":"" + }, + { + "uri":"dli_01_0440.html", + "product_code":"dli", + "code":"79", + "des":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", + "doc_type":"usermanual", + "kw":"Overview,Permissions Management,User Guide", + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0418.html", + "product_code":"dli", + "code":"80", + "des":"You can use Identity and Access Management (IAM) to implement fine-grained permissions control on DLI resources. For details, see Overview.If your cloud account does not ", + "doc_type":"usermanual", + "kw":"Creating an IAM User and Granting Permissions,Permissions Management,User Guide", + "title":"Creating an IAM User and Granting Permissions", + "githuburl":"" + }, + { + "uri":"dli_01_0451.html", + "product_code":"dli", + "code":"81", + "des":"Custom policies can be created as a supplement to the system policies of DLI. You can add actions to custom policies. For the actions supported for custom policies, see \"", + "doc_type":"usermanual", + "kw":"Creating a Custom Policy,Permissions Management,User Guide", + "title":"Creating a Custom Policy", + "githuburl":"" + }, + { + "uri":"dli_01_0417.html", + "product_code":"dli", + "code":"82", + "des":"A resource is an object that exists within a service. You can select DLI resources by specifying their paths.", + "doc_type":"usermanual", + "kw":"DLI Resources,Permissions Management,User Guide", + "title":"DLI Resources", + "githuburl":"" + }, + { + "uri":"dli_01_0475.html", + "product_code":"dli", + "code":"83", + "des":"Request conditions are useful in determining when a custom policy takes effect. A request condition consists of a condition key and operator. Condition keys are either gl", + "doc_type":"usermanual", + "kw":"DLI Request Conditions,Permissions Management,User Guide", + "title":"DLI Request Conditions", + "githuburl":"" + }, + { + "uri":"dli_01_0441.html", + "product_code":"dli", + "code":"84", + "des":"Table 1 lists the common operations supported by each system policy of DLI. Choose proper system policies according to this table. For details about the SQL statement per", + "doc_type":"usermanual", + "kw":"Common Operations Supported by DLI System Policy,Permissions Management,User Guide", + "title":"Common Operations Supported by DLI System Policy", + "githuburl":"" + }, + { + "uri":"dli_01_0539.html", + "product_code":"dli", + "code":"85", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"FAQs", + "title":"FAQs", + "githuburl":"" + }, + { + "uri":"dli_03_0037.html", + "product_code":"dli", + "code":"86", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Flink Jobs", + "title":"Flink Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0083.html", + "product_code":"dli", + "code":"87", + "des":"DLI Flink jobs support the following data formats:Avro, Avro_merge, BLOB, CSV, EMAIL, JSON, ORC, Parquet, and XML.DLI Flink jobs support data from the following data sour", + "doc_type":"usermanual", + "kw":"What Data Formats and Data Sources Are Supported by DLI Flink Jobs?,Flink Jobs,User Guide", + "title":"What Data Formats and Data Sources Are Supported by DLI Flink Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0139.html", + "product_code":"dli", + "code":"88", + "des":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", + "doc_type":"usermanual", + "kw":"How Do I Authorize a Subuser to View Flink Jobs?,Flink Jobs,User Guide", + "title":"How Do I Authorize a Subuser to View Flink Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0090.html", + "product_code":"dli", + "code":"89", + "des":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", + "doc_type":"usermanual", + "kw":"How Do I Set Auto Restart upon Exception for a Flink Job?,Flink Jobs,User Guide", + "title":"How Do I Set Auto Restart upon Exception for a Flink Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0099.html", + "product_code":"dli", + "code":"90", + "des":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", + "doc_type":"usermanual", + "kw":"How Do I Save Flink Job Logs?,Flink Jobs,User Guide", + "title":"How Do I Save Flink Job Logs?", + "githuburl":"" + }, + { + "uri":"dli_03_0043.html", + "product_code":"dli", + "code":"91", + "des":"DLI can output Flink job results to DIS. You can view the results in DIS. For details, see Retrieving Data from DIS in the Data Ingestion Service User Guide.DLI can outpu", + "doc_type":"usermanual", + "kw":"How Can I Check Flink Job Results?,Flink Jobs,User Guide", + "title":"How Can I Check Flink Job Results?", + "githuburl":"" + }, + { + "uri":"dli_03_0160.html", + "product_code":"dli", + "code":"92", + "des":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", + "doc_type":"usermanual", + "kw":"What Should I Do if \"No such user. userName:xxxx.\" Is Displayed on the Flink Job Management Page Whe", + "title":"What Should I Do if \"No such user. userName:xxxx.\" Is Displayed on the Flink Job Management Page When I Grant Permission to a User?", + "githuburl":"" + }, + { + "uri":"dli_03_0180.html", + "product_code":"dli", + "code":"93", + "des":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. After a Flink job is manually stopped, no message is display", + "doc_type":"usermanual", + "kw":"How Do I Know Which Checkpoint the Flink Job I Stopped Will Be Restored to When I Start the Job Agai", + "title":"How Do I Know Which Checkpoint the Flink Job I Stopped Will Be Restored to When I Start the Job Again?", + "githuburl":"" + }, + { + "uri":"dli_03_0130.html", + "product_code":"dli", + "code":"94", + "des":"The consumption capability of a Flink SQL job depends on the data source transmission, queue size, and job parameter settings. The peak consumption is 10 Mbit/s.", + "doc_type":"usermanual", + "kw":"How Much Data Can Be Processed in a Day by a Flink SQL Job?,Flink Jobs,User Guide", + "title":"How Much Data Can Be Processed in a Day by a Flink SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0061.html", + "product_code":"dli", + "code":"95", + "des":"The temp stream in Flink SQL is similar to a subquery. It is a logical stream used to simplify the SQL logic and does not generate data storage. Therefore, there is no ne", + "doc_type":"usermanual", + "kw":"Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the D", + "title":"Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the Data?", + "githuburl":"" + }, + { + "uri":"dli_03_0138.html", + "product_code":"dli", + "code":"96", + "des":"SymptomWhen you create a Flink SQL job and configure the parameters, you select an OBS bucket you have created. The system displays a message indicating that the OBS buck", + "doc_type":"usermanual", + "kw":"What Should I Do if a Message Is Displayed Indicating that the OBS Bucket Is Not Authorized When I S", + "title":"What Should I Do if a Message Is Displayed Indicating that the OBS Bucket Is Not Authorized When I Select an OBS Bucket for a Flink SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0089.html", + "product_code":"dli", + "code":"97", + "des":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", + "doc_type":"usermanual", + "kw":"How Do I Map an OBS Table to a Partitioned DLI Table?,Flink Jobs,User Guide", + "title":"How Do I Map an OBS Table to a Partitioned DLI Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0120.html", + "product_code":"dli", + "code":"98", + "des":"SymptomYou used Flink 1.10 to run a Flink Opensource SQL job. You set the number of Kafka partitions for the job a small value at the beginning and need to increase the n", + "doc_type":"usermanual", + "kw":"How Do I Change the Number of Kafka Partitions of a Flink SQL Job Without Stopping It?,Flink Jobs,Us", + "title":"How Do I Change the Number of Kafka Partitions of a Flink SQL Job Without Stopping It?", + "githuburl":"" + }, + { + "uri":"dli_03_0075.html", + "product_code":"dli", + "code":"99", + "des":"In this example, the day field is used as the partition field with the parquet encoding format (only the parquet format is supported currently) to dump car_info data to O", + "doc_type":"usermanual", + "kw":"How Can I Map an OBS Table to a DLI Partition Table?,Flink Jobs,User Guide", + "title":"How Can I Map an OBS Table to a DLI Partition Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0167.html", + "product_code":"dli", + "code":"100", + "des":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", + "doc_type":"usermanual", + "kw":"What Should I Do If Error Message DLI.0005 Is Displayed When I Use an EL Expression to Create a Tabl", + "title":"What Should I Do If Error Message DLI.0005 Is Displayed When I Use an EL Expression to Create a Table in a Flink SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0168.html", + "product_code":"dli", + "code":"101", + "des":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", + "doc_type":"usermanual", + "kw":"What Should I Do If No Data Is Found in the DLI Table Created Using the OBS File Path That Stores th", + "title":"What Should I Do If No Data Is Found in the DLI Table Created Using the OBS File Path That Stores the Output of a Flink job?", + "githuburl":"" + }, + { + "uri":"dli_03_0174.html", + "product_code":"dli", + "code":"102", + "des":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", + "doc_type":"usermanual", + "kw":"What Should I Do If a Flink SQL Job Fails to Be Executed, and \"connect to DIS failed java.lang.Illeg", + "title":"What Should I Do If a Flink SQL Job Fails to Be Executed, and \"connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null\" Is Recorded in the Log?", + "githuburl":"" + }, + { + "uri":"dli_03_0176.html", + "product_code":"dli", + "code":"103", + "des":"Semantic verification for a Flink SQL job (reading DIS data) fails. The following information is displayed when the job fails:Get dis channel xxxinfo failed. error info: ", + "doc_type":"usermanual", + "kw":"What Should I Do If \"Not authorized\" Is Reported When a Flink SQL Job Reads DIS Data?,Flink Jobs,Use", + "title":"What Should I Do If \"Not authorized\" Is Reported When a Flink SQL Job Reads DIS Data?", + "githuburl":"" + }, + { + "uri":"dli_03_0232.html", + "product_code":"dli", + "code":"104", + "des":"After a Flink SQL job consumed Kafka and sank data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", + "doc_type":"usermanual", + "kw":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster,F", + "title":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster", + "githuburl":"" + }, + { + "uri":"dli_03_0038.html", + "product_code":"dli", + "code":"105", + "des":"The procedure is as follows:Add the following code to the JAR package code of the Flink Jar job:// Configure the pom file on which the StreamExecutionEnvironment depends.", + "doc_type":"usermanual", + "kw":"How Do I Configure Checkpoints for Flink Jar Jobs and Save the Checkpoints to OBS?,Flink Jobs,User G", + "title":"How Do I Configure Checkpoints for Flink Jar Jobs and Save the Checkpoints to OBS?", + "githuburl":"" + }, + { + "uri":"dli_03_0044.html", + "product_code":"dli", + "code":"106", + "des":"Configuration files can be uploaded for user-defined jobs (JAR).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flin", + "doc_type":"usermanual", + "kw":"Does a Flink JAR Job Support Configuration File Upload? How Do I Upload a Configuration File?,Flink ", + "title":"Does a Flink JAR Job Support Configuration File Upload? How Do I Upload a Configuration File?", + "githuburl":"" + }, + { + "uri":"dli_03_0119.html", + "product_code":"dli", + "code":"107", + "des":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Delete your JAR package that is the", + "doc_type":"usermanual", + "kw":"What Should I Do If Job Submission Fails Due to Flink JAR Package Conflict?,Flink Jobs,User Guide", + "title":"What Should I Do If Job Submission Fails Due to Flink JAR Package Conflict?", + "githuburl":"" + }, + { + "uri":"dli_03_0161.html", + "product_code":"dli", + "code":"108", + "des":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", + "doc_type":"usermanual", + "kw":"What Should I Do If GaussDB(DWS) Fails to Be Started When a Flink Jar Job Accesses GaussDB(DWS), and", + "title":"What Should I Do If GaussDB(DWS) Fails to Be Started When a Flink Jar Job Accesses GaussDB(DWS), and a Message Indicating Too Many Client Connections is Displayed?", + "githuburl":"" + }, + { + "uri":"dli_03_0165.html", + "product_code":"dli", + "code":"109", + "des":"An exception occurred when a Flink Jar job is running. The following error information is displayed in the job log:org.apache.flink.shaded.curator.org.apache.curator.Conn", + "doc_type":"usermanual", + "kw":"What Should I Do If Error Message \"Authentication failed\" Is Displayed When a Flink Jar Job Is Runni", + "title":"What Should I Do If Error Message \"Authentication failed\" Is Displayed When a Flink Jar Job Is Running?", + "githuburl":"" + }, + { + "uri":"dli_03_0233.html", + "product_code":"dli", + "code":"110", + "des":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", + "doc_type":"usermanual", + "kw":"What Should I Do If Invalid OBS Bucket Name Is Reported After a Flink Job Submission Failed?,Flink J", + "title":"What Should I Do If Invalid OBS Bucket Name Is Reported After a Flink Job Submission Failed?", + "githuburl":"" + }, + { + "uri":"dli_03_0234.html", + "product_code":"dli", + "code":"111", + "des":"Flink Job submission failed. The exception information is as follows:Flink JAR packages conflicted. The submitted Flink JAR package conflicted with the HDFS JAR package o", + "doc_type":"usermanual", + "kw":"Flink Job Submission Fails Due to Hadoop JAR Package Conflicts,Flink Jobs,User Guide", + "title":"Flink Job Submission Fails Due to Hadoop JAR Package Conflicts", + "githuburl":"" + }, + { + "uri":"dli_03_0266.html", + "product_code":"dli", + "code":"112", + "des":"You can use Flink Jar to connect to Kafka with SASL SSL authentication enabled.", + "doc_type":"usermanual", + "kw":"How Do I Connect a Flink jar Job to SASL_SSL?,Flink Jobs,User Guide", + "title":"How Do I Connect a Flink jar Job to SASL_SSL?", + "githuburl":"" + }, + { + "uri":"dli_03_0106.html", + "product_code":"dli", + "code":"113", + "des":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", + "doc_type":"usermanual", + "kw":"How Do I Optimize Performance of a Flink Job?,Flink Jobs,User Guide", + "title":"How Do I Optimize Performance of a Flink Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0048.html", + "product_code":"dli", + "code":"114", + "des":"Add the following SQL statements to the Flink job:", + "doc_type":"usermanual", + "kw":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?,Flink Jobs,User Guide", + "title":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0096.html", + "product_code":"dli", + "code":"115", + "des":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", + "doc_type":"usermanual", + "kw":"How Do I Prevent Data Loss After Flink Job Restart?,Flink Jobs,User Guide", + "title":"How Do I Prevent Data Loss After Flink Job Restart?", + "githuburl":"" + }, + { + "uri":"dli_03_0103.html", + "product_code":"dli", + "code":"116", + "des":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", + "doc_type":"usermanual", + "kw":"How Do I Locate a Flink Job Submission Error?,Flink Jobs,User Guide", + "title":"How Do I Locate a Flink Job Submission Error?", + "githuburl":"" + }, + { + "uri":"dli_03_0105.html", + "product_code":"dli", + "code":"117", + "des":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", + "doc_type":"usermanual", + "kw":"How Do I Locate a Flink Job Running Error?,Flink Jobs,User Guide", + "title":"How Do I Locate a Flink Job Running Error?", + "githuburl":"" + }, + { + "uri":"dli_03_0136.html", + "product_code":"dli", + "code":"118", + "des":"Check the following operations:Adjusting or adding optimization parameters or the number of concurrent threads of a job, or modifying Flink SQL statements or a Flink Jar ", + "doc_type":"usermanual", + "kw":"How Do I Know Whether a Flink Job Can Be Restored from a Checkpoint After Being Restarted?,Flink Job", + "title":"How Do I Know Whether a Flink Job Can Be Restored from a Checkpoint After Being Restarted?", + "githuburl":"" + }, + { + "uri":"dli_03_0040.html", + "product_code":"dli", + "code":"119", + "des":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", + "doc_type":"usermanual", + "kw":"What Should I Do If the DIS Stream Does Not Exist During the Semantic Check?,Flink Jobs,User Guide", + "title":"What Should I Do If the DIS Stream Does Not Exist During the Semantic Check?", + "githuburl":"" + }, + { + "uri":"dli_03_0045.html", + "product_code":"dli", + "code":"120", + "des":"If the OBS bucket selected for a job is not authorized, perform the following steps:Select Enable Checkpointing or Save Job Log.Specify OBS Bucket.Select Authorize OBS.", + "doc_type":"usermanual", + "kw":"What Should I Do If the OBS Bucket Selected for a Job Is Not Authorized?,Flink Jobs,User Guide", + "title":"What Should I Do If the OBS Bucket Selected for a Job Is Not Authorized?", + "githuburl":"" + }, + { + "uri":"dli_03_0064.html", + "product_code":"dli", + "code":"121", + "des":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", + "doc_type":"usermanual", + "kw":"What Should I Do If Logs Are Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitt", + "title":"What Should I Do If Logs Are Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?", + "githuburl":"" + }, + { + "uri":"dli_03_0121.html", + "product_code":"dli", + "code":"122", + "des":"SymptomYou used Flink 1.10 to run a Flink Opensource SQL job. The job failed after the following error was reported when Flink Sink wrote data to Kafka.Caused by: org.apa", + "doc_type":"usermanual", + "kw":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?,Flink Jobs,User Guide", + "title":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?", + "githuburl":"" + }, + { + "uri":"dli_03_0235.html", + "product_code":"dli", + "code":"123", + "des":"The Flink/Spark UI was displayed with incomplete information.When a pay-per-use queue is used to run a job, the system releases the cluster and takes about 10 minutes to ", + "doc_type":"usermanual", + "kw":"What Should I Do If the Flink/Spark UI Page Cannot Be Displayed Properly?,Flink Jobs,User Guide", + "title":"What Should I Do If the Flink/Spark UI Page Cannot Be Displayed Properly?", + "githuburl":"" + }, + { + "uri":"dli_03_0236.html", + "product_code":"dli", + "code":"124", + "des":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", + "doc_type":"usermanual", + "kw":"Wha should I Do If JobManager and TaskManager Heartbeats Timed Out and the Flink Job Is Abnormal?,Fl", + "title":"Wha should I Do If JobManager and TaskManager Heartbeats Timed Out and the Flink Job Is Abnormal?", + "githuburl":"" + }, + { + "uri":"dli_03_0265.html", + "product_code":"dli", + "code":"125", + "des":"Test address connectivity.If the network is unreachable, rectify the network connection first. Ensure that the network connection between the DLI queue and the external d", + "doc_type":"usermanual", + "kw":"What Can I Do If \"Timeout expired while fetching topic metadata\" Is Repeatedly Reported in Flink Job", + "title":"What Can I Do If \"Timeout expired while fetching topic metadata\" Is Repeatedly Reported in Flink JobManager Logs?", + "githuburl":"" + }, + { + "uri":"dli_03_0020.html", + "product_code":"dli", + "code":"126", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Problems Related to SQL Jobs", + "title":"Problems Related to SQL Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0200.html", + "product_code":"dli", + "code":"127", + "des":"A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in", + "doc_type":"usermanual", + "kw":"SQL Jobs,Problems Related to SQL Jobs,User Guide", + "title":"SQL Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0086.html", + "product_code":"dli", + "code":"128", + "des":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.Set t", + "doc_type":"usermanual", + "kw":"How Do I Merge Small Files?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I Merge Small Files?", + "githuburl":"" + }, + { + "uri":"dli_03_0092.html", + "product_code":"dli", + "code":"129", + "des":"When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.If the specified path is akdc", + "doc_type":"usermanual", + "kw":"How Do I Specify an OBS Path When Creating an OBS Table?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I Specify an OBS Path When Creating an OBS Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0108.html", + "product_code":"dli", + "code":"130", + "des":"DLI allows you to associate JSON data in an OBS bucket to create tables in asynchronous mode.The statement for creating the table is as follows:", + "doc_type":"usermanual", + "kw":"How Do I Create a Table Using JSON Data in an OBS Bucket?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I Create a Table Using JSON Data in an OBS Bucket?", + "githuburl":"" + }, + { + "uri":"dli_03_0087.html", + "product_code":"dli", + "code":"131", + "des":"You can use the where condition statement in the select statement to filter data. For example:", + "doc_type":"usermanual", + "kw":"How Do I Set Local Variables in SQL Statements?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I Set Local Variables in SQL Statements?", + "githuburl":"" + }, + { + "uri":"dli_03_0069.html", + "product_code":"dli", + "code":"132", + "des":"The correct method for using the count function to perform aggregation is as follows:OrIf an incorrect method is used, an error will be reported.", + "doc_type":"usermanual", + "kw":"How Can I Use the count Function to Perform Aggregation?,Problems Related to SQL Jobs,User Guide", + "title":"How Can I Use the count Function to Perform Aggregation?", + "githuburl":"" + }, + { + "uri":"dli_03_0072.html", + "product_code":"dli", + "code":"133", + "des":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket. For details, s", + "doc_type":"usermanual", + "kw":"How Do I Synchronize DLI Table Data from One Region to Another?,Problems Related to SQL Jobs,User Gu", + "title":"How Do I Synchronize DLI Table Data from One Region to Another?", + "githuburl":"" + }, + { + "uri":"dli_03_0191.html", + "product_code":"dli", + "code":"134", + "des":"Currently, DLI does not allow you to insert table data into specific fields. To insert table data, you must insert data of all table fields at a time.", + "doc_type":"usermanual", + "kw":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?,Problems Related to SQL ", + "title":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0014.html", + "product_code":"dli", + "code":"135", + "des":"Create an OBS directory with a unique name. Alternatively, you can manually delete the existing OBS directory and submit the job again. However, exercise caution when del", + "doc_type":"usermanual", + "kw":"What Should I Do If Error \"path obs://xxx already exists\" Is Reported When Data Is Exported to OBS?,", + "title":"What Should I Do If Error \"path obs://xxx already exists\" Is Reported When Data Is Exported to OBS?", + "githuburl":"" + }, + { + "uri":"dli_03_0066.html", + "product_code":"dli", + "code":"136", + "des":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", + "doc_type":"usermanual", + "kw":"What Should I Do If \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Is D", + "title":"What Should I Do If \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Is Displayed When Two Tables Are Joined", + "githuburl":"" + }, + { + "uri":"dli_03_0071.html", + "product_code":"dli", + "code":"137", + "des":"Check whether the account is in arrears. If yes, recharge the account.If the error persists, log out and log in again.", + "doc_type":"usermanual", + "kw":"What Should I Do If the System Notify Me that I Have No Permission to Execute the Query Statement Be", + "title":"What Should I Do If the System Notify Me that I Have No Permission to Execute the Query Statement Because the Current Account Was Restricted?", + "githuburl":"" + }, + { + "uri":"dli_03_0145.html", + "product_code":"dli", + "code":"138", + "des":"Cause AnalysisWhen you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the q", + "doc_type":"usermanual", + "kw":"What Should I Do If Error Message \"There should be at least one partition pruning predicate on parti", + "title":"What Should I Do If Error Message \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" Is Displayed When a Query Statement Is Executed?", + "githuburl":"" + }, + { + "uri":"dli_03_0169.html", + "product_code":"dli", + "code":"139", + "des":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", + "doc_type":"usermanual", + "kw":"What Should I Do If Error Message IllegalArgumentException: Buffer size too small. size is Displayed", + "title":"What Should I Do If Error Message IllegalArgumentException: Buffer size too small. size is Displayed When Data Is Loaded to an OBS Foreign Table", + "githuburl":"" + }, + { + "uri":"dli_03_0189.html", + "product_code":"dli", + "code":"140", + "des":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", + "doc_type":"usermanual", + "kw":"What Should I Do If \"DLI.0002 FileNotFoundException\" Is Reported During SQL Job Running,Problems Rel", + "title":"What Should I Do If \"DLI.0002 FileNotFoundException\" Is Reported During SQL Job Running", + "githuburl":"" + }, + { + "uri":"dli_03_0046.html", + "product_code":"dli", + "code":"141", + "des":"Currently, DLI supports the Hive syntax for creating tables of the TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET file types. If the file format specified for cre", + "doc_type":"usermanual", + "kw":"What Should I Do If A Schema Parsing Error Is Reported When I Create a Hive Table Using CTAS,Problem", + "title":"What Should I Do If A Schema Parsing Error Is Reported When I Create a Hive Table Using CTAS", + "githuburl":"" + }, + { + "uri":"dli_03_0173.html", + "product_code":"dli", + "code":"142", + "des":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", + "doc_type":"usermanual", + "kw":"What Should I Do If \"org.apache.hadoop.fs.obs.OBSIOException\" Is Reported If DLI SQL Scripts Are Exe", + "title":"What Should I Do If \"org.apache.hadoop.fs.obs.OBSIOException\" Is Reported If DLI SQL Scripts Are Executed on DataArts Studio?", + "githuburl":"" + }, + { + "uri":"dli_03_0172.html", + "product_code":"dli", + "code":"143", + "des":"After the migration job is submitted, the following error information is displayed in the log:org.apache.sqoop.common.SqoopException:UQUERY_CONNECTOR_0001:Invoke DLI serv", + "doc_type":"usermanual", + "kw":"What Should I Do If \"UQUERY_CONNECTOR_0001:Invoke DLI service api failed\" Is Reported in the Job Log", + "title":"What Should I Do If \"UQUERY_CONNECTOR_0001:Invoke DLI service api failed\" Is Reported in the Job Log When I Use CDM to Migrate Data to DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0207.html", + "product_code":"dli", + "code":"144", + "des":"Error message \"File not Found\" is displayed when a SQL job is accessed.Generally, the file cannot be found due to a read/write conflict. Check whether a job is overwritin", + "doc_type":"usermanual", + "kw":"What Should I Do If \"File not Found\" Is Displayed When I Access a SQL Job?,Problems Related to SQL J", + "title":"What Should I Do If \"File not Found\" Is Displayed When I Access a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0208.html", + "product_code":"dli", + "code":"145", + "des":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.View the OBS bucket in the AccessControlException and check whether you are us", + "doc_type":"usermanual", + "kw":"What Should I Do If \"DLI.0003: AccessControlException XXX\" Is Reported When I Access a SQL Job?,Prob", + "title":"What Should I Do If \"DLI.0003: AccessControlException XXX\" Is Reported When I Access a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0209.html", + "product_code":"dli", + "code":"146", + "des":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Th", + "doc_type":"usermanual", + "kw":"What Should I Do If \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists", + "title":"What Should I Do If \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Is Reported When I Access a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0210.html", + "product_code":"dli", + "code":"147", + "des":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", + "doc_type":"usermanual", + "kw":"What Should I Do If \"The current account does not have permission to perform this operation,the curr", + "title":"What Should I Do If \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget\" Is Reported During SQL Statement Execution? Restricted for no budget.", + "githuburl":"" + }, + { + "uri":"dli_03_0196.html", + "product_code":"dli", + "code":"148", + "des":"If the job runs slowly, perform the following steps to find the causes and rectify the fault:Check whether the problem is caused by FullGC.Log in to the DLI console. In t", + "doc_type":"usermanual", + "kw":"How Do I Troubleshoot Slow SQL Jobs?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I Troubleshoot Slow SQL Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0091.html", + "product_code":"dli", + "code":"149", + "des":"You can view SQL job logs for routine O&M.Obtain the ID of the DLI job executed on the DataArts Studio console.Job IDOn the DLI console, choose Job Management > SQL Jobs.", + "doc_type":"usermanual", + "kw":"How Do I View DLI SQL Logs?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I View DLI SQL Logs?", + "githuburl":"" + }, + { + "uri":"dli_03_0116.html", + "product_code":"dli", + "code":"150", + "des":"You can view the job execution records when a job is running.Log in to the DLI management console.In the navigation pane on the left, choose Job Management > SQL Jobs.Ent", + "doc_type":"usermanual", + "kw":"How Do I View SQL Execution Records?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I View SQL Execution Records?", + "githuburl":"" + }, + { + "uri":"dli_03_0093.html", + "product_code":"dli", + "code":"151", + "des":"If the execution of an SQL statement takes a long time, you need to access the Spark UI to check the execution status.If data skew occurs, the running time of a stage exc", + "doc_type":"usermanual", + "kw":"How Do I Eliminate Data Skew by Configuring AE Parameters?,Problems Related to SQL Jobs,User Guide", + "title":"How Do I Eliminate Data Skew by Configuring AE Parameters?", + "githuburl":"" + }, + { + "uri":"dli_03_0184.html", + "product_code":"dli", + "code":"152", + "des":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", + "doc_type":"usermanual", + "kw":"What Can I Do If a Table Cannot Be Queried on the DLI Console?,Problems Related to SQL Jobs,User Gui", + "title":"What Can I Do If a Table Cannot Be Queried on the DLI Console?", + "githuburl":"" + }, + { + "uri":"dli_03_0013.html", + "product_code":"dli", + "code":"153", + "des":"A high compression ratio of OBS tables in the Parquet or ORC format (for example, a compression ratio of 5 or higher compared with text compression) will lead to large da", + "doc_type":"usermanual", + "kw":"The Compression Ratio of OBS Tables Is Too High,Problems Related to SQL Jobs,User Guide", + "title":"The Compression Ratio of OBS Tables Is Too High", + "githuburl":"" + }, + { + "uri":"dli_03_0009.html", + "product_code":"dli", + "code":"154", + "des":"DLI supports only UTF-8-encoded texts. Ensure that data is encoded using UTF-8 during table creation and import.", + "doc_type":"usermanual", + "kw":"How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?,Problems Related to SQL J", + "title":"How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?", + "githuburl":"" + }, + { + "uri":"dli_03_0175.html", + "product_code":"dli", + "code":"155", + "des":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", + "doc_type":"usermanual", + "kw":"Do I Need to Grant Table Permissions to a User and Project After I Delete a Table and Create One wit", + "title":"Do I Need to Grant Table Permissions to a User and Project After I Delete a Table and Create One with the Same Name?", + "githuburl":"" + }, + { + "uri":"dli_03_0177.html", + "product_code":"dli", + "code":"156", + "des":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", + "doc_type":"usermanual", + "kw":"What Should I Do If Table Data Fails to Be Queried After Data Is Imported to a DLI Partitioned Table", + "title":"What Should I Do If Table Data Fails to Be Queried After Data Is Imported to a DLI Partitioned Table Because the File to Be Imported Does Not Contain Data in the Partitioning Column?", + "githuburl":"" + }, + { + "uri":"dli_03_0181.html", + "product_code":"dli", + "code":"157", + "des":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", + "doc_type":"usermanual", + "kw":"How Do I Fix the Data Error Caused by CRLF Characters in a Field of the OBS File Used to Create an E", + "title":"How Do I Fix the Data Error Caused by CRLF Characters in a Field of the OBS File Used to Create an External OBS Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0182.html", + "product_code":"dli", + "code":"158", + "des":"A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.When a Spark SQL job has join operations on sma", + "doc_type":"usermanual", + "kw":"What Should I Do If a SQL Job That Has Join Operations Stays in the Running State?,Problems Related ", + "title":"What Should I Do If a SQL Job That Has Join Operations Stays in the Running State?", + "githuburl":"" + }, + { + "uri":"dli_03_0187.html", + "product_code":"dli", + "code":"159", + "des":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", + "doc_type":"usermanual", + "kw":"The on Clause Is Not Added When Tables Are Joined. Cartesian Product Query Causes High Resource Usag", + "title":"The on Clause Is Not Added When Tables Are Joined. Cartesian Product Query Causes High Resource Usage of the Queue, and the Job Fails to Be Executed", + "githuburl":"" + }, + { + "uri":"dli_03_0190.html", + "product_code":"dli", + "code":"160", + "des":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", + "doc_type":"usermanual", + "kw":"What Should I Do If Manually Added Data in a Partition of an OBS Table Cannot Be Queried?,Problems R", + "title":"What Should I Do If Manually Added Data in a Partition of an OBS Table Cannot Be Queried?", + "githuburl":"" + }, + { + "uri":"dli_03_0212.html", + "product_code":"dli", + "code":"161", + "des":"To dynamically overwrite the specified partitioned data in the DataSource table, set dli.sql.dynamicPartitionOverwrite.enabled to true and then run the insert overwrite s", + "doc_type":"usermanual", + "kw":"Why Is All Data Overwritten When insert overwrite Is Used to Overwrite Partitioned Table?,Problems R", + "title":"Why Is All Data Overwritten When insert overwrite Is Used to Overwrite Partitioned Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0213.html", + "product_code":"dli", + "code":"162", + "des":"The possible causes and solutions are as follows:After you purchase a DLI queue and submit a SQL job for the first time, wait for 5 to 10 minutes. After the cluster is st", + "doc_type":"usermanual", + "kw":"Why Is a SQL Job Stuck in the Submitting State?,Problems Related to SQL Jobs,User Guide", + "title":"Why Is a SQL Job Stuck in the Submitting State?", + "githuburl":"" + }, + { + "uri":"dli_03_0214.html", + "product_code":"dli", + "code":"163", + "des":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", + "doc_type":"usermanual", + "kw":"Why Is the create_date Field in the RDS Table Is a Timestamp in the DLI query result?,Problems Relat", + "title":"Why Is the create_date Field in the RDS Table Is a Timestamp in the DLI query result?", + "githuburl":"" + }, + { + "uri":"dli_03_0215.html", + "product_code":"dli", + "code":"164", + "des":"If the table name is changed immediately after SQL statements are executed, the data size of the table may be incorrect.If you need to change the table name, change it 5 ", + "doc_type":"usermanual", + "kw":"What Can I Do If datasize Cannot Be Changed After the Table Name Is Changed in a Finished SQL Job?,P", + "title":"What Can I Do If datasize Cannot Be Changed After the Table Name Is Changed in a Finished SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0231.html", + "product_code":"dli", + "code":"165", + "des":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", + "doc_type":"usermanual", + "kw":"Why Is the Data Volume Changes When Data Is Imported from DLI to OBS?,Problems Related to SQL Jobs,U", + "title":"Why Is the Data Volume Changes When Data Is Imported from DLI to OBS?", + "githuburl":"" + }, + { + "uri":"dli_03_0021.html", + "product_code":"dli", + "code":"166", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Problems Related to Spark Jobs", + "title":"Problems Related to Spark Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0201.html", + "product_code":"dli", + "code":"167", + "des":"DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.The Spark SQL syntax does no", + "doc_type":"usermanual", + "kw":"Spark Jobs,Problems Related to Spark Jobs,User Guide", + "title":"Spark Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0107.html", + "product_code":"dli", + "code":"168", + "des":"To use Spark to write data into a DLI table, configure the following parameters:fs.obs.access.keyfs.obs.secret.keyfs.obs.implfs.obs.endpointThe following is an example:", + "doc_type":"usermanual", + "kw":"How Do I Use Spark to Write Data into a DLI Table?,Problems Related to Spark Jobs,User Guide", + "title":"How Do I Use Spark to Write Data into a DLI Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0017.html", + "product_code":"dli", + "code":"169", + "des":"If the AK and SK are obtained, set the parameters as follows:Create SparkContext using codeval sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.ac", + "doc_type":"usermanual", + "kw":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?,Problems Related to Spark Jobs,User Guid", + "title":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0102.html", + "product_code":"dli", + "code":"170", + "des":"Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parame", + "doc_type":"usermanual", + "kw":"How Do I View the Resource Usage of DLI Spark Jobs?,Problems Related to Spark Jobs,User Guide", + "title":"How Do I View the Resource Usage of DLI Spark Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0076.html", + "product_code":"dli", + "code":"171", + "des":"If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management p", + "doc_type":"usermanual", + "kw":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the S", + "title":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?", + "githuburl":"" + }, + { + "uri":"dli_03_0082.html", + "product_code":"dli", + "code":"172", + "des":"DLI natively supports PySpark.For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are pac", + "doc_type":"usermanual", + "kw":"How Do I Run a Complex PySpark Program in DLI?,Problems Related to Spark Jobs,User Guide", + "title":"How Do I Run a Complex PySpark Program in DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0127.html", + "product_code":"dli", + "code":"173", + "des":"You can use DLI Spark jobs to access data in the MySQL database using either of the following methods:Solution 1: Purchase a pay-per-use queue, create an enhanced datasou", + "doc_type":"usermanual", + "kw":"How Does a Spark Job Access a MySQL Database?,Problems Related to Spark Jobs,User Guide", + "title":"How Does a Spark Job Access a MySQL Database?", + "githuburl":"" + }, + { + "uri":"dli_03_0068.html", + "product_code":"dli", + "code":"174", + "des":"When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.To solve this problem, you can config", + "doc_type":"usermanual", + "kw":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", + "title":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", + "githuburl":"" + }, + { + "uri":"dli_03_0118.html", + "product_code":"dli", + "code":"175", + "des":"You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get(\"Name of the uploaded file\").The file path in the Driver is different fro", + "doc_type":"usermanual", + "kw":"How Do I Read Uploaded Files for a Spark Jar Job?,Problems Related to Spark Jobs,User Guide", + "title":"How Do I Read Uploaded Files for a Spark Jar Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0023.html", + "product_code":"dli", + "code":"176", + "des":"The Spark 2.3 has changed the behavior of the internal interface Logging. If the user code directly inherits the Logging and the earlier version Spark is used during comp", + "doc_type":"usermanual", + "kw":"What Can I Do When Receiving java.lang.AbstractMethodError in the Spark Job?,Problems Related to Spa", + "title":"What Can I Do When Receiving java.lang.AbstractMethodError in the Spark Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0156.html", + "product_code":"dli", + "code":"177", + "des":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data. For details, see How Do I Set the AK/SK for a Qu", + "doc_type":"usermanual", + "kw":"What Should I Do If Errors \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Are Reported When a S", + "title":"What Should I Do If Errors \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Are Reported When a Spark Job Accesses OBS Data", + "githuburl":"" + }, + { + "uri":"dli_03_0164.html", + "product_code":"dli", + "code":"178", + "des":"Check whether the OBS bucket is used to store DLI logs on the Global Configuration > Job Configurations page. The job log bucket cannot be used for other purpose.", + "doc_type":"usermanual", + "kw":"Why is Error \"verifyBucketExists on XXXX: status [403]\" Is Reported When I Run a Spark Job with a Us", + "title":"Why is Error \"verifyBucketExists on XXXX: status [403]\" Is Reported When I Run a Spark Job with a User that has the Permission to Access the OBS Bucket?", + "githuburl":"" + }, + { + "uri":"dli_03_0157.html", + "product_code":"dli", + "code":"179", + "des":"When a Spark job accesses a large amount of data, for example, accessing data in a GaussDB(DWS) database, you are advised to set the number of concurrent tasks and enable", + "doc_type":"usermanual", + "kw":"What Should I Do if a Running Timeout Error Is Reported When a Spark Job Runs a Large Amount of Data", + "title":"What Should I Do if a Running Timeout Error Is Reported When a Spark Job Runs a Large Amount of Data?", + "githuburl":"" + }, + { + "uri":"dli_03_0188.html", + "product_code":"dli", + "code":"180", + "des":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.", + "doc_type":"usermanual", + "kw":"What Should I Do If a Spark job Used to Access Files in SFTP Fails and the Log Shows that the File D", + "title":"What Should I Do If a Spark job Used to Access Files in SFTP Fails and the Log Shows that the File Directory Is Abnormal?", + "githuburl":"" + }, + { + "uri":"dli_03_0192.html", + "product_code":"dli", + "code":"181", + "des":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", + "doc_type":"usermanual", + "kw":"What Should I Do If a Job Fails Because the Job User Does Not Have the Database and Table Permission", + "title":"What Should I Do If a Job Fails Because the Job User Does Not Have the Database and Table Permission?", + "githuburl":"" + }, + { + "uri":"dli_03_0077.html", + "product_code":"dli", + "code":"182", + "des":"If the specified Python environment cannot be found after a Python3 package is added, the current compute cluster environment is Python2 by default.You can set spark.yarn", + "doc_type":"usermanual", + "kw":"What Can I Do If the Specified Python Environment Cannot Be Found After a Python Package Is Added?,P", + "title":"What Can I Do If the Specified Python Environment Cannot Be Found After a Python Package Is Added?", + "githuburl":"" + }, + { + "uri":"dli_03_0220.html", + "product_code":"dli", + "code":"183", + "des":"The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.To view the remaining CUs of a queue, perform the following steps:Check the C", + "doc_type":"usermanual", + "kw":"Why Is a Spark Jar Job Stuck in the Submitting State?,Problems Related to Spark Jobs,User Guide", + "title":"Why Is a Spark Jar Job Stuck in the Submitting State?", + "githuburl":"" + }, + { + "uri":"dli_03_0001.html", + "product_code":"dli", + "code":"184", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Product Consultation", + "title":"Product Consultation", + "githuburl":"" + }, + { + "uri":"dli_03_0002.html", + "product_code":"dli", + "code":"185", + "des":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark, and Flink ecosystems. It frees you from managing any serv", + "doc_type":"usermanual", + "kw":"What Is DLI?,Product Consultation,User Guide", + "title":"What Is DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0025.html", + "product_code":"dli", + "code":"186", + "des":"DLI supports the following data formats:ParquetCSVORCJsonAvro", + "doc_type":"usermanual", + "kw":"Which Data Formats Does DLI Support?,Product Consultation,User Guide", + "title":"Which Data Formats Does DLI Support?", + "githuburl":"" + }, + { + "uri":"dli_03_0115.html", + "product_code":"dli", + "code":"187", + "des":"The Spark component of DLI is a fully managed service. You can only use the DLI Spark through its APIs. .The Spark component of MRS is built on the VM in an MRS cluster. ", + "doc_type":"usermanual", + "kw":"What Are the Differences Between MRS Spark and DLI Spark?,Product Consultation,User Guide", + "title":"What Are the Differences Between MRS Spark and DLI Spark?", + "githuburl":"" + }, + { + "uri":"dli_03_0029.html", + "product_code":"dli", + "code":"188", + "des":"DLI data can be stored in either of the following:OBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-base", + "doc_type":"usermanual", + "kw":"Where Can DLI Data Be Stored?,Product Consultation,User Guide", + "title":"Where Can DLI Data Be Stored?", + "githuburl":"" + }, + { + "uri":"dli_03_0117.html", + "product_code":"dli", + "code":"189", + "des":"DLI tables store data within the DLI service, and you do not need to know the data storage path.OBS tables store data in your OBS buckets, and you need to manage the sour", + "doc_type":"usermanual", + "kw":"What Are the Differences Between DLI Tables and OBS Tables?,Product Consultation,User Guide", + "title":"What Are the Differences Between DLI Tables and OBS Tables?", + "githuburl":"" + }, + { + "uri":"dli_03_0010.html", + "product_code":"dli", + "code":"190", + "des":"Currently, DLI supports analysis only on the data uploaded to the cloud. In scenarios where regular (for example, on a per day basis) one-off analysis on incremental data", + "doc_type":"usermanual", + "kw":"How Can I Use DLI If Data Is Not Uploaded to OBS?,Product Consultation,User Guide", + "title":"How Can I Use DLI If Data Is Not Uploaded to OBS?", + "githuburl":"" + }, + { + "uri":"dli_03_0129.html", + "product_code":"dli", + "code":"191", + "des":"Data in the OBS bucket shared by IAM users under the same account can be imported. You cannot import data in the OBS bucket shared with other IAM account.", + "doc_type":"usermanual", + "kw":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?,Product Consultation,User Guide", + "title":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0264.html", + "product_code":"dli", + "code":"192", + "des":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", + "doc_type":"usermanual", + "kw":"What Should I do If the System Failed to Create a Database and {\"error_code\":\"DLI.1028\";\"error_msg\":", + "title":"What Should I do If the System Failed to Create a Database and {\"error_code\":\"DLI.1028\";\"error_msg\":\"Already reached the maximum quota of databases:XXX\" Is Displayed Indicating that the Quota Is Insufficient?", + "githuburl":"" + }, + { + "uri":"dli_03_0263.html", + "product_code":"dli", + "code":"193", + "des":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", + "doc_type":"usermanual", + "kw":"Can a Member Account Use Global Variables Created by Other Member Accounts?,Product Consultation,Use", + "title":"Can a Member Account Use Global Variables Created by Other Member Accounts?", + "githuburl":"" + }, + { + "uri":"dli_03_0126.html", + "product_code":"dli", + "code":"194", + "des":"If you are suggested to perform following operations to run a large number of DLI jobs:Group the DLI jobs by type, and run each group on a queue.Alternatively, create IAM", + "doc_type":"usermanual", + "kw":"How Do I Manage Tens of Thousands of Jobs Running on DLI?,Product Consultation,User Guide", + "title":"How Do I Manage Tens of Thousands of Jobs Running on DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0162.html", + "product_code":"dli", + "code":"195", + "des":"The field names of tables that have been created cannot be changed.You can create a table, define new table fields, and migrate data from the old table to the new one.", + "doc_type":"usermanual", + "kw":"How Do I Change the Name of a Field in a Created Table?,Product Consultation,User Guide", + "title":"How Do I Change the Name of a Field in a Created Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0260.html", + "product_code":"dli", + "code":"196", + "des":"No. The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.", + "doc_type":"usermanual", + "kw":"Does DLI Have the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?,Product Consultatio", + "title":"Does DLI Have the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?", + "githuburl":"" + }, + { + "uri":"dli_03_0053.html", + "product_code":"dli", + "code":"197", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Quota", + "title":"Quota", + "githuburl":"" + }, + { + "uri":"dli_03_0031.html", + "product_code":"dli", + "code":"198", + "des":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", + "doc_type":"usermanual", + "kw":"How Do I View My Quotas?,Quota,User Guide", + "title":"How Do I View My Quotas?", + "githuburl":"" + }, + { + "uri":"dli_03_0032.html", + "product_code":"dli", + "code":"199", + "des":"The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your applica", + "doc_type":"usermanual", + "kw":"How Do I Increase a Quota?,Quota,User Guide", + "title":"How Do I Increase a Quota?", + "githuburl":"" + }, + { + "uri":"dli_03_0054.html", + "product_code":"dli", + "code":"200", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Permission", + "title":"Permission", + "githuburl":"" + }, + { + "uri":"dli_03_0100.html", + "product_code":"dli", + "code":"201", + "des":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", + "doc_type":"usermanual", + "kw":"How Do I Manage Fine-Grained DLI Permissions?,Permission,User Guide", + "title":"How Do I Manage Fine-Grained DLI Permissions?", + "githuburl":"" + }, + { + "uri":"dli_03_0008.html", + "product_code":"dli", + "code":"202", + "des":"You cannot perform permission-related operations on the partition column of a partition table. However, when you grant the permission of any non-partition column in a par", + "doc_type":"usermanual", + "kw":"What Is Column Permission Granting of a DLI Partition Table?,Permission,User Guide", + "title":"What Is Column Permission Granting of a DLI Partition Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0101.html", + "product_code":"dli", + "code":"203", + "des":"You can set actions and resources of different levels for various scenarios.Define an action.The format is Service name:Resource type:Action. You can use wildcard *. The ", + "doc_type":"usermanual", + "kw":"How Do I Create a Custom Policy?,Permission,User Guide", + "title":"How Do I Create a Custom Policy?", + "githuburl":"" + }, + { + "uri":"dli_03_0225.html", + "product_code":"dli", + "code":"204", + "des":"You can isolate queues allocated to different users by setting permissions to ensure data query performance.", + "doc_type":"usermanual", + "kw":"How Do I Manage Queue Permissions?,Permission,User Guide", + "title":"How Do I Manage Queue Permissions?", + "githuburl":"" + }, + { + "uri":"dli_03_0140.html", + "product_code":"dli", + "code":"205", + "des":"When you submit a job, a message is displayed indicating that the job fails to be submitted due to insufficient permission caused by arrears. In this case, you need to ch", + "doc_type":"usermanual", + "kw":"What Should I Do When My Account Does Not Have Permission Due To Arrears?,Permission,User Guide", + "title":"What Should I Do When My Account Does Not Have Permission Due To Arrears?", + "githuburl":"" + }, + { + "uri":"dli_03_0195.html", + "product_code":"dli", + "code":"206", + "des":"When the user update an existing program package, the following error information is displayed:\"error_code\"*CLI.0003\",\"error_msg\":\"Permission denied for resource 'resourc", + "doc_type":"usermanual", + "kw":"What Should I Do If the Permission Is Required to Update a Program Package?,Permission,User Guide", + "title":"What Should I Do If the Permission Is Required to Update a Program Package?", + "githuburl":"" + }, + { + "uri":"dli_03_0227.html", + "product_code":"dli", + "code":"207", + "des":"When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.Error information: DLI.00", + "doc_type":"usermanual", + "kw":"What Should I Do If \"DLI.0003: Permission denied for resource...\" Is Reported When I Run a SQL State", + "title":"What Should I Do If \"DLI.0003: Permission denied for resource...\" Is Reported When I Run a SQL Statement?", + "githuburl":"" + }, + { + "uri":"dli_03_0228.html", + "product_code":"dli", + "code":"208", + "des":"The table permission has been granted and verified. However, after a period of time, an error is reported indicating that the table query fails.There are two possible rea", + "doc_type":"usermanual", + "kw":"What Should I Do If the Table Permission Has Been Granted But the Table Still Cannot Be Queried?,Per", + "title":"What Should I Do If the Table Permission Has Been Granted But the Table Still Cannot Be Queried?", + "githuburl":"" + }, + { + "uri":"dli_03_0049.html", + "product_code":"dli", + "code":"209", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Queue", + "title":"Queue", + "githuburl":"" + }, + { + "uri":"dli_03_0109.html", + "product_code":"dli", + "code":"210", + "des":"Currently, you are not allowed to modify the description of a created queue. You can add the description when purchasing the queue.", + "doc_type":"usermanual", + "kw":"Does the Description of a DLI Queue Can Be Modified?,Queue,User Guide", + "title":"Does the Description of a DLI Queue Can Be Modified?", + "githuburl":"" + }, + { + "uri":"dli_03_0166.html", + "product_code":"dli", + "code":"211", + "des":"Deleting a queue does not cause table data loss in your database.", + "doc_type":"usermanual", + "kw":"Will Table Data in My Database Be Lost If I Delete a Queue?,Queue,User Guide", + "title":"Will Table Data in My Database Be Lost If I Delete a Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0170.html", + "product_code":"dli", + "code":"212", + "des":"You need to develop a mechanism to retry failed jobs. When a faulty queue is recovered, your application tries to submit the failed jobs to the queue again.", + "doc_type":"usermanual", + "kw":"How Does DLI Ensure the Reliability of Spark Jobs When a Queue Is Abnormal?,Queue,User Guide", + "title":"How Does DLI Ensure the Reliability of Spark Jobs When a Queue Is Abnormal?", + "githuburl":"" + }, + { + "uri":"dli_03_0098.html", + "product_code":"dli", + "code":"213", + "des":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", + "doc_type":"usermanual", + "kw":"How Do I Monitor Queue Exceptions?,Queue,User Guide", + "title":"How Do I Monitor Queue Exceptions?", + "githuburl":"" + }, + { + "uri":"dli_03_0095.html", + "product_code":"dli", + "code":"214", + "des":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", + "doc_type":"usermanual", + "kw":"How Do I View DLI Queue Load?,Queue,User Guide", + "title":"How Do I View DLI Queue Load?", + "githuburl":"" + }, + { + "uri":"dli_03_0183.html", + "product_code":"dli", + "code":"215", + "des":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", + "doc_type":"usermanual", + "kw":"How Do I Determine Whether There Are Too Many Jobs in the Current Queue?,Queue,User Guide", + "title":"How Do I Determine Whether There Are Too Many Jobs in the Current Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0065.html", + "product_code":"dli", + "code":"216", + "des":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", + "doc_type":"usermanual", + "kw":"How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?,Queue,User Guide", + "title":"How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0193.html", + "product_code":"dli", + "code":"217", + "des":"DLI queues do not use resources or bandwidth when no job is running. In this case, the running status of DLI queues is not displayed on CES.", + "doc_type":"usermanual", + "kw":"Why Cannot I View the Resource Running Status of DLI Queues on Cloud Eye?,Queue,User Guide", + "title":"Why Cannot I View the Resource Running Status of DLI Queues on Cloud Eye?", + "githuburl":"" + }, + { + "uri":"dli_03_0088.html", + "product_code":"dli", + "code":"218", + "des":"In DLI, 64 CU = 64 cores and 256 GB memory.In a Spark job, if the driver occupies 4 cores and 16 GB memory, the executor can occupy 60 cores and 240 GB memory.", + "doc_type":"usermanual", + "kw":"How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?,Queue,User Guid", + "title":"How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?", + "githuburl":"" + }, + { + "uri":"dli_03_0159.html", + "product_code":"dli", + "code":"219", + "des":"Queue plans create failed. The plan xxx target cu is out of quota is displayed when you create a scheduled scaling task.The CU quota of the current account is insufficien", + "doc_type":"usermanual", + "kw":"What Should I Do If \"Queue plans create failed. The plan xxx target cu is out of quota\" Is Displayed", + "title":"What Should I Do If \"Queue plans create failed. The plan xxx target cu is out of quota\" Is Displayed When I Schedule CU Changes?", + "githuburl":"" + }, + { + "uri":"dli_03_0171.html", + "product_code":"dli", + "code":"220", + "des":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", + "doc_type":"usermanual", + "kw":"What Should I Do If SQL Statements Fails to be Executed on the Default Queue, and a Timeout Exceptio", + "title":"What Should I Do If SQL Statements Fails to be Executed on the Default Queue, and a Timeout Exception is Reported?", + "githuburl":"" + }, + { + "uri":"dli_03_0022.html", + "product_code":"dli", + "code":"221", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Datasource Connections", + "title":"Datasource Connections", + "githuburl":"" + }, + { + "uri":"dli_03_0128.html", + "product_code":"dli", + "code":"222", + "des":"You need to create a VPC peering connection to enable network connectivity. Take MRS as an example. If DLI and MRS clusters are in the same VPC, and the security group is", + "doc_type":"usermanual", + "kw":"Why Do I Need to Create a VPC Peering Connection for an Enhanced Datasource Connection?,Datasource C", + "title":"Why Do I Need to Create a VPC Peering Connection for an Enhanced Datasource Connection?", + "githuburl":"" + }, + { + "uri":"dli_03_0237.html", + "product_code":"dli", + "code":"223", + "des":"An enhanced datasource connection failed to pass the network connectivity test. Datasource connection cannot be bound to a queue. The following error information is displ", + "doc_type":"usermanual", + "kw":"Failed to Bind a Queue to an Enhanced Datasource Connection,Datasource Connections,User Guide", + "title":"Failed to Bind a Queue to an Enhanced Datasource Connection", + "githuburl":"" + }, + { + "uri":"dli_03_0238.html", + "product_code":"dli", + "code":"224", + "des":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", + "doc_type":"usermanual", + "kw":"DLI Failed to Connect to GaussDB(DWS) Through an Enhanced Datasource Connection,Datasource Connectio", + "title":"DLI Failed to Connect to GaussDB(DWS) Through an Enhanced Datasource Connection", + "githuburl":"" + }, + { + "uri":"dli_03_0179.html", + "product_code":"dli", + "code":"225", + "des":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", + "doc_type":"usermanual", + "kw":"What Can I Do If the Datasource Connection is Created But the Network Connectivity Test Fails?,Datas", + "title":"What Can I Do If the Datasource Connection is Created But the Network Connectivity Test Fails?", + "githuburl":"" + }, + { + "uri":"dli_03_0186.html", + "product_code":"dli", + "code":"226", + "des":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", + "doc_type":"usermanual", + "kw":"How Do I Configure the Network Between a DLI Queue and a Data Source?,Datasource Connections,User Gu", + "title":"How Do I Configure the Network Between a DLI Queue and a Data Source?", + "githuburl":"" + }, + { + "uri":"dli_03_0257.html", + "product_code":"dli", + "code":"227", + "des":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", + "doc_type":"usermanual", + "kw":"What Can I Do If a Datasource Connection Is Stuck in Creating State When I Try to Bind a Queue to It", + "title":"What Can I Do If a Datasource Connection Is Stuck in Creating State When I Try to Bind a Queue to It?", + "githuburl":"" + }, + { + "uri":"dli_03_0258.html", + "product_code":"dli", + "code":"228", + "des":"Enhanced datasource connections support only yearly/monthly and pay-per-use queues.You can bind a datasource connection only when you select Dedicated Resource Mode when ", + "doc_type":"usermanual", + "kw":"How Do I Bind a Datasource Connection to a Pay-per-Use Queue?,Datasource Connections,User Guide", + "title":"How Do I Bind a Datasource Connection to a Pay-per-Use Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0259.html", + "product_code":"dli", + "code":"229", + "des":"DLI enhanced datasource connection uses VPC peering to directly connect the VPC networks of the desired data sources for point-to-point data exchanges.", + "doc_type":"usermanual", + "kw":"How Do I Connect DLI to Data Sources?,Datasource Connections,User Guide", + "title":"How Do I Connect DLI to Data Sources?", + "githuburl":"" + }, + { + "uri":"dli_03_0011.html", + "product_code":"dli", + "code":"230", + "des":"To perform query on data stored on services rather than DLI, perform the following steps:Assume that the data to be queried is stored on multiple services (for example, O", + "doc_type":"usermanual", + "kw":"How Can I Perform Query on Data Stored on Services Rather Than DLI?,Datasource Connections,User Guid", + "title":"How Can I Perform Query on Data Stored on Services Rather Than DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0085.html", + "product_code":"dli", + "code":"231", + "des":"Connect VPCs in different regions.Create an enhanced datasource connection on DLI and bind it to a queue.Add a DLI route.", + "doc_type":"usermanual", + "kw":"How Can I Access Data Across Regions?,Datasource Connections,User Guide", + "title":"How Can I Access Data Across Regions?", + "githuburl":"" + }, + { + "uri":"dli_03_0028.html", + "product_code":"dli", + "code":"232", + "des":"When data is inserted into DLI, set the ID field to NULL.", + "doc_type":"usermanual", + "kw":"How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS", + "title":"How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS Table When Creating a DLI and Associating It with the RDS Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0047.html", + "product_code":"dli", + "code":"233", + "des":"Possible CausesThe network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.The network connec", + "doc_type":"usermanual", + "kw":"Why Is the Error Message \"communication link failure\" Displayed When I Use a Newly Activated Datasou", + "title":"Why Is the Error Message \"communication link failure\" Displayed When I Use a Newly Activated Datasource Connection?", + "githuburl":"" + }, + { + "uri":"dli_03_0080.html", + "product_code":"dli", + "code":"234", + "des":"The cluster host information is not added to the datasource connection. As a result, the KRB authentication fails, the connection times out, and no error is recorded in l", + "doc_type":"usermanual", + "kw":"Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs,Dataso", + "title":"Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs", + "githuburl":"" + }, + { + "uri":"dli_03_0111.html", + "product_code":"dli", + "code":"235", + "des":"When you create a VPC peering connection for the datasource connection, the following error information is displayed:Before you create a datasource connection, check whet", + "doc_type":"usermanual", + "kw":"What Should I Do If the Subnet Cannot Be Found When I Create a DLI Datasource Connection?,Datasource", + "title":"What Should I Do If the Subnet Cannot Be Found When I Create a DLI Datasource Connection?", + "githuburl":"" + }, + { + "uri":"dli_03_0239.html", + "product_code":"dli", + "code":"236", + "des":"A cross-source RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incor", + "doc_type":"usermanual", + "kw":"Error Message \"Incorrect string value\" Is Displayed When insert overwrite Is Executed on a Cross-Sou", + "title":"Error Message \"Incorrect string value\" Is Displayed When insert overwrite Is Executed on a Cross-Source RDS Table", + "githuburl":"" + }, + { + "uri":"dli_03_0250.html", + "product_code":"dli", + "code":"237", + "des":"The system failed to create a cross-source RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre", + "doc_type":"usermanual", + "kw":"Null Pointer Error Is Displayed When the System Creates a Cross-Source RDS Table,Datasource Connecti", + "title":"Null Pointer Error Is Displayed When the System Creates a Cross-Source RDS Table", + "githuburl":"" + }, + { + "uri":"dli_03_0251.html", + "product_code":"dli", + "code":"238", + "des":"The system failed to execute insert overwrite on the cross-source GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was display", + "doc_type":"usermanual", + "kw":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed Wh", + "title":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed When the System Executes insert overwrite on a Cross-Source GaussDB(DWS) Table", + "githuburl":"" + }, + { + "uri":"dli_03_0252.html", + "product_code":"dli", + "code":"239", + "des":"A cross-source table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. ", + "doc_type":"usermanual", + "kw":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Cross", + "title":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Cross-Source Table", + "githuburl":"" + }, + { + "uri":"dli_03_0253.html", + "product_code":"dli", + "code":"240", + "des":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", + "doc_type":"usermanual", + "kw":"A Null Value Is Written Into a Non-Null Field When a DLI Datasource Connection Is Used to Connect to", + "title":"A Null Value Is Written Into a Non-Null Field When a DLI Datasource Connection Is Used to Connect to a GaussDB(DWS) Table", + "githuburl":"" + }, + { + "uri":"dli_03_0254.html", + "product_code":"dli", + "code":"241", + "des":"A cross-source GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job execu", + "doc_type":"usermanual", + "kw":"An Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated,Datasource C", + "title":"An Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated", + "githuburl":"" + }, + { + "uri":"dli_01_00006.html", + "product_code":"dli", + "code":"242", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Change History,User Guide", + "title":"Change History", + "githuburl":"" + } +] \ No newline at end of file diff --git a/docs/dli/umn/CLASS.TXT.json b/docs/dli/umn/CLASS.TXT.json new file mode 100644 index 00000000..c1a7318b --- /dev/null +++ b/docs/dli/umn/CLASS.TXT.json @@ -0,0 +1,2180 @@ +[ + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Service Overview", + "uri":"dli_01_0538.html", + "doc_type":"usermanual", + "p_code":"", + "code":"1" + }, + { + "desc":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing an", + "product_code":"dli", + "title":"DLI Introduction", + "uri":"dli_01_0378.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"2" + }, + { + "desc":"You do not need a background in big data to use DLI for data analysis. You only need to know SQL, and you are good to go. The SQL syntax is fully compatible with the stan", + "product_code":"dli", + "title":"Advantages", + "uri":"dli_07_0007.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"3" + }, + { + "desc":"DLI is applicable to large-scale log analysis, federated analysis of heterogeneous data sources, and big data ETL processing.Gaming operation data analysisDifferent depar", + "product_code":"dli", + "title":"Application Scenarios", + "uri":"dli_07_0002.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"4" + }, + { + "desc":"When using DLI, you may need to make adjustments according to the following restrictions:Recommended browsers for logging in to DLI:Google Chrome 43.0 or laterMozilla Fir", + "product_code":"dli", + "title":"Constraints and Limitations on Using DLI", + "uri":"dli_07_0005.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"5" + }, + { + "desc":"If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IA", + "product_code":"dli", + "title":"Permissions Management", + "uri":"dli_07_0006.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"6" + }, + { + "desc":"DLI allows multiple organizations, departments, or applications to share resources. A logical entity, also called a tenant, is provided to use diverse resources and servi", + "product_code":"dli", + "title":"Basic Concepts", + "uri":"dli_07_0003.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"7" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Getting Started", + "uri":"dli_01_0220.html", + "doc_type":"usermanual", + "p_code":"", + "code":"8" + }, + { + "desc":"You can use DLI to submit a Spark SQL job to query data. The general procedure is as follows:Step 1: Logging in to the Cloud PlatformStep 2: Uploading Data to OBSStep 3: ", + "product_code":"dli", + "title":"Creating and Submitting a Spark SQL Job", + "uri":"dli_01_0002.html", + "doc_type":"usermanual", + "p_code":"8", + "code":"9" + }, + { + "desc":"To facilitate SQL operations, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not need to wr", + "product_code":"dli", + "title":"Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template", + "uri":"dli_01_0512.html", + "doc_type":"usermanual", + "p_code":"8", + "code":"10" + }, + { + "desc":"You can use DLI to submit Spark jobs for real-time computing. The general procedure is as follows:Step 1: Logging in to the CloudStep 2: Uploading Data to OBSStep 3: Logg", + "product_code":"dli", + "title":"Creating and Submitting a Spark Jar Job", + "uri":"dli_01_0375.html", + "doc_type":"usermanual", + "p_code":"8", + "code":"11" + }, + { + "desc":"This section describes how to submit a Flink OpenSource SQL job on the DLI console for real-time computing. The general procedure is as follows:Step 1: Log In to the Clou", + "product_code":"dli", + "title":"Creating and Submitting a Flink OpenSource SQL Job", + "uri":"dli_01_0531.html", + "doc_type":"usermanual", + "p_code":"8", + "code":"12" + }, + { + "desc":"Log in to DLI and go to the Overview page.The following table describes the functional areas of the Overview page.", + "product_code":"dli", + "title":"DLI Console Overview", + "uri":"dli_01_0377.html", + "doc_type":"usermanual", + "p_code":"", + "code":"13" + }, + { + "desc":"You can use SQL statements in the SQL job editor to execute data query. DLI supports SQL 2003 and complies with Spark SQL.On the Overview page, click SQL Editor in the na", + "product_code":"dli", + "title":"SQL Editor", + "uri":"dli_01_0320.html", + "doc_type":"usermanual", + "p_code":"", + "code":"14" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Job Management", + "uri":"dli_01_0001.html", + "doc_type":"usermanual", + "p_code":"", + "code":"15" + }, + { + "desc":"SQL jobs allow you to execute SQL statements entered in the SQL job editing window, import data, and export data.SQL job management provides the following functions:Searc", + "product_code":"dli", + "title":"SQL Job Management", + "uri":"dli_01_0017.html", + "doc_type":"usermanual", + "p_code":"15", + "code":"16" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Flink Job Management", + "uri":"dli_01_0389.html", + "doc_type":"usermanual", + "p_code":"15", + "code":"17" + }, + { + "desc":"On the Job Management page of Flink jobs, you can submit a Flink job. Currently, the following job types are supported:Flink SQL uses SQL statements to define jobs and ca", + "product_code":"dli", + "title":"Overview", + "uri":"dli_01_0403.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"18" + }, + { + "desc":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", + "product_code":"dli", + "title":"Managing Flink Job Permissions", + "uri":"dli_01_0479.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"19" + }, + { + "desc":"To create a Flink job, you need to enter the data source and data output channel, that is, source and sink. To use another service as the source or sink stream, you need ", + "product_code":"dli", + "title":"Preparing Flink Job Data", + "uri":"dli_01_0454.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"20" + }, + { + "desc":"This section describes how to create a Flink OpenSource SQL job. DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the", + "product_code":"dli", + "title":"(Recommended) Creating a Flink OpenSource SQL Job", + "uri":"dli_01_0498.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"21" + }, + { + "desc":"This section describes how to create a Flink SQL job. You can use Flink SQLs to develop jobs to meet your service requirements. Using SQL statements simplifies logic impl", + "product_code":"dli", + "title":"Creating a Flink SQL job", + "uri":"dli_01_0455.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"22" + }, + { + "desc":"This section describes how to create a Flink Jar job. You can perform secondary development based on Flink APIs, build your own JAR file, and submit the JAR file to DLI q", + "product_code":"dli", + "title":"Creating a Flink Jar Job", + "uri":"dli_01_0457.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"23" + }, + { + "desc":"The job debugging function helps you check the logic correctness of your compiled SQL statements before running a job.Currently, only Flink SQL jobs support this function", + "product_code":"dli", + "title":"Debugging a Flink Job", + "uri":"dli_01_0458.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"24" + }, + { + "desc":"After a job is created, you can perform operations on the job as required.Editing a JobStarting a JobStopping a JobDeleting a JobExporting a JobImporting a JobModifying N", + "product_code":"dli", + "title":"Performing Operations on a Flink Job", + "uri":"dli_01_0461.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"25" + }, + { + "desc":"After creating a job, you can view the job details to learn about the following information:Viewing Job DetailsChecking the Job Monitoring InformationViewing the Task Lis", + "product_code":"dli", + "title":"Flink Job Details", + "uri":"dli_01_0462.html", + "doc_type":"usermanual", + "p_code":"17", + "code":"26" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Spark Job Management", + "uri":"dli_01_0465.html", + "doc_type":"usermanual", + "p_code":"15", + "code":"27" + }, + { + "desc":"Based on the open-source Spark, DLI optimizes performance and reconstructs services to be compatible with the Apache Spark ecosystem and interfaces, and executes batch pr", + "product_code":"dli", + "title":"Spark Job Management", + "uri":"dli_01_0385.html", + "doc_type":"usermanual", + "p_code":"27", + "code":"28" + }, + { + "desc":"DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.On the Overview page, click Create Job in the upper right corner of the Spark Jo", + "product_code":"dli", + "title":"Creating a Spark Job", + "uri":"dli_01_0384.html", + "doc_type":"usermanual", + "p_code":"27", + "code":"29" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Queue Management", + "uri":"dli_01_0012.html", + "doc_type":"usermanual", + "p_code":"", + "code":"30" + }, + { + "desc":"Queues in DLI are computing resources, which are the basis for using DLI. All executed jobs require computing resources.Currently, DLI provides two types of queues, For S", + "product_code":"dli", + "title":"Overview", + "uri":"dli_01_0402.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"31" + }, + { + "desc":"You can isolate queues allocated to different users by setting permissions to ensure data query performance.The administrator and queue owner have all permissions, which ", + "product_code":"dli", + "title":"Queue Permission Management", + "uri":"dli_01_0015.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"32" + }, + { + "desc":"Before executing a job, you need to create a queue.If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main acco", + "product_code":"dli", + "title":"Creating a Queue", + "uri":"dli_01_0363.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"33" + }, + { + "desc":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", + "product_code":"dli", + "title":"Deleting a Queue", + "uri":"dli_01_0016.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"34" + }, + { + "desc":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", + "product_code":"dli", + "title":"Modifying the CIDR Block", + "uri":"dli_01_0443.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"35" + }, + { + "desc":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.If Status of queue xxx is assigning, which is not available is ", + "product_code":"dli", + "title":"Elastic Scaling", + "uri":"dli_01_0487.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"36" + }, + { + "desc":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", + "product_code":"dli", + "title":"Scheduling CU Changes", + "uri":"dli_01_0488.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"37" + }, + { + "desc":"It can be used to test the connectivity between the DLI queue and the peer IP address specified by the user in common scenarios, or the connectivity between the DLI queue", + "product_code":"dli", + "title":"Testing Address Connectivity", + "uri":"dli_01_0489.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"38" + }, + { + "desc":"Once you have created a message notification topic, you can Add subscription of the topic on the Topic Management page of the Simple Message Notification service. You can", + "product_code":"dli", + "title":"Creating a Message Notification Topic", + "uri":"dli_01_0421.html", + "doc_type":"usermanual", + "p_code":"30", + "code":"39" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Data Management", + "uri":"dli_01_0004.html", + "doc_type":"usermanual", + "p_code":"", + "code":"40" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Databases and Tables", + "uri":"dli_01_0390.html", + "doc_type":"usermanual", + "p_code":"40", + "code":"41" + }, + { + "desc":"DLI database and table management provide the following functions:Database Permission ManagementTable Permission ManagementCreating a Database or a TableDeleting a Databa", + "product_code":"dli", + "title":"Overview", + "uri":"dli_01_0228.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"42" + }, + { + "desc":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "product_code":"dli", + "title":"Database Permission Management", + "uri":"dli_01_0447.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"43" + }, + { + "desc":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "product_code":"dli", + "title":"Table Permission Management", + "uri":"dli_01_0448.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"44" + }, + { + "desc":"A database, built on the computer storage device, is a data warehouse where data is organized, stored, and managed based on its structure.The table is an important part o", + "product_code":"dli", + "title":"Creating a Database or a Table", + "uri":"dli_01_0005.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"45" + }, + { + "desc":"You can delete unnecessary databases and tables based on actual conditions.You are not allowed to delete databases or tables that are being used for running jobs.The admi", + "product_code":"dli", + "title":"Deleting a Database or a Table", + "uri":"dli_01_0011.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"46" + }, + { + "desc":"During actual use, developers create databases and tables and submit them to test personnel for testing. After the test is complete, the databases and tables are transfer", + "product_code":"dli", + "title":"Modifying the Owners of Databases and Tables", + "uri":"dli_01_0376.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"47" + }, + { + "desc":"You can import data from OBS to a table created in DLI.Only one path can be specified during data import. The path cannot contain commas (,).To import data in CSV format ", + "product_code":"dli", + "title":"Importing Data to the Table", + "uri":"dli_01_0253.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"48" + }, + { + "desc":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", + "product_code":"dli", + "title":"Exporting Data from DLI to OBS", + "uri":"dli_01_0010.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"49" + }, + { + "desc":"Metadata is used to define data types. It describes information about the data, including the source, size, format, and other data features. In database fields, metadata ", + "product_code":"dli", + "title":"Viewing Metadata", + "uri":"dli_01_0008.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"50" + }, + { + "desc":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", + "product_code":"dli", + "title":"Previewing Data", + "uri":"dli_01_0007.html", + "doc_type":"usermanual", + "p_code":"41", + "code":"51" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Package Management", + "uri":"dli_01_0366.html", + "doc_type":"usermanual", + "p_code":"40", + "code":"52" + }, + { + "desc":"Package management provides the following functions:Managing Package PermissionsCreating a PackageDeleting a PackageYou can delete program packages in batches.You can del", + "product_code":"dli", + "title":"Overview", + "uri":"dli_01_0407.html", + "doc_type":"usermanual", + "p_code":"52", + "code":"53" + }, + { + "desc":"You can isolate package groups or packages allocated to different users by setting permissions to ensure data query performance.The administrator and the owner of a packa", + "product_code":"dli", + "title":"Managing Permissions on Packages and Package Groups", + "uri":"dli_01_0477.html", + "doc_type":"usermanual", + "p_code":"52", + "code":"54" + }, + { + "desc":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", + "product_code":"dli", + "title":"Creating a Package", + "uri":"dli_01_0367.html", + "doc_type":"usermanual", + "p_code":"52", + "code":"55" + }, + { + "desc":"You can delete a package based on actual conditions.On the left of the management console, choose Data Management > Package Management.Click Delete in the Operation colum", + "product_code":"dli", + "title":"Deleting a Package", + "uri":"dli_01_0369.html", + "doc_type":"usermanual", + "p_code":"52", + "code":"56" + }, + { + "desc":"To change the owner of a package, click More > Modify Owner in the Operation column of a package on the Package Management page.If the package has been grouped, you can m", + "product_code":"dli", + "title":"Modifying the Owner", + "uri":"dli_01_0478.html", + "doc_type":"usermanual", + "p_code":"52", + "code":"57" + }, + { + "desc":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", + "product_code":"dli", + "title":"Built-in Dependencies", + "uri":"dli_01_0397.html", + "doc_type":"usermanual", + "p_code":"52", + "code":"58" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Job Templates", + "uri":"dli_01_0379.html", + "doc_type":"usermanual", + "p_code":"", + "code":"59" + }, + { + "desc":"To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not n", + "product_code":"dli", + "title":"SQL Template Management", + "uri":"dli_01_0021.html", + "doc_type":"usermanual", + "p_code":"59", + "code":"60" + }, + { + "desc":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", + "product_code":"dli", + "title":"Flink Template Management", + "uri":"dli_01_0464.html", + "doc_type":"usermanual", + "p_code":"59", + "code":"61" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Appendix", + "uri":"dli_01_05110.html", + "doc_type":"usermanual", + "p_code":"59", + "code":"62" + }, + { + "desc":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", + "product_code":"dli", + "title":"TPC-H Sample Data in the SQL Template", + "uri":"dli_01_05111.html", + "doc_type":"usermanual", + "p_code":"62", + "code":"63" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Datasource Connections", + "uri":"dli_01_0422.html", + "doc_type":"usermanual", + "p_code":"", + "code":"64" + }, + { + "desc":"DLI supports the datasource capability of the native Spark and extends it. With DLI datasource connection, you can access other data storage services through SQL statemen", + "product_code":"dli", + "title":"Datasource Connection and Cross-Source Analysis", + "uri":"dli_01_0410.html", + "doc_type":"usermanual", + "p_code":"64", + "code":"65" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Enhanced Datasource Connections", + "uri":"dli_01_0426.html", + "doc_type":"usermanual", + "p_code":"64", + "code":"66" + }, + { + "desc":"The enhanced datasource connection uses VPC peering at the bottom layer to directly connect the VPC network between the DLI cluster and the destination datasource. Data i", + "product_code":"dli", + "title":"Overview", + "uri":"dli_01_0003.html", + "doc_type":"usermanual", + "p_code":"66", + "code":"67" + }, + { + "desc":"The following describes how to create a datasource HBase connection for MRS.Only enhanced datasource connection to MRS HBase is supported.Apply for a cluster in MRS.If a ", + "product_code":"dli", + "title":"Creating, Querying, and Deleting an Enhanced Datasource Connection", + "uri":"dli_01_0006.html", + "doc_type":"usermanual", + "p_code":"66", + "code":"68" + }, + { + "desc":"The CIDR block of the DLI queue that is bound with a datasource connection cannot overlap with that of the data source.The default queue cannot be bound with a connection", + "product_code":"dli", + "title":"Binding and Unbinding a Queue", + "uri":"dli_01_0009.html", + "doc_type":"usermanual", + "p_code":"66", + "code":"69" + }, + { + "desc":"Method 1: Copy hosts information in /etc/hosts of an MRS node.cat /etc/hostsMethod 2: Log in to FusionInsight Manager to obtain MRS hosts information.The host information", + "product_code":"dli", + "title":"Modifying Host Information", + "uri":"dli_01_0013.html", + "doc_type":"usermanual", + "p_code":"66", + "code":"70" + }, + { + "desc":"After an enhanced datasource connection is created and bound to a queue, the system automatically configures route information. You can also add a custom route for the qu", + "product_code":"dli", + "title":"Custom Route Information", + "uri":"dli_01_0014.html", + "doc_type":"usermanual", + "p_code":"66", + "code":"71" + }, + { + "desc":"You can grant or revoke permissions for enhanced datasource connections of other projects.AuthorizationLog in to the DLI management console, choose Datasource Connections", + "product_code":"dli", + "title":"Enhanced Datasource Connection Permission Management", + "uri":"dli_01_0018.html", + "doc_type":"usermanual", + "p_code":"66", + "code":"72" + }, + { + "desc":"You can isolate datasource connections allocated to different users by setting permissions to ensure data query performance.The administrator and datasource connection ow", + "product_code":"dli", + "title":"Managing Datasource Connection Permissions", + "uri":"dli_01_0480.html", + "doc_type":"usermanual", + "p_code":"64", + "code":"73" + }, + { + "desc":"Datasource authentication is used to manage authentication information for accessing CSS and MRS security environments and encrypt passwords for accessing DWS, RDS, DDS, ", + "product_code":"dli", + "title":"Creating and Managing Datasource Authentication", + "uri":"dli_01_0427.html", + "doc_type":"usermanual", + "p_code":"64", + "code":"74" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Global Configuration", + "uri":"dli_01_0485.html", + "doc_type":"usermanual", + "p_code":"", + "code":"75" + }, + { + "desc":"Global variables can be used to simplify complex parameters. For example, long and difficult variables can be replaced to improve the readability of SQL statements.In the", + "product_code":"dli", + "title":"Global Variables", + "uri":"dli_01_0476.html", + "doc_type":"usermanual", + "p_code":"75", + "code":"76" + }, + { + "desc":"Only the tenant account or a subaccount of user group admin can authorize access.After entering the DLI management console, you are advised to set agency permissions to e", + "product_code":"dli", + "title":"Service Authorization", + "uri":"dli_01_0486.html", + "doc_type":"usermanual", + "p_code":"75", + "code":"77" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Permissions Management", + "uri":"dli_01_0408.html", + "doc_type":"usermanual", + "p_code":"", + "code":"78" + }, + { + "desc":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", + "product_code":"dli", + "title":"Overview", + "uri":"dli_01_0440.html", + "doc_type":"usermanual", + "p_code":"78", + "code":"79" + }, + { + "desc":"You can use Identity and Access Management (IAM) to implement fine-grained permissions control on DLI resources. For details, see Overview.If your cloud account does not ", + "product_code":"dli", + "title":"Creating an IAM User and Granting Permissions", + "uri":"dli_01_0418.html", + "doc_type":"usermanual", + "p_code":"78", + "code":"80" + }, + { + "desc":"Custom policies can be created as a supplement to the system policies of DLI. You can add actions to custom policies. For the actions supported for custom policies, see \"", + "product_code":"dli", + "title":"Creating a Custom Policy", + "uri":"dli_01_0451.html", + "doc_type":"usermanual", + "p_code":"78", + "code":"81" + }, + { + "desc":"A resource is an object that exists within a service. You can select DLI resources by specifying their paths.", + "product_code":"dli", + "title":"DLI Resources", + "uri":"dli_01_0417.html", + "doc_type":"usermanual", + "p_code":"78", + "code":"82" + }, + { + "desc":"Request conditions are useful in determining when a custom policy takes effect. A request condition consists of a condition key and operator. Condition keys are either gl", + "product_code":"dli", + "title":"DLI Request Conditions", + "uri":"dli_01_0475.html", + "doc_type":"usermanual", + "p_code":"78", + "code":"83" + }, + { + "desc":"Table 1 lists the common operations supported by each system policy of DLI. Choose proper system policies according to this table. For details about the SQL statement per", + "product_code":"dli", + "title":"Common Operations Supported by DLI System Policy", + "uri":"dli_01_0441.html", + "doc_type":"usermanual", + "p_code":"78", + "code":"84" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"FAQs", + "uri":"dli_01_0539.html", + "doc_type":"usermanual", + "p_code":"", + "code":"85" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Flink Jobs", + "uri":"dli_03_0037.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"86" + }, + { + "desc":"DLI Flink jobs support the following data formats:Avro, Avro_merge, BLOB, CSV, EMAIL, JSON, ORC, Parquet, and XML.DLI Flink jobs support data from the following data sour", + "product_code":"dli", + "title":"What Data Formats and Data Sources Are Supported by DLI Flink Jobs?", + "uri":"dli_03_0083.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"87" + }, + { + "desc":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", + "product_code":"dli", + "title":"How Do I Authorize a Subuser to View Flink Jobs?", + "uri":"dli_03_0139.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"88" + }, + { + "desc":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", + "product_code":"dli", + "title":"How Do I Set Auto Restart upon Exception for a Flink Job?", + "uri":"dli_03_0090.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"89" + }, + { + "desc":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", + "product_code":"dli", + "title":"How Do I Save Flink Job Logs?", + "uri":"dli_03_0099.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"90" + }, + { + "desc":"DLI can output Flink job results to DIS. You can view the results in DIS. For details, see Retrieving Data from DIS in the Data Ingestion Service User Guide.DLI can outpu", + "product_code":"dli", + "title":"How Can I Check Flink Job Results?", + "uri":"dli_03_0043.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"91" + }, + { + "desc":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", + "product_code":"dli", + "title":"What Should I Do if \"No such user. userName:xxxx.\" Is Displayed on the Flink Job Management Page When I Grant Permission to a User?", + "uri":"dli_03_0160.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"92" + }, + { + "desc":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. After a Flink job is manually stopped, no message is display", + "product_code":"dli", + "title":"How Do I Know Which Checkpoint the Flink Job I Stopped Will Be Restored to When I Start the Job Again?", + "uri":"dli_03_0180.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"93" + }, + { + "desc":"The consumption capability of a Flink SQL job depends on the data source transmission, queue size, and job parameter settings. The peak consumption is 10 Mbit/s.", + "product_code":"dli", + "title":"How Much Data Can Be Processed in a Day by a Flink SQL Job?", + "uri":"dli_03_0130.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"94" + }, + { + "desc":"The temp stream in Flink SQL is similar to a subquery. It is a logical stream used to simplify the SQL logic and does not generate data storage. Therefore, there is no ne", + "product_code":"dli", + "title":"Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the Data?", + "uri":"dli_03_0061.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"95" + }, + { + "desc":"SymptomWhen you create a Flink SQL job and configure the parameters, you select an OBS bucket you have created. The system displays a message indicating that the OBS buck", + "product_code":"dli", + "title":"What Should I Do if a Message Is Displayed Indicating that the OBS Bucket Is Not Authorized When I Select an OBS Bucket for a Flink SQL Job?", + "uri":"dli_03_0138.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"96" + }, + { + "desc":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", + "product_code":"dli", + "title":"How Do I Map an OBS Table to a Partitioned DLI Table?", + "uri":"dli_03_0089.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"97" + }, + { + "desc":"SymptomYou used Flink 1.10 to run a Flink Opensource SQL job. You set the number of Kafka partitions for the job a small value at the beginning and need to increase the n", + "product_code":"dli", + "title":"How Do I Change the Number of Kafka Partitions of a Flink SQL Job Without Stopping It?", + "uri":"dli_03_0120.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"98" + }, + { + "desc":"In this example, the day field is used as the partition field with the parquet encoding format (only the parquet format is supported currently) to dump car_info data to O", + "product_code":"dli", + "title":"How Can I Map an OBS Table to a DLI Partition Table?", + "uri":"dli_03_0075.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"99" + }, + { + "desc":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", + "product_code":"dli", + "title":"What Should I Do If Error Message DLI.0005 Is Displayed When I Use an EL Expression to Create a Table in a Flink SQL Job?", + "uri":"dli_03_0167.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"100" + }, + { + "desc":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", + "product_code":"dli", + "title":"What Should I Do If No Data Is Found in the DLI Table Created Using the OBS File Path That Stores the Output of a Flink job?", + "uri":"dli_03_0168.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"101" + }, + { + "desc":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", + "product_code":"dli", + "title":"What Should I Do If a Flink SQL Job Fails to Be Executed, and \"connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null\" Is Recorded in the Log?", + "uri":"dli_03_0174.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"102" + }, + { + "desc":"Semantic verification for a Flink SQL job (reading DIS data) fails. The following information is displayed when the job fails:Get dis channel xxxinfo failed. error info: ", + "product_code":"dli", + "title":"What Should I Do If \"Not authorized\" Is Reported When a Flink SQL Job Reads DIS Data?", + "uri":"dli_03_0176.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"103" + }, + { + "desc":"After a Flink SQL job consumed Kafka and sank data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", + "product_code":"dli", + "title":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster", + "uri":"dli_03_0232.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"104" + }, + { + "desc":"The procedure is as follows:Add the following code to the JAR package code of the Flink Jar job:// Configure the pom file on which the StreamExecutionEnvironment depends.", + "product_code":"dli", + "title":"How Do I Configure Checkpoints for Flink Jar Jobs and Save the Checkpoints to OBS?", + "uri":"dli_03_0038.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"105" + }, + { + "desc":"Configuration files can be uploaded for user-defined jobs (JAR).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flin", + "product_code":"dli", + "title":"Does a Flink JAR Job Support Configuration File Upload? How Do I Upload a Configuration File?", + "uri":"dli_03_0044.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"106" + }, + { + "desc":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Delete your JAR package that is the", + "product_code":"dli", + "title":"What Should I Do If Job Submission Fails Due to Flink JAR Package Conflict?", + "uri":"dli_03_0119.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"107" + }, + { + "desc":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", + "product_code":"dli", + "title":"What Should I Do If GaussDB(DWS) Fails to Be Started When a Flink Jar Job Accesses GaussDB(DWS), and a Message Indicating Too Many Client Connections is Displayed?", + "uri":"dli_03_0161.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"108" + }, + { + "desc":"An exception occurred when a Flink Jar job is running. The following error information is displayed in the job log:org.apache.flink.shaded.curator.org.apache.curator.Conn", + "product_code":"dli", + "title":"What Should I Do If Error Message \"Authentication failed\" Is Displayed When a Flink Jar Job Is Running?", + "uri":"dli_03_0165.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"109" + }, + { + "desc":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", + "product_code":"dli", + "title":"What Should I Do If Invalid OBS Bucket Name Is Reported After a Flink Job Submission Failed?", + "uri":"dli_03_0233.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"110" + }, + { + "desc":"Flink Job submission failed. The exception information is as follows:Flink JAR packages conflicted. The submitted Flink JAR package conflicted with the HDFS JAR package o", + "product_code":"dli", + "title":"Flink Job Submission Fails Due to Hadoop JAR Package Conflicts", + "uri":"dli_03_0234.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"111" + }, + { + "desc":"You can use Flink Jar to connect to Kafka with SASL SSL authentication enabled.", + "product_code":"dli", + "title":"How Do I Connect a Flink jar Job to SASL_SSL?", + "uri":"dli_03_0266.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"112" + }, + { + "desc":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", + "product_code":"dli", + "title":"How Do I Optimize Performance of a Flink Job?", + "uri":"dli_03_0106.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"113" + }, + { + "desc":"Add the following SQL statements to the Flink job:", + "product_code":"dli", + "title":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?", + "uri":"dli_03_0048.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"114" + }, + { + "desc":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", + "product_code":"dli", + "title":"How Do I Prevent Data Loss After Flink Job Restart?", + "uri":"dli_03_0096.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"115" + }, + { + "desc":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", + "product_code":"dli", + "title":"How Do I Locate a Flink Job Submission Error?", + "uri":"dli_03_0103.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"116" + }, + { + "desc":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", + "product_code":"dli", + "title":"How Do I Locate a Flink Job Running Error?", + "uri":"dli_03_0105.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"117" + }, + { + "desc":"Check the following operations:Adjusting or adding optimization parameters or the number of concurrent threads of a job, or modifying Flink SQL statements or a Flink Jar ", + "product_code":"dli", + "title":"How Do I Know Whether a Flink Job Can Be Restored from a Checkpoint After Being Restarted?", + "uri":"dli_03_0136.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"118" + }, + { + "desc":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", + "product_code":"dli", + "title":"What Should I Do If the DIS Stream Does Not Exist During the Semantic Check?", + "uri":"dli_03_0040.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"119" + }, + { + "desc":"If the OBS bucket selected for a job is not authorized, perform the following steps:Select Enable Checkpointing or Save Job Log.Specify OBS Bucket.Select Authorize OBS.", + "product_code":"dli", + "title":"What Should I Do If the OBS Bucket Selected for a Job Is Not Authorized?", + "uri":"dli_03_0045.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"120" + }, + { + "desc":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", + "product_code":"dli", + "title":"What Should I Do If Logs Are Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?", + "uri":"dli_03_0064.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"121" + }, + { + "desc":"SymptomYou used Flink 1.10 to run a Flink Opensource SQL job. The job failed after the following error was reported when Flink Sink wrote data to Kafka.Caused by: org.apa", + "product_code":"dli", + "title":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?", + "uri":"dli_03_0121.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"122" + }, + { + "desc":"The Flink/Spark UI was displayed with incomplete information.When a pay-per-use queue is used to run a job, the system releases the cluster and takes about 10 minutes to ", + "product_code":"dli", + "title":"What Should I Do If the Flink/Spark UI Page Cannot Be Displayed Properly?", + "uri":"dli_03_0235.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"123" + }, + { + "desc":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", + "product_code":"dli", + "title":"Wha should I Do If JobManager and TaskManager Heartbeats Timed Out and the Flink Job Is Abnormal?", + "uri":"dli_03_0236.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"124" + }, + { + "desc":"Test address connectivity.If the network is unreachable, rectify the network connection first. Ensure that the network connection between the DLI queue and the external d", + "product_code":"dli", + "title":"What Can I Do If \"Timeout expired while fetching topic metadata\" Is Repeatedly Reported in Flink JobManager Logs?", + "uri":"dli_03_0265.html", + "doc_type":"usermanual", + "p_code":"86", + "code":"125" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Problems Related to SQL Jobs", + "uri":"dli_03_0020.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"126" + }, + { + "desc":"A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in", + "product_code":"dli", + "title":"SQL Jobs", + "uri":"dli_03_0200.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"127" + }, + { + "desc":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.Set t", + "product_code":"dli", + "title":"How Do I Merge Small Files?", + "uri":"dli_03_0086.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"128" + }, + { + "desc":"When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.If the specified path is akdc", + "product_code":"dli", + "title":"How Do I Specify an OBS Path When Creating an OBS Table?", + "uri":"dli_03_0092.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"129" + }, + { + "desc":"DLI allows you to associate JSON data in an OBS bucket to create tables in asynchronous mode.The statement for creating the table is as follows:", + "product_code":"dli", + "title":"How Do I Create a Table Using JSON Data in an OBS Bucket?", + "uri":"dli_03_0108.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"130" + }, + { + "desc":"You can use the where condition statement in the select statement to filter data. For example:", + "product_code":"dli", + "title":"How Do I Set Local Variables in SQL Statements?", + "uri":"dli_03_0087.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"131" + }, + { + "desc":"The correct method for using the count function to perform aggregation is as follows:OrIf an incorrect method is used, an error will be reported.", + "product_code":"dli", + "title":"How Can I Use the count Function to Perform Aggregation?", + "uri":"dli_03_0069.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"132" + }, + { + "desc":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket. For details, s", + "product_code":"dli", + "title":"How Do I Synchronize DLI Table Data from One Region to Another?", + "uri":"dli_03_0072.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"133" + }, + { + "desc":"Currently, DLI does not allow you to insert table data into specific fields. To insert table data, you must insert data of all table fields at a time.", + "product_code":"dli", + "title":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?", + "uri":"dli_03_0191.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"134" + }, + { + "desc":"Create an OBS directory with a unique name. Alternatively, you can manually delete the existing OBS directory and submit the job again. However, exercise caution when del", + "product_code":"dli", + "title":"What Should I Do If Error \"path obs://xxx already exists\" Is Reported When Data Is Exported to OBS?", + "uri":"dli_03_0014.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"135" + }, + { + "desc":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", + "product_code":"dli", + "title":"What Should I Do If \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Is Displayed When Two Tables Are Joined", + "uri":"dli_03_0066.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"136" + }, + { + "desc":"Check whether the account is in arrears. If yes, recharge the account.If the error persists, log out and log in again.", + "product_code":"dli", + "title":"What Should I Do If the System Notify Me that I Have No Permission to Execute the Query Statement Because the Current Account Was Restricted?", + "uri":"dli_03_0071.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"137" + }, + { + "desc":"Cause AnalysisWhen you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the q", + "product_code":"dli", + "title":"What Should I Do If Error Message \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" Is Displayed When a Query Statement Is Executed?", + "uri":"dli_03_0145.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"138" + }, + { + "desc":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", + "product_code":"dli", + "title":"What Should I Do If Error Message IllegalArgumentException: Buffer size too small. size is Displayed When Data Is Loaded to an OBS Foreign Table", + "uri":"dli_03_0169.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"139" + }, + { + "desc":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", + "product_code":"dli", + "title":"What Should I Do If \"DLI.0002 FileNotFoundException\" Is Reported During SQL Job Running", + "uri":"dli_03_0189.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"140" + }, + { + "desc":"Currently, DLI supports the Hive syntax for creating tables of the TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET file types. If the file format specified for cre", + "product_code":"dli", + "title":"What Should I Do If A Schema Parsing Error Is Reported When I Create a Hive Table Using CTAS", + "uri":"dli_03_0046.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"141" + }, + { + "desc":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", + "product_code":"dli", + "title":"What Should I Do If \"org.apache.hadoop.fs.obs.OBSIOException\" Is Reported If DLI SQL Scripts Are Executed on DataArts Studio?", + "uri":"dli_03_0173.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"142" + }, + { + "desc":"After the migration job is submitted, the following error information is displayed in the log:org.apache.sqoop.common.SqoopException:UQUERY_CONNECTOR_0001:Invoke DLI serv", + "product_code":"dli", + "title":"What Should I Do If \"UQUERY_CONNECTOR_0001:Invoke DLI service api failed\" Is Reported in the Job Log When I Use CDM to Migrate Data to DLI?", + "uri":"dli_03_0172.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"143" + }, + { + "desc":"Error message \"File not Found\" is displayed when a SQL job is accessed.Generally, the file cannot be found due to a read/write conflict. Check whether a job is overwritin", + "product_code":"dli", + "title":"What Should I Do If \"File not Found\" Is Displayed When I Access a SQL Job?", + "uri":"dli_03_0207.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"144" + }, + { + "desc":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.View the OBS bucket in the AccessControlException and check whether you are us", + "product_code":"dli", + "title":"What Should I Do If \"DLI.0003: AccessControlException XXX\" Is Reported When I Access a SQL Job?", + "uri":"dli_03_0208.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"145" + }, + { + "desc":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Th", + "product_code":"dli", + "title":"What Should I Do If \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Is Reported When I Access a SQL Job?", + "uri":"dli_03_0209.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"146" + }, + { + "desc":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", + "product_code":"dli", + "title":"What Should I Do If \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget\" Is Reported During SQL Statement Execution? Restricted for no budget.", + "uri":"dli_03_0210.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"147" + }, + { + "desc":"If the job runs slowly, perform the following steps to find the causes and rectify the fault:Check whether the problem is caused by FullGC.Log in to the DLI console. In t", + "product_code":"dli", + "title":"How Do I Troubleshoot Slow SQL Jobs?", + "uri":"dli_03_0196.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"148" + }, + { + "desc":"You can view SQL job logs for routine O&M.Obtain the ID of the DLI job executed on the DataArts Studio console.Job IDOn the DLI console, choose Job Management > SQL Jobs.", + "product_code":"dli", + "title":"How Do I View DLI SQL Logs?", + "uri":"dli_03_0091.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"149" + }, + { + "desc":"You can view the job execution records when a job is running.Log in to the DLI management console.In the navigation pane on the left, choose Job Management > SQL Jobs.Ent", + "product_code":"dli", + "title":"How Do I View SQL Execution Records?", + "uri":"dli_03_0116.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"150" + }, + { + "desc":"If the execution of an SQL statement takes a long time, you need to access the Spark UI to check the execution status.If data skew occurs, the running time of a stage exc", + "product_code":"dli", + "title":"How Do I Eliminate Data Skew by Configuring AE Parameters?", + "uri":"dli_03_0093.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"151" + }, + { + "desc":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", + "product_code":"dli", + "title":"What Can I Do If a Table Cannot Be Queried on the DLI Console?", + "uri":"dli_03_0184.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"152" + }, + { + "desc":"A high compression ratio of OBS tables in the Parquet or ORC format (for example, a compression ratio of 5 or higher compared with text compression) will lead to large da", + "product_code":"dli", + "title":"The Compression Ratio of OBS Tables Is Too High", + "uri":"dli_03_0013.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"153" + }, + { + "desc":"DLI supports only UTF-8-encoded texts. Ensure that data is encoded using UTF-8 during table creation and import.", + "product_code":"dli", + "title":"How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?", + "uri":"dli_03_0009.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"154" + }, + { + "desc":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", + "product_code":"dli", + "title":"Do I Need to Grant Table Permissions to a User and Project After I Delete a Table and Create One with the Same Name?", + "uri":"dli_03_0175.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"155" + }, + { + "desc":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", + "product_code":"dli", + "title":"What Should I Do If Table Data Fails to Be Queried After Data Is Imported to a DLI Partitioned Table Because the File to Be Imported Does Not Contain Data in the Partitioning Column?", + "uri":"dli_03_0177.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"156" + }, + { + "desc":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", + "product_code":"dli", + "title":"How Do I Fix the Data Error Caused by CRLF Characters in a Field of the OBS File Used to Create an External OBS Table?", + "uri":"dli_03_0181.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"157" + }, + { + "desc":"A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.When a Spark SQL job has join operations on sma", + "product_code":"dli", + "title":"What Should I Do If a SQL Job That Has Join Operations Stays in the Running State?", + "uri":"dli_03_0182.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"158" + }, + { + "desc":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", + "product_code":"dli", + "title":"The on Clause Is Not Added When Tables Are Joined. Cartesian Product Query Causes High Resource Usage of the Queue, and the Job Fails to Be Executed", + "uri":"dli_03_0187.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"159" + }, + { + "desc":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", + "product_code":"dli", + "title":"What Should I Do If Manually Added Data in a Partition of an OBS Table Cannot Be Queried?", + "uri":"dli_03_0190.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"160" + }, + { + "desc":"To dynamically overwrite the specified partitioned data in the DataSource table, set dli.sql.dynamicPartitionOverwrite.enabled to true and then run the insert overwrite s", + "product_code":"dli", + "title":"Why Is All Data Overwritten When insert overwrite Is Used to Overwrite Partitioned Table?", + "uri":"dli_03_0212.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"161" + }, + { + "desc":"The possible causes and solutions are as follows:After you purchase a DLI queue and submit a SQL job for the first time, wait for 5 to 10 minutes. After the cluster is st", + "product_code":"dli", + "title":"Why Is a SQL Job Stuck in the Submitting State?", + "uri":"dli_03_0213.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"162" + }, + { + "desc":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", + "product_code":"dli", + "title":"Why Is the create_date Field in the RDS Table Is a Timestamp in the DLI query result?", + "uri":"dli_03_0214.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"163" + }, + { + "desc":"If the table name is changed immediately after SQL statements are executed, the data size of the table may be incorrect.If you need to change the table name, change it 5 ", + "product_code":"dli", + "title":"What Can I Do If datasize Cannot Be Changed After the Table Name Is Changed in a Finished SQL Job?", + "uri":"dli_03_0215.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"164" + }, + { + "desc":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", + "product_code":"dli", + "title":"Why Is the Data Volume Changes When Data Is Imported from DLI to OBS?", + "uri":"dli_03_0231.html", + "doc_type":"usermanual", + "p_code":"126", + "code":"165" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Problems Related to Spark Jobs", + "uri":"dli_03_0021.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"166" + }, + { + "desc":"DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.The Spark SQL syntax does no", + "product_code":"dli", + "title":"Spark Jobs", + "uri":"dli_03_0201.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"167" + }, + { + "desc":"To use Spark to write data into a DLI table, configure the following parameters:fs.obs.access.keyfs.obs.secret.keyfs.obs.implfs.obs.endpointThe following is an example:", + "product_code":"dli", + "title":"How Do I Use Spark to Write Data into a DLI Table?", + "uri":"dli_03_0107.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"168" + }, + { + "desc":"If the AK and SK are obtained, set the parameters as follows:Create SparkContext using codeval sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.ac", + "product_code":"dli", + "title":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?", + "uri":"dli_03_0017.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"169" + }, + { + "desc":"Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parame", + "product_code":"dli", + "title":"How Do I View the Resource Usage of DLI Spark Jobs?", + "uri":"dli_03_0102.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"170" + }, + { + "desc":"If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management p", + "product_code":"dli", + "title":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?", + "uri":"dli_03_0076.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"171" + }, + { + "desc":"DLI natively supports PySpark.For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are pac", + "product_code":"dli", + "title":"How Do I Run a Complex PySpark Program in DLI?", + "uri":"dli_03_0082.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"172" + }, + { + "desc":"You can use DLI Spark jobs to access data in the MySQL database using either of the following methods:Solution 1: Purchase a pay-per-use queue, create an enhanced datasou", + "product_code":"dli", + "title":"How Does a Spark Job Access a MySQL Database?", + "uri":"dli_03_0127.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"173" + }, + { + "desc":"When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.To solve this problem, you can config", + "product_code":"dli", + "title":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", + "uri":"dli_03_0068.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"174" + }, + { + "desc":"You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get(\"Name of the uploaded file\").The file path in the Driver is different fro", + "product_code":"dli", + "title":"How Do I Read Uploaded Files for a Spark Jar Job?", + "uri":"dli_03_0118.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"175" + }, + { + "desc":"The Spark 2.3 has changed the behavior of the internal interface Logging. If the user code directly inherits the Logging and the earlier version Spark is used during comp", + "product_code":"dli", + "title":"What Can I Do When Receiving java.lang.AbstractMethodError in the Spark Job?", + "uri":"dli_03_0023.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"176" + }, + { + "desc":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data. For details, see How Do I Set the AK/SK for a Qu", + "product_code":"dli", + "title":"What Should I Do If Errors \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Are Reported When a Spark Job Accesses OBS Data", + "uri":"dli_03_0156.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"177" + }, + { + "desc":"Check whether the OBS bucket is used to store DLI logs on the Global Configuration > Job Configurations page. The job log bucket cannot be used for other purpose.", + "product_code":"dli", + "title":"Why is Error \"verifyBucketExists on XXXX: status [403]\" Is Reported When I Run a Spark Job with a User that has the Permission to Access the OBS Bucket?", + "uri":"dli_03_0164.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"178" + }, + { + "desc":"When a Spark job accesses a large amount of data, for example, accessing data in a GaussDB(DWS) database, you are advised to set the number of concurrent tasks and enable", + "product_code":"dli", + "title":"What Should I Do if a Running Timeout Error Is Reported When a Spark Job Runs a Large Amount of Data?", + "uri":"dli_03_0157.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"179" + }, + { + "desc":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.", + "product_code":"dli", + "title":"What Should I Do If a Spark job Used to Access Files in SFTP Fails and the Log Shows that the File Directory Is Abnormal?", + "uri":"dli_03_0188.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"180" + }, + { + "desc":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", + "product_code":"dli", + "title":"What Should I Do If a Job Fails Because the Job User Does Not Have the Database and Table Permission?", + "uri":"dli_03_0192.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"181" + }, + { + "desc":"If the specified Python environment cannot be found after a Python3 package is added, the current compute cluster environment is Python2 by default.You can set spark.yarn", + "product_code":"dli", + "title":"What Can I Do If the Specified Python Environment Cannot Be Found After a Python Package Is Added?", + "uri":"dli_03_0077.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"182" + }, + { + "desc":"The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.To view the remaining CUs of a queue, perform the following steps:Check the C", + "product_code":"dli", + "title":"Why Is a Spark Jar Job Stuck in the Submitting State?", + "uri":"dli_03_0220.html", + "doc_type":"usermanual", + "p_code":"166", + "code":"183" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Product Consultation", + "uri":"dli_03_0001.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"184" + }, + { + "desc":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark, and Flink ecosystems. It frees you from managing any serv", + "product_code":"dli", + "title":"What Is DLI?", + "uri":"dli_03_0002.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"185" + }, + { + "desc":"DLI supports the following data formats:ParquetCSVORCJsonAvro", + "product_code":"dli", + "title":"Which Data Formats Does DLI Support?", + "uri":"dli_03_0025.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"186" + }, + { + "desc":"The Spark component of DLI is a fully managed service. You can only use the DLI Spark through its APIs. .The Spark component of MRS is built on the VM in an MRS cluster. ", + "product_code":"dli", + "title":"What Are the Differences Between MRS Spark and DLI Spark?", + "uri":"dli_03_0115.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"187" + }, + { + "desc":"DLI data can be stored in either of the following:OBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-base", + "product_code":"dli", + "title":"Where Can DLI Data Be Stored?", + "uri":"dli_03_0029.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"188" + }, + { + "desc":"DLI tables store data within the DLI service, and you do not need to know the data storage path.OBS tables store data in your OBS buckets, and you need to manage the sour", + "product_code":"dli", + "title":"What Are the Differences Between DLI Tables and OBS Tables?", + "uri":"dli_03_0117.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"189" + }, + { + "desc":"Currently, DLI supports analysis only on the data uploaded to the cloud. In scenarios where regular (for example, on a per day basis) one-off analysis on incremental data", + "product_code":"dli", + "title":"How Can I Use DLI If Data Is Not Uploaded to OBS?", + "uri":"dli_03_0010.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"190" + }, + { + "desc":"Data in the OBS bucket shared by IAM users under the same account can be imported. You cannot import data in the OBS bucket shared with other IAM account.", + "product_code":"dli", + "title":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?", + "uri":"dli_03_0129.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"191" + }, + { + "desc":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", + "product_code":"dli", + "title":"What Should I do If the System Failed to Create a Database and {\"error_code\":\"DLI.1028\";\"error_msg\":\"Already reached the maximum quota of databases:XXX\" Is Displayed Indicating that the Quota Is Insufficient?", + "uri":"dli_03_0264.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"192" + }, + { + "desc":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", + "product_code":"dli", + "title":"Can a Member Account Use Global Variables Created by Other Member Accounts?", + "uri":"dli_03_0263.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"193" + }, + { + "desc":"If you are suggested to perform following operations to run a large number of DLI jobs:Group the DLI jobs by type, and run each group on a queue.Alternatively, create IAM", + "product_code":"dli", + "title":"How Do I Manage Tens of Thousands of Jobs Running on DLI?", + "uri":"dli_03_0126.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"194" + }, + { + "desc":"The field names of tables that have been created cannot be changed.You can create a table, define new table fields, and migrate data from the old table to the new one.", + "product_code":"dli", + "title":"How Do I Change the Name of a Field in a Created Table?", + "uri":"dli_03_0162.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"195" + }, + { + "desc":"No. The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.", + "product_code":"dli", + "title":"Does DLI Have the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?", + "uri":"dli_03_0260.html", + "doc_type":"usermanual", + "p_code":"184", + "code":"196" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Quota", + "uri":"dli_03_0053.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"197" + }, + { + "desc":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", + "product_code":"dli", + "title":"How Do I View My Quotas?", + "uri":"dli_03_0031.html", + "doc_type":"usermanual", + "p_code":"197", + "code":"198" + }, + { + "desc":"The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your applica", + "product_code":"dli", + "title":"How Do I Increase a Quota?", + "uri":"dli_03_0032.html", + "doc_type":"usermanual", + "p_code":"197", + "code":"199" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Permission", + "uri":"dli_03_0054.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"200" + }, + { + "desc":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", + "product_code":"dli", + "title":"How Do I Manage Fine-Grained DLI Permissions?", + "uri":"dli_03_0100.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"201" + }, + { + "desc":"You cannot perform permission-related operations on the partition column of a partition table. However, when you grant the permission of any non-partition column in a par", + "product_code":"dli", + "title":"What Is Column Permission Granting of a DLI Partition Table?", + "uri":"dli_03_0008.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"202" + }, + { + "desc":"You can set actions and resources of different levels for various scenarios.Define an action.The format is Service name:Resource type:Action. You can use wildcard *. The ", + "product_code":"dli", + "title":"How Do I Create a Custom Policy?", + "uri":"dli_03_0101.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"203" + }, + { + "desc":"You can isolate queues allocated to different users by setting permissions to ensure data query performance.", + "product_code":"dli", + "title":"How Do I Manage Queue Permissions?", + "uri":"dli_03_0225.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"204" + }, + { + "desc":"When you submit a job, a message is displayed indicating that the job fails to be submitted due to insufficient permission caused by arrears. In this case, you need to ch", + "product_code":"dli", + "title":"What Should I Do When My Account Does Not Have Permission Due To Arrears?", + "uri":"dli_03_0140.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"205" + }, + { + "desc":"When the user update an existing program package, the following error information is displayed:\"error_code\"*CLI.0003\",\"error_msg\":\"Permission denied for resource 'resourc", + "product_code":"dli", + "title":"What Should I Do If the Permission Is Required to Update a Program Package?", + "uri":"dli_03_0195.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"206" + }, + { + "desc":"When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.Error information: DLI.00", + "product_code":"dli", + "title":"What Should I Do If \"DLI.0003: Permission denied for resource...\" Is Reported When I Run a SQL Statement?", + "uri":"dli_03_0227.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"207" + }, + { + "desc":"The table permission has been granted and verified. However, after a period of time, an error is reported indicating that the table query fails.There are two possible rea", + "product_code":"dli", + "title":"What Should I Do If the Table Permission Has Been Granted But the Table Still Cannot Be Queried?", + "uri":"dli_03_0228.html", + "doc_type":"usermanual", + "p_code":"200", + "code":"208" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Queue", + "uri":"dli_03_0049.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"209" + }, + { + "desc":"Currently, you are not allowed to modify the description of a created queue. You can add the description when purchasing the queue.", + "product_code":"dli", + "title":"Does the Description of a DLI Queue Can Be Modified?", + "uri":"dli_03_0109.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"210" + }, + { + "desc":"Deleting a queue does not cause table data loss in your database.", + "product_code":"dli", + "title":"Will Table Data in My Database Be Lost If I Delete a Queue?", + "uri":"dli_03_0166.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"211" + }, + { + "desc":"You need to develop a mechanism to retry failed jobs. When a faulty queue is recovered, your application tries to submit the failed jobs to the queue again.", + "product_code":"dli", + "title":"How Does DLI Ensure the Reliability of Spark Jobs When a Queue Is Abnormal?", + "uri":"dli_03_0170.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"212" + }, + { + "desc":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", + "product_code":"dli", + "title":"How Do I Monitor Queue Exceptions?", + "uri":"dli_03_0098.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"213" + }, + { + "desc":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", + "product_code":"dli", + "title":"How Do I View DLI Queue Load?", + "uri":"dli_03_0095.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"214" + }, + { + "desc":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", + "product_code":"dli", + "title":"How Do I Determine Whether There Are Too Many Jobs in the Current Queue?", + "uri":"dli_03_0183.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"215" + }, + { + "desc":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", + "product_code":"dli", + "title":"How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?", + "uri":"dli_03_0065.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"216" + }, + { + "desc":"DLI queues do not use resources or bandwidth when no job is running. In this case, the running status of DLI queues is not displayed on CES.", + "product_code":"dli", + "title":"Why Cannot I View the Resource Running Status of DLI Queues on Cloud Eye?", + "uri":"dli_03_0193.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"217" + }, + { + "desc":"In DLI, 64 CU = 64 cores and 256 GB memory.In a Spark job, if the driver occupies 4 cores and 16 GB memory, the executor can occupy 60 cores and 240 GB memory.", + "product_code":"dli", + "title":"How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?", + "uri":"dli_03_0088.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"218" + }, + { + "desc":"Queue plans create failed. The plan xxx target cu is out of quota is displayed when you create a scheduled scaling task.The CU quota of the current account is insufficien", + "product_code":"dli", + "title":"What Should I Do If \"Queue plans create failed. The plan xxx target cu is out of quota\" Is Displayed When I Schedule CU Changes?", + "uri":"dli_03_0159.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"219" + }, + { + "desc":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", + "product_code":"dli", + "title":"What Should I Do If SQL Statements Fails to be Executed on the Default Queue, and a Timeout Exception is Reported?", + "uri":"dli_03_0171.html", + "doc_type":"usermanual", + "p_code":"209", + "code":"220" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Datasource Connections", + "uri":"dli_03_0022.html", + "doc_type":"usermanual", + "p_code":"85", + "code":"221" + }, + { + "desc":"You need to create a VPC peering connection to enable network connectivity. Take MRS as an example. If DLI and MRS clusters are in the same VPC, and the security group is", + "product_code":"dli", + "title":"Why Do I Need to Create a VPC Peering Connection for an Enhanced Datasource Connection?", + "uri":"dli_03_0128.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"222" + }, + { + "desc":"An enhanced datasource connection failed to pass the network connectivity test. Datasource connection cannot be bound to a queue. The following error information is displ", + "product_code":"dli", + "title":"Failed to Bind a Queue to an Enhanced Datasource Connection", + "uri":"dli_03_0237.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"223" + }, + { + "desc":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", + "product_code":"dli", + "title":"DLI Failed to Connect to GaussDB(DWS) Through an Enhanced Datasource Connection", + "uri":"dli_03_0238.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"224" + }, + { + "desc":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", + "product_code":"dli", + "title":"What Can I Do If the Datasource Connection is Created But the Network Connectivity Test Fails?", + "uri":"dli_03_0179.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"225" + }, + { + "desc":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", + "product_code":"dli", + "title":"How Do I Configure the Network Between a DLI Queue and a Data Source?", + "uri":"dli_03_0186.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"226" + }, + { + "desc":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", + "product_code":"dli", + "title":"What Can I Do If a Datasource Connection Is Stuck in Creating State When I Try to Bind a Queue to It?", + "uri":"dli_03_0257.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"227" + }, + { + "desc":"Enhanced datasource connections support only yearly/monthly and pay-per-use queues.You can bind a datasource connection only when you select Dedicated Resource Mode when ", + "product_code":"dli", + "title":"How Do I Bind a Datasource Connection to a Pay-per-Use Queue?", + "uri":"dli_03_0258.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"228" + }, + { + "desc":"DLI enhanced datasource connection uses VPC peering to directly connect the VPC networks of the desired data sources for point-to-point data exchanges.", + "product_code":"dli", + "title":"How Do I Connect DLI to Data Sources?", + "uri":"dli_03_0259.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"229" + }, + { + "desc":"To perform query on data stored on services rather than DLI, perform the following steps:Assume that the data to be queried is stored on multiple services (for example, O", + "product_code":"dli", + "title":"How Can I Perform Query on Data Stored on Services Rather Than DLI?", + "uri":"dli_03_0011.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"230" + }, + { + "desc":"Connect VPCs in different regions.Create an enhanced datasource connection on DLI and bind it to a queue.Add a DLI route.", + "product_code":"dli", + "title":"How Can I Access Data Across Regions?", + "uri":"dli_03_0085.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"231" + }, + { + "desc":"When data is inserted into DLI, set the ID field to NULL.", + "product_code":"dli", + "title":"How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS Table When Creating a DLI and Associating It with the RDS Table?", + "uri":"dli_03_0028.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"232" + }, + { + "desc":"Possible CausesThe network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.The network connec", + "product_code":"dli", + "title":"Why Is the Error Message \"communication link failure\" Displayed When I Use a Newly Activated Datasource Connection?", + "uri":"dli_03_0047.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"233" + }, + { + "desc":"The cluster host information is not added to the datasource connection. As a result, the KRB authentication fails, the connection times out, and no error is recorded in l", + "product_code":"dli", + "title":"Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs", + "uri":"dli_03_0080.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"234" + }, + { + "desc":"When you create a VPC peering connection for the datasource connection, the following error information is displayed:Before you create a datasource connection, check whet", + "product_code":"dli", + "title":"What Should I Do If the Subnet Cannot Be Found When I Create a DLI Datasource Connection?", + "uri":"dli_03_0111.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"235" + }, + { + "desc":"A cross-source RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incor", + "product_code":"dli", + "title":"Error Message \"Incorrect string value\" Is Displayed When insert overwrite Is Executed on a Cross-Source RDS Table", + "uri":"dli_03_0239.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"236" + }, + { + "desc":"The system failed to create a cross-source RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre", + "product_code":"dli", + "title":"Null Pointer Error Is Displayed When the System Creates a Cross-Source RDS Table", + "uri":"dli_03_0250.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"237" + }, + { + "desc":"The system failed to execute insert overwrite on the cross-source GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was display", + "product_code":"dli", + "title":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed When the System Executes insert overwrite on a Cross-Source GaussDB(DWS) Table", + "uri":"dli_03_0251.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"238" + }, + { + "desc":"A cross-source table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. ", + "product_code":"dli", + "title":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Cross-Source Table", + "uri":"dli_03_0252.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"239" + }, + { + "desc":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", + "product_code":"dli", + "title":"A Null Value Is Written Into a Non-Null Field When a DLI Datasource Connection Is Used to Connect to a GaussDB(DWS) Table", + "uri":"dli_03_0253.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"240" + }, + { + "desc":"A cross-source GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job execu", + "product_code":"dli", + "title":"An Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated", + "uri":"dli_03_0254.html", + "doc_type":"usermanual", + "p_code":"221", + "code":"241" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Change History", + "uri":"dli_01_00006.html", + "doc_type":"usermanual", + "p_code":"", + "code":"242" + } +] \ No newline at end of file diff --git a/docs/dli/umn/PARAMETERS.txt b/docs/dli/umn/PARAMETERS.txt new file mode 100644 index 00000000..6da8d5f0 --- /dev/null +++ b/docs/dli/umn/PARAMETERS.txt @@ -0,0 +1,3 @@ +version="" +language="en-us" +type="" \ No newline at end of file diff --git a/docs/dli/umn/dli_01_00006.html b/docs/dli/umn/dli_01_00006.html new file mode 100644 index 00000000..936c0e4c --- /dev/null +++ b/docs/dli/umn/dli_01_00006.html @@ -0,0 +1,20 @@ + + +

Change History

+
+
+ + + + + + + +

Released On

+

What's New

+

2023-01-30

+

This issue is the first official release.

+
+
+
+ diff --git a/docs/dli/umn/dli_01_0001.html b/docs/dli/umn/dli_01_0001.html new file mode 100644 index 00000000..f1620ca5 --- /dev/null +++ b/docs/dli/umn/dli_01_0001.html @@ -0,0 +1,15 @@ + + +

Job Management

+
+
+ +
+ diff --git a/docs/dli/umn/dli_01_0002.html b/docs/dli/umn/dli_01_0002.html new file mode 100644 index 00000000..c2e2316d --- /dev/null +++ b/docs/dli/umn/dli_01_0002.html @@ -0,0 +1,61 @@ + + +

Creating and Submitting a Spark SQL Job

+

You can use DLI to submit a Spark SQL job to query data. The general procedure is as follows:

+

Step 1: Logging in to the Cloud Platform

+

Step 2: Uploading Data to OBS

+

Step 3: Logging In to the DLI Management Console

+

Step 4: Creating a Queue

+

Step 5: Creating a Database

+

Step 6: Creating a Table

+

Step 7: Querying Data

+

The following illustrates how to query OBS data using DLI. Operations to query DLI data are similar.

+

Step 1: Logging in to the Cloud Platform

  1. Open the DLI homepage.
  2. On the login page, enter the username and password, and click Log In.
+
+

Step 2: Uploading Data to OBS

DLI allows you to query data stored on OBS. Before querying the data, you need to upload data to OBS.

+
  1. In the services displayed, click Object Storage Service (OBS) in Storage.
  2. The OBS console page is displayed.
  3. Create a bucket. The bucket name must be globally unique. In this example, assume that the bucket name is obs1.
    1. Click Create Bucket in the upper right corner.
    2. On the displayed Create Bucket page, enter the Bucket Name. Retain the default values for other parameters or set them as required.
    3. Click Create Now.
    +
  4. Click obs1 to switch to the Overview page.
  5. In the left navigation pane, click Objects. Click Upload Object. In the displayed dialog box, drag files or folders to upload or add file, for example, sampledata.csv to the file upload box. Then, click Upload.
    You can create a sampledata.txt file, copy the following content separated by commas (,), and save the file as sampledata.csv.
    12,test
    +
    +

    After the file is uploaded successfully, the file path is obs://obs1/sampledata.csv.

    +

    For more information about OBS operations, see the Object Storage Service Console Operation Guide.

    +

    For more information about the tool, see the OBS Tool Guide.

    +

    You are advised to use an OBS tool, such as OBS Browser+, to upload large files because OBS Console has restrictions on the file size and quantity.

    +
    • OBS Browser+ is a graphical tool that provides complete functions for managing your buckets and objects in OBS.
    +
    +
+
+

Step 3: Logging In to the DLI Management Console

  1. .
  2. The DLI management console page is displayed. If you log in to the DLI management console for the first time, you need to be authorized to access OBS.

    Note: For this guide, you need at least the Tenant Administrator (Global service) permission.

    +
+
+

Step 4: Creating a Queue

A queue is the basis for using DLI. Before executing an SQL job, you need to create a queue.

+ +
+

Step 5: Creating a Database

Before querying data, create a database, for example, db1.

+

The default database is a built-in database. You cannot create the database named default.

+
+
  1. On the DLI management console, click SQL Editor in the navigation pane on the left. The SQL Editor page is displayed.
  2. In the editing window on the right of the SQL Editor page, enter the following SQL statement and click Execute. Read and agree to the privacy agreement, and click OK.
    create database db1;
    +

    After database db1 is successfully created, db1 will be displayed in the Database list.

    +

    When you execute a query on the DLI management console for the first time, you need to read the privacy agreement. You can perform operations only after you agree to the agreement. For later queries, you will not need to read the privacy agreement again.

    +
    +
+
+

Step 6: Creating a Table

After database db1 is created, create a table (for example, table1) containing data in the sample file obs://obs1/sampledata.csv stored on OBS in db1.

+
  1. In the SQL editing window of the SQL Editor page, select the default queue and database db1.
  2. Enter the following SQL statement in the job editor window and click Execute:
    create table table1 (id int, name string) using csv options (path 'obs://obs1/sampledata.csv');
    +

    After the table is created, click the Databases tab and then select db1. The created table table1 is displayed in the Table area.

    +
+
+

Step 7: Querying Data

After performing the preceding steps, you can start querying data.

+
  1. In the Table tab on the SQL Editor page, double-click the created table table1. The SQL statement is automatically displayed in the SQL job editing window in the right pane. Run following statement to query 1,000 records in the table1 table:
    select * from db1.table1 limit 1000;
    +
  2. Click Execute. The system starts the query.

    After the SQL statement is executed successfully, you can view the query result in View Result under the SQL job editing window.

    +
+
+
+
+
+
Parent topic: Getting Started
+
+
+ diff --git a/docs/dli/umn/dli_01_0003.html b/docs/dli/umn/dli_01_0003.html new file mode 100644 index 00000000..c3df2daa --- /dev/null +++ b/docs/dli/umn/dli_01_0003.html @@ -0,0 +1,59 @@ + + +

Overview

+

The enhanced datasource connection uses VPC peering at the bottom layer to directly connect the VPC network between the DLI cluster and the destination datasource. Data is exchanged in point-to-point mode. The enhanced datasource connection function supports all cross-source services implemented by DLI, including CloudTable HBase, CloudTableOpenTSDB, MRS OpenTSDB, DWS, RDS, CSS, DCS, and DDS. In addition, UDFs, Spark jobs, and Flink jobs can be used to access self-built data sources.

+ +
  • The CIDR block of the DLI queue bound with a datasource connection cannot overlap with that of the data source.
  • Datasource connections cannot be created for the default queue.
  • To access a datasource connection table, you need to use the queue for which a datasource connection has been created.
  • The preview function is not supported for datasource tables.
+
+

The enhanced datasource scenario provides the following functions:

+ +

Enhanced Datasource Connection Page

This page displays all enhanced datasource connections. If there are a large number of connections, they are displayed on multiple pages.

+ +
+ + + + + + + + + + + + + + + + + + + + + + +
Table 1 Datasource connection list parameters

Parameter

+

Description

+

Connection Name

+

Name of the created datasource connection.

+

Connection Status

+

Status of a datasource connection. Currently, the console displays only connections in the Active state.

+

VPC

+

VPC used by the destination data source.

+

Subnet

+

Subnet used by the destination data source.

+

Created

+

Time when a connection is created. The connections in the connection list can be displayed according to the creation time in ascending or descending order.

+

Operation

+
  • Delete Connection: Delete a created enhanced datasource connection.
  • Manage Route: Used to display the custom route information of the enhanced datasource connection.
  • More:
    • Modify Host: Customize the IP address corresponding to the host or domain name.
    • Bind Queue: Bind a queue to an enhanced datasource connection.
    • Unbind Queue: Unbind an enhanced datasource connection from a queue.
    • Add Route: Add a custom route for the enhanced datasource connection.
    • Delete Route: Delete a custom route for an enhanced datasource connection.
    • Manage Permissions: Authorize or reclaim permissions for other projects.
    +
+
+
+
+
+
+
+
Parent topic: Enhanced Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0004.html b/docs/dli/umn/dli_01_0004.html new file mode 100644 index 00000000..1d24cba4 --- /dev/null +++ b/docs/dli/umn/dli_01_0004.html @@ -0,0 +1,13 @@ + + +

Data Management

+
+
+ +
+ diff --git a/docs/dli/umn/dli_01_0005.html b/docs/dli/umn/dli_01_0005.html new file mode 100644 index 00000000..f6a4f3b3 --- /dev/null +++ b/docs/dli/umn/dli_01_0005.html @@ -0,0 +1,218 @@ + + +

Creating a Database or a Table

+

Definition of Database and Table in DLI

A database, built on the computer storage device, is a data warehouse where data is organized, stored, and managed based on its structure.

+

The table is an important part of the database. It consists of rows and columns. Each column functions as a field. Each value in a field (column) represents a type of data.

+

The database is a framework and the table contains data content. A database has one or more tables.

+

You can create databases and tables on the management console or using SQL statements. This section describes how to create a database and a table on the management console.

+

A view can be created only by using SQL statements. You cannot create a view on the Create Table page.

+
+
+

Precautions

+
+

Creating a Database

  1. You can create a database on either the Data Management page or the SQL Editor page.

    • To create a database on the Data Management page:
      1. On the left of the management console, choose Data Management > Databases and Tables.
      2. In the upper right corner of the Databases and Tables page, click Create Database to create a database.
      +
    • To create a database on the SQL Editor page:
      1. On the left of the management console, click SQL Editor.
      2. In the navigation pane on the left, click beside Databases.
      +
    +

  2. In the displayed Create Database dialog box, specify Name and Description by referring to Table 1.

    +

    + + + + + + + + + + + + + +
    Table 1 Description

    Parameter

    +

    Description

    +

    Example

    +

    Database Name

    +
    • The database name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
    • The database name is case insensitive and cannot be left blank.
    • The length of the database name cannot exceed 128 characters.
    +
    NOTE:

    The default database is a built-in database. You cannot create the database named default.

    +
    +

    DB01

    +

    Description

    +

    Description of a database.

    +

    -

    +
    +
    +

  3. Click OK.

    After a database is created, you can view and select the database for use on the Databases and Tables page or SQL Editor page.

    +

+
+

Creating a Table

Before creating a table, ensure that a database has been created.

+
  1. You can create a table on either the Databases and Tables page or the SQL Editor page.

    Datasource connection tables, such as View tables, HBase (MRS) tables, OpenTSDB (MRS) tables, DWS tables, RDS tables, and CSS tables, cannot be created. You can use SQL to create views and datasource connection tables. For details, see sections Creating a View and Creating a Datasource Connection Table in the Data Lake Insight SQL Syntax Reference.

    +
    +
    • To create a table on the Data Management page:
      1. On the left of the management console, choose Data Management > Databases and Tables.
      2. On the Databases and Tables page, select the database for which you want to create a table. In the Operation column, click More > Create Table to create a table in the current database.
      +
    • To create a table on the SQL Editor page:
      1. On the left of the management console, click SQL Editor.
      2. In the navigation pane of the displayed SQL Editor page, click Databases. You can create a table in either of the following ways:
        • Click a database name. In the Tables area, click on the right to create a table in the current database.
        • Click on the right of the database and choose Create Table from the shortcut menu to create a table in the current database.
        +
      +
    +

  2. In the displayed Create Table dialog box, set parameters as required.

    • If you set Data Location to DLI, set related parameters by referring to Table 2.
    • If you set Data Location to OBS, set related parameters by referring to Table 2 and Table 3. +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Table 2 Common parameters

      Parameter

      +

      Description

      +

      Example

      +

      Table Name

      +
      • The table name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
      • The table name is case insensitive and cannot be left unspecified.
      • The table name can contain the dollar sign ($). An example value is $test.
      • The length of the table name cannot exceed 128 characters.
      +

      table01

      +

      Data Location

      +

      Data storage location. Currently, DLI and OBS are supported.

      +

      OBS

      +

      Description

      +

      Description of the table.

      +

      -

      +

      Column Type

      +

      Available values: Normal or Partition

      +

      Normal

      +

      Column

      +

      Name of a column in a table. The column name must contain at least one letter and can contain underscores (_). It cannot contain only digits.

      +

      You can select Normal or Partition. Partition columns are dedicated to partition tables. User data is partitioned to improve query efficiency.

      +
      NOTE:

      The column name is case-insensitive and must be unique.

      +
      +

      name

      +

      Type

      +

      Data type of a column. This parameter corresponds to Column Name.

      +
      • string: The data is of the string type.
      • int: Each integer is stored on four bytes.
      • date: The value ranges from 0000-01-01 to 9999-12-31.
      • double: Each number is stored on eight bytes.
      • boolean: Each value is stored on one byte.
      • decimal: The valid bits are positive integers between 1 to 38, including 1 and 38. The decimal digits are integers less than 10.
      • smallint/short: The number is stored on two bytes.
      • bigint/long: The number is stored on eight bytes.
      • timestamp: The data indicates a date and time. The value can be accurate to six decimal points.
      • float: Each number is stored on four bytes.
      • tinyint: Each number is stored on one byte. Only OBS tables support this data type.
      +

      string

      +

      Column Description

      +

      Description of a column.

      +

      -

      +

      Operation

      +
      • Add Column
      • Delete
        NOTE:

        If the table to be created includes a great number of columns, you are advised to use SQL statements to create the table or import column information from the local EXCEL file.

        +
        +
      +

      -

      +
      +
      + +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Table 3 Parameter description when Data Location is set to OBS

      Parameter

      +

      Description

      +

      Example

      +

      Data Format

      +

      DLI supports the following data formats:

      +
      • Parquet: DLI can read non-compressed data or data that is compressed using Snappy and gzip.
      • CSV: DLI can read non-compressed data or data that is compressed using gzip.
      • ORC: DLI can read non-compressed data or data that is compressed using Snappy.
      • JSON: DLI can read non-compressed data or data that is compressed using gzip.
      • Avro: DLI can read uncompressed Avro data.
      +

      CSV

      +

      Storage Path

      +

      Enter or select an OBS path. The path can be a folder or a path.

      +
      NOTE:

      If you need to import data stored in OBS to the OBS table, set this parameter to the path of a folder. If the table creation path is a file, data fails to be imported.

      +
      +

      obs://obs1/sampledata.csv

      +

      Table Header: No/Yes

      +

      This parameter is valid only when Data Format is set to CSV. Whether the data source to be imported contains the table header.

      +

      Click Advanced Settings and select the check box next to Table Header: No. If the check box is selected, the table header is displayed. If the check box is deselected, no table header is displayed.

      +

      -

      +

      User-defined Delimiter

      +

      This parameter is valid only when Data Format is set to CSV and you select User-defined Delimiter.

      +

      The following delimiters are supported:

      +
      • Comma (,)
      • Vertical bar (|)
      • Tab character (\t)
      • Others: Enter a user-defined delimiter.
      +

      Comma (,)

      +

      User-defined Quotation Character

      +

      This parameter is valid only when Data Format is set to CSV and you select User-defined Quotation Character.

      +

      The following quotation characters are supported:

      +
      • Single quotation mark (')
      • Double quotation marks (")
      • Others: Enter a user-defined quotation character.
      +

      Single quotation mark (')

      +

      User-defined Escape Character

      +

      This parameter is valid only when Data Format is set to CSV and you select User-defined Escape Character.

      +

      The following escape characters are supported:

      +
      • Backslash (\)
      • Others: Enter a user-defined escape character.
      +

      Backslash (\)

      +

      Date Format

      +

      This parameter is valid only when Data Format is set to CSV or JSON.

      +

      This parameter specifies the format of the date in the table and is valid only Advanced Settings is selected. The default value is yyyy-MM-dd. For definition of characters involved in the date pattern, see Table 3 in the

      +

      2000-01-01

      +

      Timestamp Format

      +

      This parameter is valid only when Data Format is set to CSV or JSON.

      +

      This parameter specifies the format of the timestamp in the table and is valid only Advanced Settings is selected. The default value is yyyy-MM-dd HH:mm:ss. For definition of characters involved in the time pattern, see Table 3 in the

      +

      2000-01-01 09:00:00

      +
      +
      +
    +

  3. Click OK.

    After a table is created, you can view and select the table for use on the Data Management page or SQL Editor page.

    +

  4. (Optional) After a DLI table is created, you can decide whether to directly import data to the table.
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0006.html b/docs/dli/umn/dli_01_0006.html new file mode 100644 index 00000000..dfcf005e --- /dev/null +++ b/docs/dli/umn/dli_01_0006.html @@ -0,0 +1,70 @@ + + +

Creating, Querying, and Deleting an Enhanced Datasource Connection

+

Creating an Enhanced Datasource Connection

The following describes how to create a datasource HBase connection for MRS.

+

Only enhanced datasource connection to MRS HBase is supported.

+
+
  1. Apply for a cluster in MRS.

    If a cluster is available, you do not need to apply for one.

    +
  2. In the navigation pane of the DLI management console, choose Datasource Connections.
  3. Click the Enhanced tab and click Create in the upper left corner.

    Enter the Connection Name, select the Bind Queue (optional), VPC, and Subnet, and enter the Host Information (optional). For details about the parameters, see Table 1.

    +

    + +
    + + + + + + + + + + + + + + + + + + + +
    Table 1 Parameter description

    Parameter

    +

    Description

    +

    Connection Name

    +

    Name of the created datasource connection.

    +
    • The name can contain only letters, digits, and underscores (_). The parameter must be specified.
    • A maximum of 64 characters are allowed.
    +

    Bound Queue

    +

    (Optional) Bind a queue that requires datasource connections.

    +
    NOTE:

    Before using an enhanced datasource connection, you must bind a queue and ensure that the VPC peering connection is in the Active state.

    +
    +

    VPC

    +

    VPC used by the destination data source.

    +

    Subnet

    +

    Subnet used by the destination data source.

    +

    Host Information

    +

    (Optional) When connecting to the HBase cluster of MRS, enter the host name and IP address of the ZooKeeper instance. Enter one record in each line. The format is as follows: IP address Host name/Domain name.

    +

    To obtain the host name and IP address of the MRS cluster, perform the following steps (with MRS3.x as an example):

    +
    1. Log in to the MRS management console.
    2. In the navigation pane, choose Clusters > Active Clusters. Click the target cluster name to access the cluster details page.
    3. Click Component Management.
    4. Click Zookeeper.
    5. Click the Instance tab to view the corresponding service IP address. You can select any service IP address.
    +
    NOTE:

    If the MRS cluster has multiple IP addresses, enter any service IP address when creating a datasource connection.

    +
    +
    +
    +
  4. Click OK.
  5. To connect to Kafka, DWS, and RDS instances, add security group rules for the DLI network segment to the security group where the instances belong. The following uses RDS as an example:
    1. Choose Resources > Queue Management in the navigation pane. Select the target queue, and click to expand the row containing the target queue to view its CIDR block.
    2. On the Instance Management page of the RDS console, click the instance name. In the Connection Information area, view the port number of the RDS database instance.
    3. In the Connection Information area locate the Security Group and click the group name to switch to the security group management page. Select the Inbound Rules tab and click Add Rule. Set the priority to 1, protocol to TCP, port to the database port number, and source to the CIDR block of the DLI queue. Click OK.
    +
  6. Test the connectivity between the DLI queue and the connection instance. The following example describes how to test the connectivity between DLI and an RDS DB instance.
    1. On the Instance Management page, click the target DB instance. On the displayed page, locate the Connection Information pane and view the floating IP address. In the Connection Information pane, locate the Database Port to view the port number of the RDS DB instance.
    2. Go back to the DLI console. On the Resources > Queue Management page, locate the target queue. In the Operation column, click More > Test Address Connectivity.
    3. Enter the connection address of the RDS DB instance and port number in the format of IP address:port to test the network connectivity.
    +
+
+

Querying an Enhanced Datasource Connection

On the Enhanced tab page, you can enter the keyword of a connection name in the search box to search for the matching connection.

+
+

Viewing Details

On the Enhanced tab page, select a connection and click to view its details. The connection ID and host information are displayed.

+
+

Deleting an Enhanced Datasource Connection

On the Enhanced tab page, click Delete Connection in the Operation column to delete an unnecessary connection.

+

A connection with Connection Status of Creating cannot be deleted.

+
+
+
+
+
+
Parent topic: Enhanced Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0007.html b/docs/dli/umn/dli_01_0007.html new file mode 100644 index 00000000..414e1795 --- /dev/null +++ b/docs/dli/umn/dli_01_0007.html @@ -0,0 +1,16 @@ + + +

Previewing Data

+

The Preview page displays the first 10 records in the table.

+

Procedure

You can preview data on either the Data Management page or the SQL Editor page.
  • To preview data on the Data Management page:
    1. On the left of the management console, choose Data Management > Databases and Tables.
    2. On the displayed Data Management page, click the name of the database where the target table whose data you want to export resides to switch to the Manage Tables page.
    3. Click More in the Operation column of the target table and select View Properties.
    4. Click the Preview tab to preview the table data.
    +
  • To preview data on the SQL Editor page:
    1. On the left of the management console, click SQL Editor.
    2. In the navigation pane of the displayed SQL Editor page, click Databases.
    3. Click the corresponding database name to view the tables in the database.
    4. Click on the right of the corresponding table, choose View Properties from the list menu, and click the Preview tab to preview the data of the table.
    +
+
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0008.html b/docs/dli/umn/dli_01_0008.html new file mode 100644 index 00000000..846fa36c --- /dev/null +++ b/docs/dli/umn/dli_01_0008.html @@ -0,0 +1,17 @@ + + +

Viewing Metadata

+

Metadata Description

+
+

Procedure

You can view metadata on either the Data Management page or the SQL Editor page.
  • To view metadata on the Data Management page:
    1. On the left of the management console, choose Data Management > Databases and Tables.
    2. On the displayed Data Management page, click the name of the database where the target table whose data you want to export resides to switch to the Manage Tables page.
    3. Click More in the Operation column of the target table and select View Properties. In the Metadata tab, view the metadata of the table.
    +
  • To view metadata on the SQL Editor page:
    1. On the left of the management console, click SQL Editor.
    2. In the navigation pane of the displayed SQL Editor page, click Databases.
    3. Click the corresponding database name to view the tables in the database.
    4. Click on the right of the table and choose View Properties from the shortcut menu. On the Metadata tab page, view the metadata of the table.
    +
+
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0009.html b/docs/dli/umn/dli_01_0009.html new file mode 100644 index 00000000..7cef7491 --- /dev/null +++ b/docs/dli/umn/dli_01_0009.html @@ -0,0 +1,68 @@ + + +

Binding and Unbinding a Queue

+

Constraints

+
+

Binding a Queue

Before using an enhanced datasource connection, you must bind a queue and ensure that the VPC peering connection is in the Active state.

+ + +
+

Viewing Details about a Bound Queue

On the Enhanced tab page, select a connection and click the connection name to view the information about the bound queue. +
+ + + + + + + + + + + + + + + + + + + +
Table 1 Parameters in the details list of datasource connection queues

Parameter

+

Description

+

VPC Peering ID

+

ID of the VPC peering connection created in the cluster to which the queue belongs.

+
NOTE:

A VPC peering connection is created for each queue bound to an enhanced datasource connection. The VPC peering connection is used for cross-VPC communication. Ensure that the security group used by the data source allows access from the DLI queue CIDR block, and do not delete the VPC peering connection during the datasource connection.

+
+

Name

+

Name of a bound queue.

+

Connection Status

+

Datasource connection status. The following three statuses are available:

+
  • Creating
  • Active
  • Failed
+
NOTE:

If the connection status is Failed, click on the left to view the detailed error information.

+
+

Updated

+

Time when a connection is updated. The connections in the connection list can be displayed according to the update time in ascending or descending order.

+

Operation

+

Unbind Queue: This operation is used to unbind a datasource connection from a queue.

+
+
+
+
+

Unbinding a Queue

If you do not need to use an enhanced datasource connection, you can unbind the queue from it to release resources.

+ + +
+
+
+
+
Parent topic: Enhanced Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0010.html b/docs/dli/umn/dli_01_0010.html new file mode 100644 index 00000000..5d31126e --- /dev/null +++ b/docs/dli/umn/dli_01_0010.html @@ -0,0 +1,72 @@ + + +

Exporting Data from DLI to OBS

+

You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.

+

Precautions

+
+

Procedure

  1. You can export data on either the Data Management page or the SQL Editor page.

    • To export data on the Data Management page:
      1. On the left of the management console, choose Data Management > Databases and Tables.
      2. Click the name of the database corresponding to the table where data is to be exported to switch to the Manage Tables page.
      3. Select the corresponding table (DLI table) and choose More > Export in the Operation column. The Export Data page is displayed.
      +
    • To export data on the SQL Editor page:
      1. On the left of the management console, click SQL Editor.
      2. In the navigation tree on the left, click Databases to see all databases. Click the database name corresponding to the table to which data is to be exported. The tables are displayed.
      3. Click on the right of the managed table (DLI table) whose data is to be exported, and choose Export from the shortcut menu.

        +
      +
    +

  2. In the displayed Export Data dialog box, specify parameters by referring to Table 1.

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 1 Parameter description

    Parameter

    +

    Description

    +

    Databases

    +

    Database where the current table is located.

    +

    Table Name

    +

    Name of the current table.

    +

    Data Format

    +

    Format of the file storing data to be exported. Formats other than JSON will be supported in later versions.

    +

    Queue

    +

    Select a queue.

    +

    Compression Format

    +

    Compression format of the data to be exported. The following compression formats are supported:

    +
    • none
    • bzip2
    • deflate
    • gzip
    +

    Storage Path

    +
    • Enter or select an OBS path.
    • The export path must be a folder that does not exist in the OBS bucket. Specifically, you need to create a folder in the target OBS directory.
    • The folder name cannot contain the special characters of \/:*? "<>|, and cannot start or end with a dot (.).
    +

    Export Mode

    +

    Storage mode of the data to be exported.

    +
    • New OBS directory: If the specified export directory exists, an error is reported and the export operation cannot be performed.
    • Existing OBS directory (Overwritten): If you create a file in the specified directory, the existing file will be overwritten.
    +

    Table Header: No/Yes

    +

    Whether the data to be exported contains the table header.

    +
    +
    +

  3. Click OK.
  4. (Optional) You can view the job status (indicated by Status), statements (indicated by Statement), and other information about exporting jobs on the SQL Jobs page.

    1. Select EXPORT from the Job Type drop-down list box and specify the time range for exporting data. The jobs meeting the requirements are displayed in the job list.
    2. Click to view details about an exporting job.
    +

+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0011.html b/docs/dli/umn/dli_01_0011.html new file mode 100644 index 00000000..82363e7a --- /dev/null +++ b/docs/dli/umn/dli_01_0011.html @@ -0,0 +1,24 @@ + + +

Deleting a Database or a Table

+

You can delete unnecessary databases and tables based on actual conditions.

+

Precautions

+
+

Deleting a Table

You can delete a table on either the Data Management page or the SQL Editor page.
  • Delete the table on the Data Management page.
    1. On the left of the management console, choose Data Management > Databases and Tables.
    2. Locate the row where the database whose tables you want to delete, click the database name to switch to the Table Management page.
    3. Locate the row where the target table locates and click More > Delete in the Operation column.
    4. In the displayed dialog box, click Yes.
    +
  • Delete a table on the SQL Editor page.
    1. On the top menu bar of the DLI management console, click SQL Editor.
    2. In the navigation tree on the left, click Databases. Click the name of a database where the table you want belongs. The tables of the selected database are displayed.
    3. Click on the right of the table and choose Delete from the shortcut menu.
    4. In the dialog box that is displayed, click OK.
    +
+
+
+

Deleting a Database

  1. On the left of the management console, choose Data Management > Databases and Tables.
  2. Locate the row where the target database locates and click More > Drop Database in the Operation column.

    You cannot delete databases that contain tables. To delete a database containing tables, delete the tables first.

    +
    +
  3. In the displayed dialog box, click Yes.
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0012.html b/docs/dli/umn/dli_01_0012.html new file mode 100644 index 00000000..b61c0969 --- /dev/null +++ b/docs/dli/umn/dli_01_0012.html @@ -0,0 +1,27 @@ + + +

Queue Management

+
+
+ +
+ diff --git a/docs/dli/umn/dli_01_0013.html b/docs/dli/umn/dli_01_0013.html new file mode 100644 index 00000000..d9d4f426 --- /dev/null +++ b/docs/dli/umn/dli_01_0013.html @@ -0,0 +1,23 @@ + + +

Modifying Host Information

+

Modifying Hosts Information

Method 1: Copy hosts information in /etc/hosts of an MRS node.

+
  1. Log in to any MRS node as the root user.
  2. Run the following command to obtain MRS hosts information. Copy and save the information.

    cat /etc/hosts

    +
    Figure 1 Obtaining hosts information
    +

  3. Log in to the DLI console, choose Datasource Connections > Enhanced. On the Enhanced tab page, select a connection and click Modify Host. In the displayed dialog box, paste the host information copied in 2. Click OK.
+

Method 2: Log in to FusionInsight Manager to obtain MRS hosts information.

+
  1. Log in to FusionInsight Manager.
  2. On FusionInsight Manager, click Hosts. On the Hosts page, obtain the host names and service IP addresses of the MRS hosts.
  3. Log in to the DLI console, choose Datasource Connections > Enhanced. On the Enhanced tab page, select a connection and click Modify Host. In the displayed dialog box, enter the host information. Click OK.

    The host information is in the format of Service IP address Host name. Specify the IP addresses and host names obtained in 2, and separate multiple records by line breaks.

    +

    For example:

    +

    192.168.0.22 node-masterxxx1.com

    +

    192.168.0.23 node-masterxxx2.com

    +

+
  • After this configuration, all hosts information is overwritten.
  • A host name or domain name can contain a maximum of 128 characters, including digits, letters, underscores (_), hyphens (-), and periods (.). It must start with a letter.
  • The host information is IP address Host name/Domain name format. Separate multiple records by line breaks.
+
+
+
+
+
+
Parent topic: Enhanced Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0014.html b/docs/dli/umn/dli_01_0014.html new file mode 100644 index 00000000..c65aac77 --- /dev/null +++ b/docs/dli/umn/dli_01_0014.html @@ -0,0 +1,38 @@ + + +

Custom Route Information

+

Customizing Route Information

After an enhanced datasource connection is created and bound to a queue, the system automatically configures route information. You can also add a custom route for the queue to which the enhanced connection is bound.

+ + + +
+
+
+
+
Parent topic: Enhanced Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0015.html b/docs/dli/umn/dli_01_0015.html new file mode 100644 index 00000000..115e2f5d --- /dev/null +++ b/docs/dli/umn/dli_01_0015.html @@ -0,0 +1,44 @@ + + +

Queue Permission Management

+

Scenarios

+
+

Operations

  1. On the top menu bar of the DLI management console, click Resources > Queue Management.
  2. Select the queue to be configured and choose Manage Permissions in the Operation column. The User Permission Info area displays the list of users who have permissions on the queue.

    You can assign queue permissions to new users, modify permissions for users who have some permissions of a queue, and revoke all permissions of a user on a queue.

    +
    • Assign permissions to a new user.
      A new user does not have permissions on the queue.
      1. Click Set Permission on the right of User Permissions page. The Set Permission dialog box is displayed.
      2. Specify Username and select corresponding permissions.
      3. Click OK.

        Table 1 describes the related parameters.

        + +
        + + + + + + + + + + +
        Table 1 Parameter description

        Parameter

        +

        Description

        +

        Username

        +

        Name of the authorized user.

        +
        NOTE:

        The username is an existing IAM user name and has logged in to the DLI management console.

        +
        +

        Permission Settings

        +
        • Delete Queues: This permission allows you to delete the queue.
        • Submit Jobs: This permission allows you to submit jobs using this queue.
        • Terminate Jobs: This permission allows you to terminate jobs submitted using this queue.
        • Grant Permission: This permission allows you to grant queue permissions to other users.
        • Revoke Permission: This permission allows you to revoke the queue permissions that other users have but cannot revoke the queue owner's permissions.
        • View Other User's Permissions: This permission allows you to view the queue permissions of other users.
        • Restart Queues: This permission allows you to restart queues.
        • Modify Queue Specifications: This permission allows you to modify queue specifications.
        +
        +
        +
      +
      +
    • To assign or revoke permissions of a user who has some permissions on the queue, perform the following steps:
      1. In the list under User Permission Info for a queue, select the user whose permissions need to be modified and click Set Permission in the Operation column.
      2. In the displayed Set Permission dialog box, modify the permissions of the current user. Table 1 lists the detailed permission descriptions.

        If all options under Set Permission are gray, you are not allowed to change permissions on this queue. You can apply to the administrator, queue owner, or other authorized users for queue permission granting and revoking.

        +
      3. Click OK.
      +
    • To revoke all permission of a user on the queue, perform the following steps:

      In the user list under Permission Info, select the user whose permission needs to be revoked and click Revoke Permission under Operation. In the Revoke Permission dialog box, click OK. All permissions on this queue are revoked.

      +
    +

+
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0016.html b/docs/dli/umn/dli_01_0016.html new file mode 100644 index 00000000..f99b73a5 --- /dev/null +++ b/docs/dli/umn/dli_01_0016.html @@ -0,0 +1,17 @@ + + +

Deleting a Queue

+

You can delete a queue based on actual conditions.

+
  • This operation will fail if there are jobs in the Submitting or Running state on this queue.
  • Deleting a queue does not cause table data loss in your database.
+
+

Procedure

  1. On the left of the DLI management console, click Resources >Queue Management.
  2. Locate the row where the target queue locates and click Delete in the Operation column.

    If Delete in the Operation column is gray, the current user does not have the permission of deleting the queue. You can apply to the administrator for the permission.

    +
    +

  3. In the displayed dialog box, click OK.
+
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0017.html b/docs/dli/umn/dli_01_0017.html new file mode 100644 index 00000000..d4587d4b --- /dev/null +++ b/docs/dli/umn/dli_01_0017.html @@ -0,0 +1,142 @@ + + +

SQL Job Management

+

SQL jobs allow you to execute SQL statements entered in the SQL job editing window, import data, and export data.

+

SQL job management provides the following functions:

+ +

SQL Jobs page

On the Overview page of the DLI console, click SQL Jobs to go to the SQL job management page. Alternatively, you can click Job Management > SQL Jobs. The job list displays all SQL jobs. If there are a large number of jobs, they will be displayed on multiple pages. You can switch to the specified page as needed. DLI allows you to view jobs in all statuses. By default, jobs in the job list are displayed in descending order of the job creation time.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 SQL Job management parameters

Parameter

+

Description

+

Queues

+

Name of the queue to which a job belongs

+

Username

+

Name of the user who executed the job.

+

Type

+

Job type. The following types are supported:

+
  • IMPORT: A job that imports data to DLI
  • EXPORT: A job that exports data from DLI
  • DCL: Conventional DCLs and operations related to queue permissions
  • DDL:Conventional DDLs, including creating and deleting databases and tables
  • QUERY: A job that querys data by running SQL statements
  • INSERT: A job that inserts data by running SQL statements
  • UPDATE: A job that updates data.
  • DELETE: A job that deletes a SQL job.
  • DATA_MIGRATION: A job that migrates data.
  • RESTART_QUEUE: A job that restarts a queue.
  • SCALE_QUEUE: A job that changes queue specifications, including sale-out and scale-in.
+

Status

+

Job status. Possible values are as follows:

+
  • Submitting
  • Running
  • Finished
  • Canceled
  • Failed
  • Scaling
+

Query

+

SQL statements for operations such as exporting and creating tables

+

You can click to copy the query statement.

+

Duration

+

Running duration of a job

+

Created

+

Time when a job is created. Jobs can be displayed in ascending or descending order of the job creation time.

+

Operation

+
  • Edit: Edit the job.
  • Cancel
    • You can terminate a job only when the job is in Submitting or Running status.
    • A job whose status is Finished, Failed, or Canceled cannot be terminated.
    • If the Cancel button is gray, you are not allowed to perform this operation.
    +
  • Re-execute: Execute the job again.
  • SparkUI: Display the Spark job execution page.
    NOTE:
    • When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click SparkUI before the cluster is created, an empty projectID will be cached. The SparkUI page cannot be displayed. You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check SparkUI.
    • Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.
    +
    +
+
  • In addition to the preceding operations, the following operations are available for QUERY jobs and asynchronous DDL jobs.
    • View Result: View the job running result.
    • Export Result: Export the job running result to the created OBS bucket. For details, see section Exporting Query Results.
    +
  • In addition to the preceding operations, the EXPORT job also includes the following operations:
    • Download
    +
  • View Log: Save job logs to the temporary OBS bucket created by DLI.
  • Export Log: Export logs to the created OBS bucket. If the job is in the Running state, logs cannot be exported.
+
NOTE:

To export the logs, you need to obtain the permission to create an OBS bucket.

+

Log archiving and export are not available for synchronization jobs and jobs running on the default queue.

+
+
+
+
+

Searching for a Job

On the SQL Jobs page, you can search jobs with any of the following operations.

+ +
+

Viewing Job Details

On the SQL Jobs page, you can click in front of a job record to view details about the job.

+

The following describes how to load data, create a table, and select a job. The job details vary depending on the job types, status, and configuration options.

+ +
+

Terminating a Job

On the SQL Jobs page, you can click Terminate in the Operation column to stop a submitting or running job.

+
+

Exporting Query Results

A maximum of 1000 records can be displayed in the query result on the console. To view more or all data, you can export the data to OBS. The procedure is as follows:

+

You can export results on the SQL Jobs page or the SQL Editor page.

+ +

If no column of the numeric type is displayed in the query result, the result cannot be exported.

+
+

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 Exporting parameters

Parameter

+

Description

+

Data Format

+

Format of the exported query result file. Formats other than JSON will be supported in later versions.

+

Queues

+

The queue where the jobs are executed. SQL jobs can be executed only in SQL queues. For details about how to create a queue, see Creating a Queue.

+

Compression Format

+

Compression format of the data to be exported. The following options are supported:

+
  • none
  • bzip2
  • deflate
  • gzip
+

Storage Path

+

OBS path to store the result.

+
NOTE:
  • After selecting an OBS bucket, enter a name for the folder. If the folder does not exist, it will be created in OBS.
  • The folder name cannot contain the special characters of \ / : * ? "< > |, and cannot start or end with a dot (.).
+
+

Export Mode

+

Mode for saving the exported query result data.

+
  • New OBS directory: If the specified export directory exists, an error is reported and the export operation cannot be performed.
  • Existing OBS directory (Overwritten): If you create a file in the specified directory, the existing file will be overwritten.
+

Number of Results

+

Number of exported query results. If no value is entered or the value is 0, all results are exported.

+

Table Header

+

Whether the data to be exported contains table headers.

+
+
+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0018.html b/docs/dli/umn/dli_01_0018.html new file mode 100644 index 00000000..f84ac7e1 --- /dev/null +++ b/docs/dli/umn/dli_01_0018.html @@ -0,0 +1,22 @@ + + +

Enhanced Datasource Connection Permission Management

+

Permission Management

You can grant or revoke permissions for enhanced datasource connections of other projects.

+ +
+
+
+
+
Parent topic: Enhanced Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0021.html b/docs/dli/umn/dli_01_0021.html new file mode 100644 index 00000000..21a1da0b --- /dev/null +++ b/docs/dli/umn/dli_01_0021.html @@ -0,0 +1,140 @@ + + +

SQL Template Management

+

To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not need to compile SQL statements. Instead, you can directly perform the SQL operations using the templates.

+

SQL templates include sample templates and custom templates. The default sample template contains 22 standard TPC-H query statements, which can meet most TPC-H test requirements. For details, see TPC-H Sample Data in the SQL Template.

+

SQL template management provides the following functions:

+ +

Table Settings

In the upper right corner of the SQL Template page, click Set Property to determine whether to display templates by group.

+

+

If you select Display by Group, the following display modes are available:

+ +
+

Sample Templates

The current sample template contains 22 standard TPC-H query statements. You can view the template name, description, and statements. For details about TPC-H examples, see TPC-H Sample Data in the SQL Template.

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Template management parameters

Parameter

+

Description

+

Name

+

Indicates the template name.

+
  • A template name can contain only digits, letters, and underscores (_), but cannot start with an underscore (_) or contain only digits. It cannot be left empty.
  • The template name can contain a maximum of 50 characters.
+

Description

+

Description of the template you create.

+

Statement

+

SQL statement created as the template.

+

Operation

+

Execute: After you click this button, the system switches to the SQL Editor page, where you can modify or directly perform the statement as required. For details, see Executing the Template.

+
+
+

The existing sample templates apply to the following scenarios:

+ +
+

Custom Templates

The custom template list displays all templates you have created. You can view the template name, description, statements, and more.

+ +
+ + + + + + + + + + + + + + + + +
Table 2 Template management parameters

Parameter

+

Description

+

Name

+

Indicates the template name.

+
  • A template name can contain only digits, letters, and underscores (_), but cannot start with an underscore (_) or contain only digits. It cannot be left empty.
  • The template name can contain a maximum of 50 characters.
+

Description

+

Description of the template you create.

+

Statement

+

SQL statement created as the template.

+

Operation

+
  • Execute: After you click this button, the system switches to the SQL Editor page, where you can modify or directly perform the statement as required. For details, see Executing the Template.
  • Modify: Click Modify. In the displayed Modify Template dialog box, modify the template information as required. For details, see Modifying a Template.
+
+
+
+

Creating a Template

You can create a template on either the Job Templates or the SQL Editor page.

+ +
+

Executing the Template

Perform the template operation as follows:
  1. On the left of the management console, choose Job Templates > SQL Templates.
  2. On the SQL Templates page, select a template and click Execute in the Operation column. The SQL Editor page is displayed, and the corresponding SQL statement is automatically entered in the SQL job editing window.
  3. In the upper right corner of the SQL job editing window, Click Execute to run the SQL statement. After the execution is complete, you can view the execution result below the current SQL job editing window.
+
+
+

Searching for a Template

On the SQL Templates page, you can enter the template name keyword in the search box on the upper right corner to search for the desired template.

+
+

Modifying a Template

Only custom templates can be modified. To modify a template, perform the following steps:

+
  1. On the SQL Templates page, locate the target template and click Modify in the Operation column.
  2. In the displayed Modify Template dialog box, modify the template name, statement, and description as required.
  3. Click OK.
+
+

Deleting a Template

On the SQL Templates page, select one or more templates to be deleted and click Delete to delete the selected templates.

+
+
+
+
+
Parent topic: Job Templates
+
+
+ diff --git a/docs/dli/umn/dli_01_0220.html b/docs/dli/umn/dli_01_0220.html new file mode 100644 index 00000000..3ffaab0d --- /dev/null +++ b/docs/dli/umn/dli_01_0220.html @@ -0,0 +1,17 @@ + + +

Getting Started

+
+
+ +
+ diff --git a/docs/dli/umn/dli_01_0228.html b/docs/dli/umn/dli_01_0228.html new file mode 100644 index 00000000..3652407c --- /dev/null +++ b/docs/dli/umn/dli_01_0228.html @@ -0,0 +1,119 @@ + + +

Overview

+

DLI database and table management provide the following functions:

+ +

Difference Between DLI Tables and OBS Tables

+
+

Databases and Tables Page

The Databases and Tables page displays all created databases. You can view the database information, such as the owner and the number of tables.

+ +
+ + + + + + + + + + + + + + + + + + + + + + +
Table 1 Database and table management parameters

Parameter

+

Description

+

Database Name

+
  • The database name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
  • The database name is case insensitive and cannot be left unspecified.
  • It cannot exceed 128 characters.
+

Username

+

Database owner.

+

Tables

+

Number of tables in the database.

+

Description

+

Description of the database specified during database creation. If no description is provided, -- is displayed.

+

Enterprise Project

+

Enterprise project to which the database belongs. An enterprise project facilitates project-level management and grouping of cloud resources and users.

+

For details about how to set enterprise projects, see .

+

Operation

+
  • Permissions: View the permission information and perform user authorization, permission settings, and user permission revocation.
  • Tables: View the tables in the corresponding database. For details, see Table Management Page.
  • Create Table: This permission allows you to create a table in the corresponding database.
  • Drop Database: This permission allows you to delete the selected database.
  • Modify Database. This permission allows you to change the owner of the database. The username must exist under the same account.
+
+
+
+

Table Management Page

From the Data Management page, click the database name or Tables in the Operation column to switch to the Table Management page.

+

The displayed page lists all tables created in the current database. You can view the table type, data storage location, and other information. Tables are listed in chronological order by default, with the most recently created tables displayed at the top.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 Table management parameters

Parameter

+

Description

+

Table Name

+
  • The table name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
  • The table name is case insensitive and cannot be left unspecified.
  • The table name can contain the dollar sign ($). An example value is $test.
  • It cannot exceed 128 characters.
+

Table Type

+

Table type. Available options are as follows:

+
  • Managed: Indicates that data is stored in a DLI table.
  • External: Indicates that data is stored in an OBS table.
  • View: Indicates the view type. You can only create views using SQL statements.
    NOTE:

    The table or view information contained in the view cannot be modified. If the table or view information is modified, the query may fail.

    +
    +
+

Owner

+

User who creates the table.

+

Storage Location

+

DLI, OBS, View, CloudTable, and CSS data location

+

Size

+

Size of the data in the table. The value is displayed only for tables of the Managed type. For tables of other types, -- is displayed.

+

Data Source Path

+
  • If Data Location is OBS, the corresponding OBS path is displayed.
  • If Data Location is DLI and View, -- is displayed.
  • When the data storage location is a datasource connection service such as CloudTable and CSS, the corresponding URL is displayed.
+

Created

+

Time when the table is created.

+

Last Accessed

+

Last time when an operation was performed on the table.

+

Operation

+
  • Manage Permissions: This operation allows you to view the permission information and perform user authorization, permission settings, and user permission revocation.
  • More:
    • Delete: Delete a table from the corresponding database.
    • Modify Owner: Change the owner of a table The username must exist under the same account.
    • Import: Import data stored in an OBS bucket to a DLI or OBS table.
    • Export: Export data from a DLI table to an OBS bucket.
    • Properties: View data in Metadata and Preview tabs.
    +
+
+
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0253.html b/docs/dli/umn/dli_01_0253.html new file mode 100644 index 00000000..22dbcbce --- /dev/null +++ b/docs/dli/umn/dli_01_0253.html @@ -0,0 +1,134 @@ + + +

Importing Data to the Table

+

You can import data from OBS to a table created in DLI.

+

Precautions

+
+

Prerequisites

The data to be imported has been stored on OBS.

+
+

Procedure

  1. You can import data on either the Data Management page or the SQL Editor page.

    • To import data on the Data Management page:
      1. On the left of the management console, choose Data Management > Databases and Tables.
      2. Click the name of the database corresponding to the table where data is to be imported to switch to the table management page.
      3. Locate the row where the target table resides and choose More > Import in the Operation column. The Import dialog box is displayed.

        +
      +
    • To import data on the SQL Editor page:
      1. On the left of the management console, click SQL Editor.
      2. In the navigation tree on the left of SQL Editor, click Databases to see all databases. Click the database where the target table belongs. The table list is displayed.
      3. Click on the right of the table and choose Import from the shortcut menu. The Import page is displayed.

        +
      +
    +

  2. In the Import dialog box, set the parameters based on Table 1.

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 1 Description

    Parameter

    +

    Description

    +

    Example

    +

    Databases

    +

    Database where the current table is located.

    +

    -

    +

    Table Name

    +

    Name of the current table.

    +

    -

    +

    Queues

    +

    Queue where the imported data will be used

    +

    -

    +

    File Format

    +

    Format of the data source file to be imported. The CSV, Parquet, ORC, JSON, and Avro formats are supported. Encoding format. Only UTF-8 is supported.

    +

    CSV

    +

    Path

    +

    You can directly enter a path or click and select an OBS path. If no bucket is available, you can directly switch to the OBS management console and create an OBS bucket.

    +
    • When creating an OBS table, you must specify a folder as the directory. If a file is specified, data import may be failed.
    • If a folder and a file have the same name in the OBS directory, the file path is preferred as the path of the data to be imported.
    +
    NOTE:

    The path can be a file or folder.

    +
    +

    obs://DLI/sampledata.csv

    +

    Table Header: No/Yes

    +

    This parameter is valid only when File Format is set to CSV. Whether the data source to be imported contains the table header.

    +

    Click Advanced Settings and select the check box next to Table Header: No. If the check box is selected, the table header is displayed. If the check box is deselected, no table header is displayed.

    +

    -

    +

    User-defined Delimiter

    +

    This parameter is valid only when File Format is set to CSV and you select User-defined Delimiter.

    +

    The following delimiters are supported:

    +
    • Comma (,)
    • Vertical bar (|)
    • Tab character (\t)
    • Others: Enter a user-defined delimiter.
    +

    Default value: comma (,)

    +

    User-defined Quotation Character

    +

    This parameter is valid only when File Format is set to CSV and User-defined Quotation Character is selected.

    +

    The following quotation characters are supported:

    +
    • Single quotation mark (')
    • Double quotation marks (")
    • Others: Enter a user-defined quotation character.
    +

    Default value: double quotation marks (")

    +

    User-defined Escape Character

    +

    This parameter is valid only when File Format is set to CSV and you select User-defined Escape Character.

    +

    The following escape characters are supported:

    +
    • Backslash (\)
    • Others: Enter a user-defined escape character.
    +

    Default value: backslash (\)

    +

    Date Format

    +

    This parameter is valid only when File Format is set to CSV or JSON.

    +

    This parameter specifies the format of the date in the table and is valid only Advanced Settings is selected. The default value is yyyy-MM-dd.

    +

    2000-01-01

    +

    Timestamp Format

    +

    This parameter is valid only when File Format is set to CSV or JSON.

    +

    This parameter specifies the format of the timestamp in the table and is valid only Advanced Settings is selected. The default value is yyyy-MM-dd HH:mm:ss. For definition of characters involved in the time pattern, see Table 3 in the

    +

    2000-01-01 09:00:00

    +

    Error Records Path

    +

    This parameter is valid only when File Format is set to CSV or JSON.

    +

    The parameter specifies the error data is stored in the corresponding OBS path and is valid only Advanced Settings is selected.

    +

    obs://DLI/

    +
    +
    +

  3. Click OK.
  4. You can view the imported data in either of the following ways:

    Currently, only the first 10 records are displayed.

    +
    +
    • Choose Data Management > Databases and Tables in the navigation pane of the console. Locate the row that contains the database where the target table belongs and click More > View Properties in the Operation column. In the displayed dialog box, click the Preview tab to view the imported data.
    • In the Databases tab of the SQL Editor, click the database name to go to the table list. Click on the right of a table name and choose View Properties from the shortcut menu. In the displayed dialog box, click Preview to view the imported data.
    +

  5. (Optional) View the status and execution result of the importing job on the Job Management > SQL Jobs page.
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0320.html b/docs/dli/umn/dli_01_0320.html new file mode 100644 index 00000000..06da4b31 --- /dev/null +++ b/docs/dli/umn/dli_01_0320.html @@ -0,0 +1,296 @@ + + +

SQL Editor

+

You can use SQL statements in the SQL job editor to execute data query. DLI supports SQL 2003 and complies with Spark SQL.

+

On the Overview page, click SQL Editor in the navigation pane on the left or Create Job in the upper right corner of the SQL Jobs pane. The SQL Editor page is displayed.

+

A message is displayed, indicating that a temporary DLI data bucket will be created. The created bucket is used to store temporary data generated by DLI, such as job logs. You cannot view job logs if you choose not to create it. You can to periodically delete objects in a bucket or transit objects between different storage classes. The bucket name is set by default.

+

If you do not need to create a DLI temporary data bucket and do not want to view this message, select Do not show again and click Cancel.

+

The SQL Editor page is composed of the following parts:

+

Navigation pane

The navigation pane on the left consists of Databases, Queues, and Templates tabs. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Navigation pane buttons

No.

+

Tab/Button

+

Name

+

Description

+

1

+

+

Databases

+

Displays all the existing databases and tables in these databases.

+
  • Click a database name to view the tables in the database.
  • Click a table name to view the metadata in the table. A maximum of 20 metadata records can be displayed.
  • After you double-click a table name, a SQL query statement is automatically entered in the editing window.
+

2

+

+

Queues

+

Displays existing queues.

+

3

+

+

Templates

+

Click the drop-down button to view 22 built-in standard TPC-H query templates and custom templates.

+

4

+

+

Create

+

Click the button on the related page to create a queue, database, or table. For details, see Creating a Queue and Creating a Database or a Table.

+

5

+

+

Refresh

+

Click the button to refresh the existing queue, database, and table lists.

+

6

+

+

Search

+

Enter a keyword to search for a database and table.

+
+
+
+
+

SQL Editing Window

The SQL job editing window is displayed in the upper right corner. The following describes the components of the window. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 Components of the SQL job editing window

No.

+

Button

+

Description

+

2

+

Queues

+

Select a queue from the drop-down list box. If no queue is available, the default queue is displayed. Refer to Creating a Queue and create a queue.

+

SQL jobs can be executed only on SQL queues.

+

3

+

Database

+

Select a database from the drop-down list box. If no database is available, the default database is displayed. For details about how to create a database, see section Creating a Database or a Table.

+
NOTE:

If you specify the database in the SQL statements, the database you choose from the drop-down list will not be used.

+
+

4

+

Execute

+

Click this button to run the SQL statements in the job editing window.

+

5

+

Format

+

Click this button to format the SQL statements.

+

6

+

Syntax Reference

+

Click this button to view the Data Lake Insight SQL Syntax Reference.

+

7

+

Settings

+

Add parameters and tags.

+

Parameter Settings: Set parameters in key/value format for SQL jobs.

+

Tags: Set tags in key/value format for SQL jobs.

+

8

+

More

+

The drop-down list includes the following options:

+
  • Click Verify Syntax to check whether the SQL statements are correct.
  • Click Set as Template to set SQL statements as a template. For details, see SQL Template Management.
  • Click Change Theme to switch between dark and light modes.
+

9

+

SQL statement editing area

+

Area for you to write and edit SQL statements.

+

10

+

Cursor position

+

The line and column where the cursor is in the job editing area.

+

11

+

Shortcuts

+

For details, see Table 3.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + +
Table 3 Keyboard shortcut description

Shortcut

+

Description

+

Ctrl+Enter

+

Execute SQL statements. You can run SQL statements by pressing Ctrl+R or Ctrl + Enter on the keyboard.

+

Ctrl+F

+

Search for SQL statements. You can press Ctrl+F to search for a required SQL statement.

+

Shift+Alt+F

+

Format SQL statements. You can press Shift + Alt + F to format a SQL statement.

+

Ctrl+Q

+

Syntax verification. You can press Ctrl + Q to verify the syntax of SQL statements.

+

F11

+

Full screen. You can press F11 to display the SQL Job Editor window in full screen. Press F11 again to leave the full screen.

+
+
+
+
+

Execution Queries and View Result

The Executed Queries (Last Day) and View Result tab pages are displayed in the lower part of the page.

+ +
+ +

SQL Query Procedure

  1. Log in to the Data Lake Insight management console, choose Job Management > SQL Jobs. In the displayed page, click Create Job to go to the SQL Editor page.
  2. After you enter the SQL Editor page, the system prompts you to create a temporary OBS bucket to store temporary data generated by DLI, such as job logs. You cannot view job logs if you choose not to create it. Click OK. The system displays a message indicating that the temporary bucket is created successfully.
  3. Select a queue from the queue list in the upper left corner of the SQL job editing window. For details about how to create a queue, see Creating a Queue.
  4. In the upper right corner of the SQL job editing window, select a database, for example, qw, from the Databases drop-down list.
  5. Create a table, for example, qw. For details about how to create a database and table, see Creating a Database or a Table.
  6. In the SQL job editing window, enter the following SQL statement:
    1
    SELECT * FROM qw.qw LIMIT 10;
+
    + +
    +

    Alternatively, you can double-click the table name qw. The query statement is automatically entered in the SQL job editing window.

    +
  7. On top of the editing window, click More > Verify Syntax to check whether the SQL statement is correct.
    1. If the verification fails, check the SQL statement syntax by referring to Data Lake Insight SQL Syntax Reference.
    2. If the syntax verification is successful, click Execute. Read and agree to the privacy agreement. Click OK to execute the SQL statement.
    3. After the execution is complete, you can view the execution result in the area under the SQL job editing window.
    +
  8. (Optional) A maximum of 1000 records can be displayed in the query result on the current console. To view more or all data, click to export the data to OBS.

    +
  9. (Optional) In the View Result tab page, click in the upper right corner to display the result in a graph. Click to display the query result in a table.

    +
    • If no column of the numeric type is displayed in the execution result, the result cannot be represented in charts.
    • You can view the data in a bar chart, line chart, or fan chart.
    • In the bar chart and line chart, the X axis can be any column, while the Y axis can only be columns of the numeric type. The fan chart displays the corresponding legends and indicators.
    +
    +
+
+

Operation Skills in the Job Editing Window

+
+
+
+ diff --git a/docs/dli/umn/dli_01_0363.html b/docs/dli/umn/dli_01_0363.html new file mode 100644 index 00000000..58e280bc --- /dev/null +++ b/docs/dli/umn/dli_01_0363.html @@ -0,0 +1,66 @@ + + +

Creating a Queue

+

Before executing a job, you need to create a queue.

+
  • If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main account and keep records in the DLI database before creating a queue.
  • It takes 6 to 10 minutes for a job running on a new queue for the first time.
  • After a queue is created, if no job is run within one hour, the system releases the queue.
+
+

Procedure

  1. You can create a queue on the Overview, SQL Editor, or Queue Management page.
    • In the upper right corner of the Overview page, click Create Queue.
    • To create a queue on the Queue Management page:
      1. In the navigation pane of the DLI management console, choose Resources >Queue Management.
      2. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
      +
    • To create a queue on the SQL Editor page:
      1. In the navigation pane of the DLI management console, click SQL Editor.
      2. On the left pane of the displayed SQL Editor page, click to the right of Queues.
      +
    +
  2. In the displayed Create Queue dialog box, set related parameters by referring to Table 1.
    +
    + + + + + + + + + + + + + + + + + + + +
    Table 1 Parameters

    Parameter

    +

    Description

    +

    Name

    +

    Name of a queue.

    +
    • The queue name can contain only digits, letters, and underscores (_), but cannot contain only digits, start with an underscore (_), or be left unspecified.
    • The length of the name cannot exceed 128 characters.
    +
    NOTE:

    The queue name is case-insensitive. Uppercase letters will be automatically converted to lowercase letters.

    +
    +

    Type

    +
    • For SQL: compute resources used for SQL jobs.
    • For general purpose: compute resources used for Spark and Flink jobs.
      NOTE:

      In Dedicated Resource Mode, you can create enhanced datasource connections.

      +
      +
    +

    Specifications

    +

    Select queue specifications as required. A CU includes one core and 4 GB memory. You can set the total number of CUs on all compute nodes of a queue. DLI automatically allocates the memory and vCPUs for each node.

    +
    • Fixed specifications include 16 CUs, 64 CUs, 256 CUs, and 512 CUs.
    • Custom: Set the specifications as required.
    +

    Description

    +

    Description of the queue to be created. The length of the queue name cannot exceed 256 characters.

    +

    Advanced Settings

    +
    In the Queue Type area, select Dedicated Resource Mode and then click Advanced Settings.
    • Default: The system automatically configures the parameter.
    • Custom

      CIDR Block: You can specify the CIDR block. For details, see Modifying the Queue CIDR Block. If DLI enhanced datasource connection is used, the CIDR block of the DLI queue cannot overlap with that of the data source.

      +

      Queue Type: When running an AI-related SQL job, select AI-enhanced. When running other jobs, select Basic.

      +
    +
    +
    +
    +
    +
  3. Click Create Now to create a queue.

    After a queue is created, you can view and select the queue for use on the Queue Management page.

    +

    It takes 6 to 10 minutes for a job running on a new queue for the first time.

    +
    +
+
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0366.html b/docs/dli/umn/dli_01_0366.html new file mode 100644 index 00000000..1352042b --- /dev/null +++ b/docs/dli/umn/dli_01_0366.html @@ -0,0 +1,25 @@ + + +

Package Management

+
+
+ + +
+
Parent topic: Data Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0367.html b/docs/dli/umn/dli_01_0367.html new file mode 100644 index 00000000..ba124d42 --- /dev/null +++ b/docs/dli/umn/dli_01_0367.html @@ -0,0 +1,54 @@ + + +

Creating a Package

+

DLI allows you to submit program packages in batches to the general-use queue for running.

+

If you need to update a package, you can use the same package or file to upload it to the same location (in the same group) on DLI to overwrite the original package or file.

+
+

Prerequisites

All software packages must be uploaded to OBS for storage in advance.

+
+

Procedure

  1. On the left of the management console, choose Data Management > Package Management.
  2. On the Package Management page, click Create in the upper right corner to create a package.
  3. In the displayed Create Package dialog box, set related parameters by referring to Table 1. +
    + + + + + + + + + + + + + + + + +
    Table 1 Parameter description

    Parameter

    +

    Description

    +

    Package Type

    +

    The following package types are supported:

    +
    • JAR: JAR file
    • PyFile: User Python file
    • File: User file
    +

    Package File Path

    +

    Select the OBS path of the corresponding packages.

    +
    NOTE:
    • The packages must be uploaded to OBS for storage in advance.
    • Only files can be selected.
    +
    +

    Group Policy

    +

    You can select Use existing group, Use new group, or No grouping.

    +

    Group Name

    +
    • Use existing group: Select an existing group.
    • Use new group: Enter a custom group name.
    • No grouping: No need to select or enter a group name.
    +
    NOTE:
    • If you select a group, the permission management refers to the permissions of the corresponding package group.
    • If no group is selected, the permission management refers to the permissions of the corresponding package.
    +

    For details about how to manage permissions on package groups and packages, see Managing Permissions on Packages and Package Groups.

    +
    +
    +
    +
  4. Click OK.

    After a package is created, you can view and select the package for use on the Package Management page.

    +
+
+
+
+
+
Parent topic: Package Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0369.html b/docs/dli/umn/dli_01_0369.html new file mode 100644 index 00000000..3c5dbe60 --- /dev/null +++ b/docs/dli/umn/dli_01_0369.html @@ -0,0 +1,13 @@ + + +

Deleting a Package

+

You can delete a package based on actual conditions.

+

Procedure

  1. On the left of the management console, choose Data Management > Package Management.
  2. Click Delete in the Operation column of the package to be deleted.
  3. In the dialog box that is displayed, click Yes.
+
+
+
+
+
Parent topic: Package Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0375.html b/docs/dli/umn/dli_01_0375.html new file mode 100644 index 00000000..07f02728 --- /dev/null +++ b/docs/dli/umn/dli_01_0375.html @@ -0,0 +1,52 @@ + + +

Creating and Submitting a Spark Jar Job

+

You can use DLI to submit Spark jobs for real-time computing. The general procedure is as follows:

+

Step 1: Logging in to the Cloud

+

Step 2: Uploading Data to OBS

+

Step 3: Logging In to the DLI Management Console

+

Step 4: Creating a Queue

+

Step 5: Creating a Package

+

Step 6: Submitting a Spark Job

+

Step 1: Logging in to the Cloud

To use DLI, you need to log in to the cloud.

+
+
  1. Open the DLI home page.
  2. On the login page, enter the username and password, and click Log In.
+

Step 2: Uploading Data to OBS

Write a Spark Jar job program , and compile and package it as spark-examples.jar. Perform the following steps to submit the job:

+

Before submitting Spark Jar jobs, upload data files to OBS.

+
  1. In the services displayed, click Object Storage Service (OBS) in Storage.
  2. The OBS console page is displayed.
  3. Create a bucket. The bucket name must be globally unique. In this example, assume that the bucket name is dli-test-obs01.
    1. Click Create Bucket.
    2. On the displayed Create Bucket page, enter the Bucket Name. Retain the default values for other parameters or set them as required.
    3. Click Create Now.
    +
  4. Click dli-test-obs01 to switch to the Overview page.
  5. From the left navigation tree, click Objects. Click Upload Object. In the displayed dialog box, drag files or folders to upload or add file to the file upload box, for example, spark-examples.jar. Then, click Upload.

    After the file is uploaded successfully, the file path is obs://dli-test-obs01/spark-examples.jar.

    +

    For more information about OBS operations, see the Object Storage Service Console Operation Guide.

    +

    For more information about the tool, see the OBS Tool Guide.

    +

    You are advised to use an OBS tool, such as OBS Browser+, to upload large files because OBS Console has restrictions on the file size and quantity.

    +
    • OBS Browser+ is a graphical tool that provides complete functions for managing your buckets and objects in OBS. You are advised to use this tool to create buckets or upload objects.
    +
    +
+
+

Step 3: Logging In to the DLI Management Console

To submit Spark jobs, you need to enter the Spark job creation page first.

+
  1. In the list, choose .
  2. The DLI management console page is displayed.

    If you log in to the DLI management console for the first time, you need to be authorized to access OBS. For details, see "Service Authorization".

    +
    +
+
+

Step 4: Creating a Queue

If you submit a Spark job for the first time, you need to create a queue first. For example, create a queue named sparktest and set Queue Type to General Queue.

+
  1. In the navigation pane of the DLI management console, choose Queue Management.
  2. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
  3. Create a queue, name it sparktest, and set the queue usage to for general purpose. For details, see Creating a Queue.

    +
  4. Click Create Now to create a queue.
+
+

Step 5: Creating a Package

Before submitting a Spark job, you need to create a package, for example, spark-examples.jar.

+
  1. In the navigation pane on the left of the DLI console, choose Data Management > Package Management.
  2. On the Package Management page, click Create in the upper right corner to create a package.
  3. In the Create Package dialog box, set Type to JAR, OBS Path to the path of the spark-examples.jar package in Step 2: Uploading Data to OBS, and Group to Do not use.

    +
  4. Click OK.

    You can view and select the package on the Package Management page.

    +
+

For details about how to create a package, see "Creating a Package".

+
+

Step 6: Submitting a Spark Job

  1. On the DLI management console, choose Job Management > Spark Jobs in the navigation pane on the left. On the displayed page, click Create Job.
  2. On the Spark job editing page, set Queues to the queue created in Step 4: Creating a Queue and Application to the package created in Step 5: Creating a Package.

    For details about other parameters, see the description of the Spark job editing page in "Creating a Spark Job".

    +
  3. Click Execute in the upper right corner of the Spark job editing window, read and agree to the privacy agreement, and click OK. Submit the job. A message is displayed, indicating that the job is submitted.

    +
  4. (Optional) Switch to the Job Management > Spark Jobs page to view the status and logs of the submitted Spark job.

    When you click Execute on the DLI management console for the first time, you need to read the privacy agreement. Once agreed to the agreement, you will not receive any privacy agreement messages for subsequent operations.

    +
    +
+
+
+
+
+
Parent topic: Getting Started
+
+
+ diff --git a/docs/dli/umn/dli_01_0376.html b/docs/dli/umn/dli_01_0376.html new file mode 100644 index 00000000..1632ded9 --- /dev/null +++ b/docs/dli/umn/dli_01_0376.html @@ -0,0 +1,18 @@ + + +

Modifying the Owners of Databases and Tables

+

During actual use, developers create databases and tables and submit them to test personnel for testing. After the test is complete, the databases and tables are transferred to O&M personnel for user experience. In this case, you can change the owner of the databases and tables to transfer data to other owners.

+

Modifying the Database Owner

You can change the owner of a database on either the Data Management page or the SQL Editor page.
  • On the Data Management page, change the database owner.
    1. On the left of the management console, choose Data Management > Databases and Tables.
    2. On the Databases and Tables page, locate the database you want and click More > Modify Database in the Operation column.
    3. In the displayed dialog box, enter a new owner name (an existing username) and click OK.
    +
  • Change the database owner on the SQL Editor page.
    1. On the left of the management console, click SQL Editor.
    2. In the navigation tree on the left, click Databases, click on the right of the database you want to modify, and choose Modify Database from the shortcut menu.
    3. In the displayed dialog box, enter a new owner name (an existing username) and click OK.
    +
+
+
+

Modifying the Table Owner

  1. On the left of the management console, choose Data Management > Databases and Tables.
  2. Click the name of the database corresponding to the table to be modified. The Manage Tables page of the database is displayed.
  3. In the Operation column of the target table, choose More > Modify Owner.
  4. In the displayed dialog box, enter a new owner name (an existing username) and click OK.
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0377.html b/docs/dli/umn/dli_01_0377.html new file mode 100644 index 00000000..b92ff84b --- /dev/null +++ b/docs/dli/umn/dli_01_0377.html @@ -0,0 +1,91 @@ + + +

DLI Console Overview

+

Overview

Log in to DLI and go to the Overview page.

+

The following table describes the functional areas of the Overview page.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Areas

No.

+

Area

+

Description

+

1

+

Navigation pane

+

The navigation bar on the DLI management console, including Overview, SQL Editor, Job Management, Queue Management, Data Management, Job Templates, Datasource Connections, and Global Configuration.

+

2

+

DLI introduction

+

Data Lake Insight (DLI) is a fully hosted big data processing and analysis service. It frees you from managing any server. DLI supports standard SQL and is compatible with Spark and Flink SQL. It also supports multiple access modes, and is compatible with mainstream data formats.

+

3

+

Upper right corner

+

Click Create Queue to create a queue.

+

4

+

SQL job introduction

+

Click Create Job in this area to access the SQL Editor page. For details about how to create a package, see SQL Editor.

+

5

+

Flink job introduction

+

Click Create Job in this area to go to the Flink Jobs page. For details about how to create a package, see Flink Job Management.

+

6

+

Spark job introduction

+

Click Create Job in this area to access the Spark Jobs page. For details about how to create a Spark job, see Creating a Spark Job.

+

8

+

Queue usage (hours)

+

This area displays the usage of all queues of the last day, week, or month.

+

9

+

Selected queue usage (hours)

+

This area displays the usage of a queue in the last week or month.

+

10

+

Common links

+

Including:

+
  • User Guide
  • API Reference
  • SDK Download
+

11

+

Get started in four simple steps

+
  1. Create a queue
  2. Preparing data
  3. Editing and submitting a job
  4. Viewing job status
+
+
+
+
+ diff --git a/docs/dli/umn/dli_01_0378.html b/docs/dli/umn/dli_01_0378.html new file mode 100644 index 00000000..5b2a5346 --- /dev/null +++ b/docs/dli/umn/dli_01_0378.html @@ -0,0 +1,35 @@ + + +

DLI Introduction

+

What Is Data Lake Insight?

Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing any server.

+

DLI supports standard SQL and is compatible with Spark SQL and Flink SQL. It also supports multiple access modes, and is compatible with mainstream data formats. DLI supports SQL statements and Spark applications for heterogeneous data sources, including CloudTable, RDS, GaussDB(DWS), CSS, OBS, custom databases on ECSs, and offline databases.

+
+

Function Description

You can query and analyze heterogeneous data sources such as RDS, and DWS on the cloud using access methods, such as visualized interface, RESTful API, JDBC, ODBC, and Beeline. The data format is compatible with five mainstream data formats: CSV, JSON, Parquet, and ORC.

+ +
+

DLI Core Engine: Spark+Flink

+
+

Serverless Architecture

DLI is a serverless big data query and analysis service. It has the following advantages:

+ +
+

Accessing DLI

A web-based service management platform is provided. You can access DLI using the management console or HTTPS-based APIs, or connect to the DLI server through a client such as JDBC or ODBC.

+ + +
+
+
+
+
Parent topic: Service Overview
+
+
+ diff --git a/docs/dli/umn/dli_01_0379.html b/docs/dli/umn/dli_01_0379.html new file mode 100644 index 00000000..ab673752 --- /dev/null +++ b/docs/dli/umn/dli_01_0379.html @@ -0,0 +1,15 @@ + + +

Job Templates

+
+
+ +
+ diff --git a/docs/dli/umn/dli_01_0384.html b/docs/dli/umn/dli_01_0384.html new file mode 100644 index 00000000..20181d7d --- /dev/null +++ b/docs/dli/umn/dli_01_0384.html @@ -0,0 +1,261 @@ + + +

Creating a Spark Job

+

DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.

+

On the Overview page, click Create Job in the upper right corner of the Spark Jobs tab or click Create Job in the upper right corner of the Spark Jobs page. The Spark job editing page is displayed.

+

On the Spark job editing page, a message is displayed, indicating that a temporary DLI data bucket will be created. The created bucket is used to store temporary data generated by DLI, such as job logs and job results. You cannot view job logs if you choose not to create it. The bucket will be created and the default bucket name is used.

+

If you do not need to create a DLI temporary data bucket and do not want to receive this message, select Do not show again and click Cancel.

+

Prerequisites

+
+

GUI Description

+ +
+

Creating a Spark Job

  1. In the Spark job editing window, set related parameters. For details, see the description of the Spark job editing window.
  2. Click Execute in the upper right corner of the Spark job editing window to submit the job. The message "The batch job is submitted." is displayed.
  3. (Optional) Go to the Spark Jobs page to view the status and logs of the submitted Spark job.
+
+
+
+
+
Parent topic: Spark Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0385.html b/docs/dli/umn/dli_01_0385.html new file mode 100644 index 00000000..265cabc1 --- /dev/null +++ b/docs/dli/umn/dli_01_0385.html @@ -0,0 +1,80 @@ + + +

Spark Job Management

+

Based on the open-source Spark, DLI optimizes performance and reconstructs services to be compatible with the Apache Spark ecosystem and interfaces, and executes batch processing tasks.

+

DLI also allows you to use Spark jobs to access DLI metadata.

+

Spark job management provides the following functions:

+ +

In addition, you can click Quick Links to switch to the details on User Guide.

+

Spark Jobs Page

On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Job management parameters

Parameter

+

Description

+

Job ID

+

ID of a submitted Spark job, which is generated by the system by default.

+

Name

+

Name of a submitted Spark job.

+

Queues

+

Queue where the submitted Spark job runs

+

Username

+

Name of the user who executed the Spark job

+

Status

+

Job status. The following values are available:

+
  • Starting: The job is being started.
  • Running: The job is being executed.
  • Failed: The session has exited.
  • Finished: The session is successfully executed.
  • Restoring: The job is being restored.
+

Created

+

Time when a job is created. Jobs can be displayed in ascending or descending order of the job creation time.

+

Last Modified

+

Time when a job is completed.

+

Operation

+
  • Edit: You can modify the current job configuration and re-execute the job.
  • SparkUI: After you click this button, the Spark job execution page is displayed.
    NOTE:
    • The SparkUI page cannot be viewed for jobs in the Starting state.
    • Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.
    +
    +
  • Terminate Job: Cancel a job that is being started or running.
  • Re-execute: Run the job again.
  • Archive Log: Save job logs to the temporary bucket created by DLI.
  • Export Log: Export logs to the created OBS bucket.
    NOTE:
    • You have the permission to create OBS buckets.
    • If the job is in the Running state, logs cannot be exported.
    +
    +
  • Commit Log: View the logs of submitted jobs.
  • Driver Log: View the logs of running jobs.
+
+
+
+

Re-executing a Job

On the Spark Jobs page, click Edit in the Operation column of the job. On the Spark job creation page that is displayed, modify parameters as required and execute the job.

+
+

Searching for a Job

On the Spark Jobs page, select Status or Queues. The system displays the jobs that meet the filter condition in the job list.

+
+

Terminating a Job

On the Spark Jobs page, choose More > Terminate Job in the Operation column of the job that you want to stop.

+
+

Exporting Logs

On the Spark Jobs page, choose More > Export Log in the Operation column of the corresponding job. In the dialog box that is displayed, enter the path of the created OBS bucket and click OK.

+
+
+
+
+
Parent topic: Spark Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0389.html b/docs/dli/umn/dli_01_0389.html new file mode 100644 index 00000000..07866ad6 --- /dev/null +++ b/docs/dli/umn/dli_01_0389.html @@ -0,0 +1,31 @@ + + +

Flink Job Management

+
+
+ + +
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0390.html b/docs/dli/umn/dli_01_0390.html new file mode 100644 index 00000000..81f94a67 --- /dev/null +++ b/docs/dli/umn/dli_01_0390.html @@ -0,0 +1,33 @@ + + +

Databases and Tables

+
+
+ + +
+
Parent topic: Data Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0397.html b/docs/dli/umn/dli_01_0397.html new file mode 100644 index 00000000..1f66abb2 --- /dev/null +++ b/docs/dli/umn/dli_01_0397.html @@ -0,0 +1,18 @@ + + +

Built-in Dependencies

+

DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar jobs.

+

Spark 2.3.2 Dependencies

+
+

Flink 1.7.2 Dependencies

+
+

Flink 1.10 Dependencies

Only queues created after December 2020 can use the Flink 1.10 dependencies.

+ +
+
+
+
+
Parent topic: Package Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0402.html b/docs/dli/umn/dli_01_0402.html new file mode 100644 index 00000000..3ddd71fb --- /dev/null +++ b/docs/dli/umn/dli_01_0402.html @@ -0,0 +1,123 @@ + + +

Overview

+

Queue

Queues in DLI are computing resources, which are the basis for using DLI. All executed jobs require computing resources.

+

Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earlier versions and are used to run Spark and Flink jobs.

+
  • The SQL queue automatically restarts every 12 hours.
  • Queues of different types cannot be directly switched. If you need to change a queue, purchase a new queue.
  • The default queue is preset. Do not delete it.
+
+
+

Difference Between Computing and Storage Resources

+
+ + + + + + + + + + + + + +
Table 1 Difference between computing and storage resources

Resource

+

How to Obtain

+

Function

+

Compute resource

+

Create queue on the DLI management console.

+

Used for executing queries.

+

Storage resource

+

DLI has a 5 GB quota.

+

Used for storing data in the database and DLI tables.

+
+
+
  • Storage resources are internal storage resources of DLI for storing database and DLI tables and represent the amount of data stored in DLI.
  • By default, DLI provides a 5 GB quota for storage resources.
  • A queue named default is preset in DLI. If you are uncertain about the required queue capacity or have no available queue capacity to run queries, you can execute jobs using this queue.
+
  • The default queue is used only for user experience. It may be occupied by multiple users at a time. Therefore, it is possible that you fail to obtain the resource for related operations. You are advised to use a self-built queue to execute jobs.
+
+
+

Dedicated Queue

Resources of a dedicated queue are not released when the queue is idle. That is, resources are reserved regardless of whether the queue is used. Dedicated queues ensure that resources exist when jobs are submitted.

+
+

Elastic Scaling

DLI allows you to flexibly scale in or out queues on demand. After a queue with specified specifications is created, you can scale it in and out as required.

+

To change the queue specifications, see Elastic Scaling.

+

Scaling can be performed for a newly created queue only when jobs are running on this queue.

+
+
+

Scheduled Elastic Scaling

DLI allows you to schedule tasks for periodic queue scaling. After creating a queue, the scheduled scaling tasks can be executed.

+
You can schedule an automatic scale-out/scale-in based on service requirements. The system periodically triggers queue scaling. For details, see Scheduling CU Changes.

Scaling can be performed for a newly created queue only when jobs are running on this queue.

+
+
+
+

Automatic Queue Scaling

Flink jobs use queues. DLI can automatically trigger scaling for jobs based on the job size.

+
+

Scaling can be performed for a newly created queue only when there are jobs running on this queue.

+
+

Queue Management Page

Queue Management provides the following functions:

+ +

To receive notifications when a DLI job fails, SMN Administrator permissions are required.

+
+

The queue list displays all queues created by you and the default queue. Queues are listed in chronological order by default in the queue list, with the most recently created queues displayed at the top.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 Parameter description

Parameter

+

Description

+

Name

+

Name of a queue.

+

Type

+

Queue type.

+
  • For SQL
  • For general purpose
  • Spark queue (compatible with earlier versions)
+

Specifications

+

Queue size. Unit: CU

+

CU is the pricing unit of queues. A CU consists of 1 vCPU and 4-GB memory. The computing capabilities of queues vary with queue specifications. The higher the specifications, the stronger the computing capability.

+

Actual CUs

+

Actual size of the current queue.

+

Elastic Scaling

+

Target CU value for scheduled scaling, or the maximum and minimum CU values of the current specifications.

+

Username

+

Queue owner

+

Description

+

Description of a queue specified during queue creation. If no description is provided, -- is displayed.

+

Operation

+
  • Delete: Allow you to delete the selected queue. You cannot delete a queue where there are running jobs or jobs are being submitted.
  • Manage Permissions: You can view the user permissions corresponding to the queue and grant permissions to other users.
  • More
    • Restart: Forcibly restart a queue.
      NOTE:

      Only the SQL queue has the Restart operation.

      +
      +
    • Elastic Scaling: You can select Scale-out or Scale-in as required. The number of CUs after modification must be an integer multiple of 16.
    • Schedule CU Changes: You can set different queue sizes at different time or in different periods based on the service period or usage. The system automatically performs scale-out or scale-in as scheduled. The After Modification value must be an integer multiple of 16.
    • Modifying CIDR Block: When DLI enhanced datasource connection is used, the CIDR block of the DLI queue cannot overlap with that of the data source. You can modify the CIDR block as required.

      +
    • Test Address Connectivity: Test whether the queue is reachable to the specified address. The domain name and IP address are supported. The port can be specified.
    +
+
+
+
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0403.html b/docs/dli/umn/dli_01_0403.html new file mode 100644 index 00000000..e2e1d2c1 --- /dev/null +++ b/docs/dli/umn/dli_01_0403.html @@ -0,0 +1,96 @@ + + +

Overview

+

On the Job Management page of Flink jobs, you can submit a Flink job. Currently, the following job types are supported:

+ +

Flink job management provides the following functions:

+ +

Assigning Agency Permissions

Agencies are required for DLI to execute Flink jobs. You can set the agency when logging in to the management console for the first time or go to Global Configurations > Service Authorization to modify the agencies.

+

The permissions are as follows:

+ +
+

Flink Jobs Page

On the Overview page, click Flink Jobs to go to the Flink job management page. Alternatively, you can choose Job Management > Flink Jobs from the navigation pane on the left. The page displays all Flink jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Job management parameters

Parameter

+

Description

+

ID

+

ID of a submitted Flink job, which is generated by the system by default.

+

Name

+

Name of the submitted Flink job.

+

Type

+

Type of the submitted Flink job. Including:

+
  • Flink SQL: Flink SQL jobs
  • Flink Jar: Flink Jar jobs
+

Status

+

Job statuses, including:

+
  • Draft
  • Submitting
  • Submission failed
  • Running: After the job is submitted, a normal result is returned.
  • Running exception: The job stops running due to an exception.
  • Downloading
  • Idle
  • Stopping
  • Stopped
  • Stopping failed
  • Creating the savepoint
  • Completed
+

Description

+

Description of the submitted Flink job.

+

Username

+

Name of the user who submits a job.

+

Created

+

Time when a job is created.

+

Started

+

Time when a Flink job starts to run.

+

Duration

+

Time consumed by job running.

+

Operation

+
  • Edit: Edit a created job. For details, see Editing a Job.
  • Start: Start and run a job. For details, see Starting a Job.
  • More
    • FlinkUI: After you click this button, the Flink job execution page is displayed.
      NOTE:

      When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click FlinkUI before the cluster is created, an empty projectID will be cached. The FlinkUI page cannot be displayed.

      +

      You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check FlinkUI.

      +
      +
    • Stop: Stop a Flink job. If this function is unavailable, jobs in the current status cannot be stopped.
    • Delete: Delete a job.
      NOTE:

      A deleted job cannot be restored.

      +
      +
    • Modify Name and Description: You can modify the name and description of a job. For details, see Modifying Name and Description.
    • Import Savepoint: Import the data exported from the original CS job. For details, see Importing to a Savepoint.
    • Trigger Savepoint: You can click this button for jobs in the Running status to save the job status. For details, see Triggering a Savepoint.
    • Permissions: You can view the user permissions corresponding to the job and grant permissions to other users. For details, see Managing Flink Job Permissions.
    • Runtime Configuration: You can enable Alarm Generation upon Job Exception and Auto Restart upon Exception. For details, see Runtime Configuration.
    +
+
+
+
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0407.html b/docs/dli/umn/dli_01_0407.html new file mode 100644 index 00000000..09875e40 --- /dev/null +++ b/docs/dli/umn/dli_01_0407.html @@ -0,0 +1,70 @@ + + +

Overview

+

Package management provides the following functions:

+ +

Package Management Page

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Package management parameters

Parameter

+

Description

+

Group Name

+

Name of the group to which the package belongs. If the package is not grouped, -- is displayed.

+

Package Name

+

Name of a package.

+

Owner

+

Name of the user who uploads the package.

+

Type

+

Type of a package. The following package types are supported:

+
  • JAR: JAR file
  • PyFile: User Python file
  • File: User file
+

Status

+

Status of the package to be created.

+
  • Uploading: The file is being uploaded.
  • Finished: The resource package has been uploaded.
  • Failed: The resource package upload failed.
+

Created

+

Time when a package is created.

+

Updated

+

Time when the package is updated.

+

Operation

+

Manage Permissions: Manage user permissions for a package.

+

Delete: Delete the package.

+

More:

+
  • Modify Owner: Modify the owner of the package.
+
+
+
+
+
+
+
Parent topic: Package Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0408.html b/docs/dli/umn/dli_01_0408.html new file mode 100644 index 00000000..97078b44 --- /dev/null +++ b/docs/dli/umn/dli_01_0408.html @@ -0,0 +1,21 @@ + + +

Permissions Management

+
+
+ +
+ diff --git a/docs/dli/umn/dli_01_0410.html b/docs/dli/umn/dli_01_0410.html new file mode 100644 index 00000000..c97f558e --- /dev/null +++ b/docs/dli/umn/dli_01_0410.html @@ -0,0 +1,31 @@ + + +

Datasource Connection and Cross-Source Analysis

+

DLI supports the datasource capability of the native Spark and extends it. With DLI datasource connection, you can access other data storage services through SQL statements, Spark jobs, and Flink jobs and import, query, analyze, and process data in the services.

+

Datasource Connections

Before using DLI to perform cross source analysis, you need to set up a datasource connection to enable the network between data sources.

+

The enhanced datasource connection uses VPC peering at the bottom layer to directly connect the VPC network between the DLI queue and the destination datasource. Data is exchanged in point-to-point mode.

+
  • Datasource connections cannot be created for the default queue.
  • VPC Administrator permissions are required to use the VPC, subnet, route, VPC peering connection, and port for DLI datasource connections. You can set this parameter in Service Authorization.
+
+
+

Cross-Source Analysis

The enhanced datasource connection supports all cross-source services implemented by DLI and implements access to self-built data sources by using UDFs, Spark jobs, and Flink jobs.

+

Currently, DLI supports datasource connection to the following data sources: CloudTable HBase, CloudTable OpenTSDB, CSS, DCS Redis, DDS Mongo, DIS, DMS, DWS, MRS HBase, MRS Kafka, MRS OpenTSDB, OBS, RDS MySQL, RDS PostGre, and SMN.

+
  • To access a datasource connection table, you need to use the queue for which a datasource connection has been created.
  • The preview function is not supported for datasource tables.
+
+
+

Datasource Connection Analysis Process

To use DLI for cross-source analysis, you need to create datasource connections and then develop different jobs to access data sources. Perform the following steps:

+
  1. Create a datasource connection. You can create a connection in either of the following ways:
    • Create a datasource connection on the management console.
    • Create a datasource connection through an API.
    +
  2. Develop a DLI job to connect to a datasource. You can connect to a datasource in one of the following ways:
    • Develop a SQL job to connect to a datasource.
    • Develop a Spark job to connect to a datasource.
    • Develop a Flink job to connect to a datasource.
    +
+

The following describes the basic processes of developing SQL jobs, Spark jobs, and Flink jobs for datasource connection.

+ +
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0417.html b/docs/dli/umn/dli_01_0417.html new file mode 100644 index 00000000..f38cac64 --- /dev/null +++ b/docs/dli/umn/dli_01_0417.html @@ -0,0 +1,79 @@ + + +

DLI Resources

+

A resource is an object that exists within a service. You can select DLI resources by specifying their paths.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 DLI resources and their paths

Resource Type

+

Resource Names

+

Path

+

queue

+

DLI queue

+

queues.queuename

+

database

+

DLI database

+

databases.dbname

+

table

+

DLI table

+

databases.dbname.tables.tbname

+

column

+

DLI column

+

databases.dbname.tables.tbname.columns.colname

+

jobs

+

DLI Flink job

+

jobs.flink.jobid

+

resource

+

DLI package

+

resources.resourcename

+

group

+

DLI package group

+

groups.groupname

+

datasourceauth

+

DLI cross-source authentication information

+

datasourceauth.name

+
+
+
+
+
+
Parent topic: Permissions Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0418.html b/docs/dli/umn/dli_01_0418.html new file mode 100644 index 00000000..6ee7a0e6 --- /dev/null +++ b/docs/dli/umn/dli_01_0418.html @@ -0,0 +1,25 @@ + + +

Creating an IAM User and Granting Permissions

+

You can use Identity and Access Management (IAM) to implement fine-grained permissions control on DLI resources. For details, see Overview.

+

If your cloud account does not need individual IAM users, then you may skip over this chapter.

+

This section describes how to create an IAM user and grant DLI permissions to the user. Figure 1 shows the procedure.

+

Prerequisites

Before assigning permissions to user groups, you should learn about system policies and select the policies based on service requirements. For details about system permissions supported by DLI, see DLI System Permissions.

+
+

Process Flow

Figure 1 Process for granting DLI permissions
+

+

+
  1. Create a user group and grant the permission to it.

    Create a user group on the IAM console, and assign the DLI ReadOnlyAccess permission to the group.

    +
  2. Create a user and add the user to the user group.

    Create a user on the IAM console and add the user to the group created in 1.

    +
  3. Log in and verify the permission.

    Log in to the management console using the newly created user, and verify that the user's permissions.

    +
    • Choose Service List > Data Lake Insight. The DLI management console is displayed. If you can view the queue list on the Queue Management page but cannot buy DLI queues by clicking Buy Queue in the upper right corner (assume that the current permission contains only DLI ReadOnlyAccess), the DLI ReadOnlyAccess permission has taken effect.
    • Choose any other service in Service List. If a message appears indicating that you have insufficient permissions to access the service, the DLI ReadOnlyAccess permission has already taken effect.
    +
+

+
+
+
+
+
Parent topic: Permissions Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0421.html b/docs/dli/umn/dli_01_0421.html new file mode 100644 index 00000000..7feff532 --- /dev/null +++ b/docs/dli/umn/dli_01_0421.html @@ -0,0 +1,15 @@ + + +

Creating a Message Notification Topic

+

Once you have created a message notification topic, you can Add subscription of the topic on the Topic Management page of the Simple Message Notification service. You can select different ways (such as text messages or emails) to subscribe. After the subscription succeeds, any job failure will automatically be sent to your subscription endpoints. The operation is as follows:

+
  1. On the Queue Management page, click Create SMN Topic on the upper left side. The Create SMN Topic dialog box is displayed.
  2. Select a queue and click OK. The dialog box is displayed.
    • You can select a single queue or all queues. If the subscription endpoints of a single queue are different from those of all queues, when a single queue and all queues are both selected, the SMN of all queues does not include the message of the single queue.
    • After a notification topic is created, you will receive a notification only when a session or a batch job fails to be created.
    +
    +
  3. Click Topic Management to go to the Topic Management page of the SMN service.
  4. In the Operation column of the corresponding topic in , click Add Subscription. Select Protocol to determine the subscription mode.
    • If you select Email, enter the email address for receiving the confirmation email in the Endpoint text box.
    +
  5. After you click the link in the email, you will receive a message indicating that the subscription is successful.
  6. Return to the Topic Management page and click Subscriptions on the left. If the subscription status is Confirmed, the subscription is successful.
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0422.html b/docs/dli/umn/dli_01_0422.html new file mode 100644 index 00000000..75ff693d --- /dev/null +++ b/docs/dli/umn/dli_01_0422.html @@ -0,0 +1,17 @@ + + +

Datasource Connections

+
+
+ +
+ diff --git a/docs/dli/umn/dli_01_0426.html b/docs/dli/umn/dli_01_0426.html new file mode 100644 index 00000000..6b5043d6 --- /dev/null +++ b/docs/dli/umn/dli_01_0426.html @@ -0,0 +1,25 @@ + + +

Enhanced Datasource Connections

+
+
+ + +
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0427.html b/docs/dli/umn/dli_01_0427.html new file mode 100644 index 00000000..b3f51674 --- /dev/null +++ b/docs/dli/umn/dli_01_0427.html @@ -0,0 +1,184 @@ + + +

Creating and Managing Datasource Authentication

+

Scenarios

Datasource authentication is used to manage authentication information for accessing CSS and MRS security environments and encrypt passwords for accessing DWS, RDS, DDS, and DCS data sources.

+ +
+

Creating a Datasource Authentication

  1. Create a data source to be accessed.

    If a cluster is available, you do not need to apply for one.

    +
    +
    • Create a CSS cluster in the security mode: Select 6.5.4 or a later version for Cluster Version and enable the Security Mode.
    • Create an MRS cluster in the security mode: On the MRS management console, apply for a cluster, select Custom Config, set Cluster Version to MRS 2.1.0 or later, and enable Kerberos Authentication.
    • Creating a Kafka cluster: You can configure the Kafka cluster on the Service Configuration page of MRS and Kafka. For details about how to configure SSL on MRS, see Security Configuration in the MapReduce Service User Guide.
    • Creating a DWS cluster: Apply for a data warehouse cluster from DWS. For details, see the Data Warehouse Service Management Guide.
    • Creating an RDS data source: Apply for a database instance in the RDS service. For details, see the Relational Database Service Getting Started.
    • Creating a DCS data source: Apply for a DCS instance in DCS. For details, see the Distributed Cache Service User Guide.
    • Creating a DDS data source: Apply for a DDS DB instance. For details, see Document Database Service Getting Started.
    +
  2. Download the authentication credential. Skip this step when you access DWS, RDS, DCS, or DDS data sources.
    • CSS security cluster: On the Cluster Management page, click the cluster name. On the Basic Information page that is displayed, find the Security Mode and download the certificate of the CSS security cluster.
    • MRS security cluster: On the cluster list page, click the cluster name. On the displayed cluster information page, download the authentication credential. After downloading the authentication credential, decompress it to obtain the krb5.conf and user.keytab files.
    +
  3. Upload the authentication credential. Skip this step when you access DWS, RDS, DCS, or DDS data sources.

    Upload the obtained authentication credential file to the user-defined OBS bucket.

    +
  4. On the DLI management console, click Datasource Connections.
  5. On the Datasource Authentication tab page, click Create to create an authentication information.
    • CSS +
      + + + + + + + + + + + + + + + + + + + +
      Table 1 Parameters

      Parameter

      +

      Description

      +

      Type

      +

      Select CSS.

      +

      Authentication Certificate

      +

      Name of the datasource authentication information to be created.

      +
      • The name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
      • The length of the database name cannot exceed 128 characters.
      • It is recommended that the name contain the CSS security cluster name to distinguish security authentication information of different clusters.
      +

      Username

      +

      Username for logging in to the security cluster.

      +

      Password

      +

      The password of the security cluster

      +

      Certificate Path

      +

      The OBS path for uploading the security certificate

      +
      +
      +
    • MRS +
      + + + + + + + + + + + + + + + + + + + +
      Table 2 Description

      Parameter

      +

      Description

      +

      Type

      +

      Select Kerberos.

      +

      Authentication Certificate

      +

      Name of the datasource authentication information to be created.

      +
      • The name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
      • The length of the database name cannot exceed 128 characters.
      • It is recommended that the name contain the MRS security cluster name to distinguish security authentication information of different clusters.
      +

      Username

      +

      Username for logging in to the security cluster.

      +

      krb5_conf Path

      +

      OBS path to which the krb5.conf file is uploaded

      +
      NOTE:

      The renew_lifetime configuration item under [libdefaults] must be removed from krb5.conf. Otherwise, the Message stream modified (41) problem may occur.

      +
      +

      keytab Path

      +

      OBS path to which the user.keytab file is uploaded

      +
      +
      +
    • Kafka +
      + + + + + + + + + + + + + + + + + + + + + + + + + +
      Table 3 Parameters

      Parameter

      +

      Description

      +

      Type

      +

      Select Kafka_SSL.

      +

      Authentication Certificate

      +

      Name of the datasource authentication information to be created.

      +
      • The name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
      • The length of the database name cannot exceed 128 characters.
      +

      Truststore Path

      +

      OBS path to which the SSL truststore file is uploaded.

      +

      Truststore Password

      +

      Truststore password. The default value is dms@kafka.

      +

      Keystore Path

      +

      OBS path to which the SSL keystore file (key and certificate) is uploaded.

      +

      Keystore Password

      +

      Keystore (key and certificate) password.

      +

      Key Password

      +

      Password of the private key in the keystore file.

      +
      +
      +
    • Password (datasource password authentication)

      Create datasource authentication for accessing DWS, RDS, DCS, and DDS data sources.

      +

      Currently, database password authentication supports Spark SQL jobs only.

      +
      +

      + +
      + + + + + + + + + + + + + + + + +
      Table 4 Parameters

      Parameter

      +

      Description

      +

      Type

      +

      Select Password.

      +

      Authentication Certificate

      +

      Name of the datasource authentication information to be created.

      +
      • The name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
      • The length of the database name cannot exceed 128 characters.
      +

      Username

      +

      Username for accessing the datasource

      +

      Password

      +

      Password for accessing the datasource

      +
      +
      +
    +
+
+

Searching for a Datasource Authentication

On the Datasource Authentication tab, you can enter the authentication information name in the search box to search for the matching authentication information. To ensure user information security, the password field is not returned.

+
+

Updating Authentication Information

On the Datasource Authentication tab, click Update in the Operation column of the authentication information to be modified. Currently, only the username and password can be updated. If you need to update the certificate, delete the authentication information and create a new one.

+

The username and password are optional. If they are not set, the field is not modified.

+
+
+

Deleting a Datasource Authentication

On the Datasource Authentication tab, click Delete in the Operation column of the authentication information to be deleted.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0440.html b/docs/dli/umn/dli_01_0440.html new file mode 100644 index 00000000..5c0bb8ba --- /dev/null +++ b/docs/dli/umn/dli_01_0440.html @@ -0,0 +1,192 @@ + + +

Overview

+

DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM to manage DLI permissions. You can use both the DLI's permission control mechanism and the IAM service for permission management.

+

Application Scenarios of IAM Authentication

When using DLI on the cloud, enterprise users need to manage DLI resources (queues) used by employees in different departments, including creating, deleting, using, and isolating resources. In addition, data of different departments needs to be managed, including data isolation and sharing.

+

DLI uses IAM for refined enterprise-level multi-tenant management. IAM provides identity authentication, permissions management, and access control, helping you securely access to your cloud resources.

+

With IAM, you can use your cloud account to create IAM users for your employees, and assign permissions to the users to control their access to specific resource types. For example, some software developers in your enterprise need to use DLI resources but must not delete them or perform any high-risk operations. To achieve this result, you can create IAM users for the software developers and grant them only the permissions required for using DLI resources.

+

For a new user, you need to log in for the system to record the metadata before using DLI.

+
+

IAM is free of charge. You pay only for the resources you use.

+

If your cloud account does not need individual IAM users for permissions management, skip this chapter.

+
+

DLI System Permissions

Table 1 lists all the system-defined roles and policies supported by DLI.

+
Type: There are roles and policies.
  • Roles: A type of coarse-grained authorization mechanism that defines permissions related to user responsibilities. Only a limited number of service-level roles are available. When using roles to grant permissions, you also need to assign other roles on which the permissions depend. However, roles are not an ideal choice for fine-grained authorization and secure access control.
  • Policies: A type of fine-grained authorization mechanism that defines permissions required to perform operations on specific cloud resources under certain conditions. This mechanism allows for more flexible policy-based authorization, meeting requirements for secure access control. For example, you can grant DLI users only the permissions for managing a certain type of ECSs.
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 DLI system permissions

Role/Policy Name

+

Description

+

Category

+

DLI FullAccess

+

Full permissions for DLI.

+

System-defined policy

+

DLI ReadOnlyAccess

+

Read-only permissions for DLI.

+

With read-only permissions, you can use DLI resources and perform operations that do not require fine-grained permissions. For example, create global variables, create packages and package groups, submit jobs to the default queue, create tables in the default database, create datasource connections, and delete datasource connections.

+

System-defined policy

+

Tenant Administrator

+

Tenant administrator

+
  • Administer permissions for managing and accessing all cloud services. After a database or a queue is created, the user can use the ACL to assign rights to other users.
  • Scope: project-level service
+

System-defined role

+

DLI Service Admin

+

DLI administrator

+
  • Administer permissions for managing and accessing the queues and data of DLI. After a database or a queue is created, the user can use the ACL to assign rights to other users.
  • Scope: project-level service
+

System-defined role

+
+
+

For details, see Creating an IAM User and Granting Permissions.

+
+

DLI Permission Types

Table 2 lists the DLI service permissions. For details about the resources that can be controlled by DLI, see Table 4.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 DLI permission types

Permission Type

+

Subtype

+

Console Operations

+

SQL Syntax

+

API Definition

+

Queue Permissions

+

Queue management permissions

+

For details, see Queue Permission Management.

+

None

+

For details, see "Granting Users with the Queue Usage Permission" in the Data Lake Insight API Reference.

+

Queue usage permission

+

Data Permissions

+

Database permissions

+

For details, see Database Permission Management and Table Permission Management.

+

For details, see SQL Syntax of Batch Jobs > Data Permissions Management > Data Permissions List in the Data Lake Insight SQL Syntax Reference.

+

For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

+

Table permissions

+

Column permissions

+

Job Permissions

+

Flink job permissions

+

For details, see Managing Flink Job Permissions.

+

None

+

For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

+

Package Permissions

+

+

Package group permissions

+

For details, see Managing Permissions on Packages and Package Groups.

+

None

+

For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

+

Package permissions

+

Datasource Connection Permissions

+

Datasource connection permissions

+

For details, see Managing Datasource Connection Permissions.

+

None

+

For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

+
+
+
+

Examples

An Internet company mainly provides game and music services. DLI is used to analyze user behaviors and assist decision making.

+

As shown in Figure 1, the Leader of the Basic Platform Team has applied for a Tenant Administrator account to manage and use cloud services. Since the Big Data Platform Team needs DLI for data analysis, the Leader of the Basic Platform Team adds a subaccount with the permission of DLI Service Admin to manage and use DLI. The Leader of the Basic Platform Team creates a Queue A and assigns it to Data Engineer A to analyze the gaming data. A Queue B is also assigned to Data Engineer B to analyze the music data. Besides granting the queue usage permission, the Leader of the Basic Platform Team grants data (except the database) management and usage permissions to the two engineers.

+
Figure 1 Granting permissions
+

The Data Engineer A creates a table named gameTable for storing game prop data and a table named userTable for storing game user data. The music service is a new service. To explore potential music users among existing game users, the Data Engineer A assigns the query permission on the userTable to the Data Engineer B. In addition, Data Engineer B creates a table named musicTable for storing music copyrights information.

+

Table 3 describes the queue and data permissions of Data Engineer A and Data Engineer B.

+ +
+ + + + + + + + + + + + + + + + +
Table 3 Permission description

User

+

Data Engineer A (game data analysis)

+

Data Engineer B (music data analysis)

+

Queues

+

Queue A (queue usage permission)

+

Queue B (queue usage permission)

+

Data (Table)

+

gameTable (table management and usage permission)

+

musicTable (table management and usage permission)

+

userTable (table management and usage permission)

+

userTable (table query permission)

+
+
+

The queue usage permission includes job submitting and terminating permissions.

+
+
+
+
+
+
Parent topic: Permissions Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0441.html b/docs/dli/umn/dli_01_0441.html new file mode 100644 index 00000000..8f85f55e --- /dev/null +++ b/docs/dli/umn/dli_01_0441.html @@ -0,0 +1,596 @@ + + +

Common Operations Supported by DLI System Policy

+

Table 1 lists the common operations supported by each system policy of DLI. Choose proper system policies according to this table. For details about the SQL statement permission matrix in DLI in terms of permissions on databases, tables, and roles, see SQL Syntax of Batch Jobs > Data Permissions Management > Data Permissions List in the Data Lake Insight SQL Syntax Reference.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Common operations supported by each system policy

Resources

+

Operation

+

Description

+

DLI FullAccess

+

DLI ReadOnlyAccess

+

Tenant Administrator

+

DLI Service Admin

+

Queue

+

DROP_QUEUE

+

Deleting a queue

+

√

+

×

+

√

+

√

+

SUBMIT_JOB

+

Submitting the job

+

√

+

×

+

√

+

√

+

CANCEL_JOB

+

Terminating the job

+

√

+

×

+

√

+

√

+

RESTART

+

Restarting a queue

+

√

+

×

+

√

+

√

+

GRANT_PRIVILEGE

+

Granting permissions to the queue

+

√

+

×

+

√

+

√

+

REVOKE_PRIVILEGE

+

Revoking permissions from the queue

+

√

+

×

+

√

+

√

+

SHOW_PRIVILEGES

+

Viewing the queue permissions of other users

+

√

+

×

+

√

+

√

+

Database

+

DROP_DATABASE

+

Deleting a database

+

√

+

×

+

√

+

√

+

CREATE_TABLE

+

Creating a table

+

√

+

×

+

√

+

√

+

CREATE_VIEW

+

Creating a view

+

√

+

×

+

√

+

√

+

EXPLAIN

+

Explaining the SQL statement as an execution plan

+

√

+

×

+

√

+

√

+

CREATE_ROLE

+

Creating a role

+

√

+

×

+

√

+

√

+

DROP_ROLE

+

Deleting a role

+

√

+

×

+

√

+

√

+

SHOW_ROLES

+

Displaying a role

+

√

+

×

+

√

+

√

+

GRANT_ROLE

+

Binding a role

+

√

+

×

+

√

+

√

+

REVOKE_ROLE

+

Unbinding the role

+

√

+

×

+

√

+

√

+

SHOW_USERS

+

Displaying the binding relationships between all roles and users

+

√

+

×

+

√

+

√

+

GRANT_PRIVILEGE

+

Granting permissions to the database

+

√

+

×

+

√

+

√

+

REVOKE_PRIVILEGE

+

Revoking permissions to the database

+

√

+

×

+

√

+

√

+

SHOW_PRIVILEGES

+

Viewing database permissions of other users

+

√

+

×

+

√

+

√

+

DISPLAY_ALL_TABLES

+

Displaying tables in the database

+

√

+

√

+

√

+

√

+

DISPLAY_DATABASE

+

Displaying databases

+

√

+

√

+

√

+

√

+

CREATE_FUNCTION

+

Creating a function

+

√

+

×

+

√

+

√

+

DROP_FUNCTION

+

Deleting a function

+

√

+

×

+

√

+

√

+

SHOW_FUNCTIONS

+

Displaying all functions

+

√

+

×

+

√

+

√

+

DESCRIBE_FUNCTION

+

Displaying function details

+

√

+

×

+

√

+

√

+

Table

+

DROP_TABLE

+

Deleting a table

+

√

+

×

+

√

+

√

+

SELECT

+

Querying a table

+

√

+

×

+

√

+

√

+

INSERT_INTO_TABLE

+

Inserting

+

√

+

×

+

√

+

√

+

ALTER_TABLE_ADD_COLUMNS

+

Adding a column

+

√

+

×

+

√

+

√

+

INSERT_OVERWRITE_TABLE

+

Rewriting

+

√

+

×

+

√

+

√

+

ALTER_TABLE_RENAME

+

Renaming a table

+

√

+

×

+

√

+

√

+

ALTER_TABLE_ADD_PARTITION

+

Adding partitions to the partition table

+

√

+

×

+

√

+

√

+

ALTER_TABLE_RENAME_PARTITION

+

Renaming a table partition

+

√

+

×

+

√

+

√

+

ALTER_TABLE_DROP_PARTITION

+

Deleting partitions from a partition table

+

√

+

×

+

√

+

√

+

SHOW_PARTITIONS

+

Displaying all partitions

+

√

+

×

+

√

+

√

+

ALTER_TABLE_RECOVER_PARTITION

+

Restoring table partitions

+

√

+

×

+

√

+

√

+

ALTER_TABLE_SET_LOCATION

+

Setting the partition path

+

√

+

×

+

√

+

√

+

GRANT_PRIVILEGE

+

Granting permissions to the table

+

√

+

×

+

√

+

√

+

REVOKE_PRIVILEGE

+

Revoking permissions from the table

+

√

+

×

+

√

+

√

+

SHOW_PRIVILEGES

+

Viewing table permissions of other users

+

√

+

×

+

√

+

√

+

DISPLAY_TABLE

+

Displaying a table

+

√

+

√

+

√

+

√

+

DESCRIBE_TABLE

+

Displaying table information

+

√

+

×

+

√

+

√

+
+
+
+
+
+
Parent topic: Permissions Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0443.html b/docs/dli/umn/dli_01_0443.html new file mode 100644 index 00000000..aa7598f1 --- /dev/null +++ b/docs/dli/umn/dli_01_0443.html @@ -0,0 +1,15 @@ + + +

Modifying the CIDR Block

+

If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.

+

If the queue whose CIDR block is to be modified has jobs that are being submitted or running, or the queue has been bound to enhanced datasource connections, the CIDR block cannot be modified.

+
+

Procedure

  1. On the left of the DLI management console, click Resources >Queue Management.
  2. Select the queue to be modified and click Modify CIDR Block in the Operation column.
  3. Enter the required CIDR block and click OK.
+
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0447.html b/docs/dli/umn/dli_01_0447.html new file mode 100644 index 00000000..0e51dd9d --- /dev/null +++ b/docs/dli/umn/dli_01_0447.html @@ -0,0 +1,81 @@ + + +

Database Permission Management

+

Operation Scenarios

+
+

Precautions

+
+

Viewing Database Permissions

  1. On the left of the management console, choose Data Management > Databases and Tables.
  2. Locate the row where the target database resides and click Manage Permissions in the Operation column.

    Permissions can be granted to new users or projects, modified for users or projects with existing permissions, or revoked from a user or project.

    +
+
+

Granting Permissions to a New User or Project

Here, the new user or project refers to a user or a project that does not have permissions on the database.
  1. Click a database you need. In the displayed Database Permission Management page, click Grant Permission in the upper right corner.
  2. In the displayed dialog box, select User or Project, enter the username or select the project to be authorized, and select the required permissions. For details about the permissions, see Table 1. +
    + + + + + + + + + + + + + + + + +
    Table 1 Parameters

    Parameter

    +

    Description

    +

    Authorization Object

    +

    Select User or Project.

    +

    Username/Project Name

    +
    • If you select User, enter the IAM username when adding a user to the database.
      NOTE:

      The username is an existing IAM user name and has logged in to the DLI management console.

      +
      +
    • If you select Project, select the project to be authorized in the current region.
      NOTE:

      When Project is selected:

      +
      • If you set Non-inheritable Permissions, you cannot view tables in the corresponding database within the project.
      • If you set Inheritable Permissions, you can view all tables in the database within the project.
      +
      +
    +

    Non-Inherited Permissions

    +

    Select a permission to grant it to the user, or deselect a permission to revoke it.

    +

    Non-inherited permissions apply only to the current database.

    +
    • The following permissions are applicable to both user and project authorization:
      • Drop Database: This permission allows you to delete the current database.
      • Create Table: This permission allows you to create tables in the current database.
      • Create View: This permission allows you to create views in the current database.
      • Execute SQL EXPLAIN: This permission allows you to execute an EXPLAIN statement and view information about how this database executes a query.
      • Create Role: This permission allows you to create roles in the current database.
      • Delete Role: This permission allows you to delete roles of the current database.
      • View Role: This permission allows you to view the role of the current user.
      • Bind Role: This permission allows you to bind roles to the current database.
      • Unbind Role: This permission allows you to bind roles from the current database.
      • View All Binding Relationships: This permission allows you to view the binding relationships between all roles and users.
      • Create Function: This permission allows you to create a function in the current database.
      • Delete Function: This permission allows you to delete functions from the current database.
      • View All Functions: This permission allows you to view all functions in the current database.
      • View Function Details: This permission allows you to view details about the current function.
      +
    • The following permissions can only be granted to users:
      • View All Tables: This permission allows you to view all tables in the current database.
        NOTE:

        If this permission of a specific database is not granted, all tables in the database will not be displayed.

        +
        +
      • View Database: This permission allows you to view the information about the current database.
        NOTE:

        If this permission is not granted, the database will not be displayed.

        +
        +
      +
    +

    Inherited Permissions

    +

    Select a permission to grant it to the user, or deselect a permission to revoke it.

    +

    Inherited permissions are applicable to the current database and all its tables. However, only the query permission is applicable to table columns.

    +

    The following permissions can be granted to both user and project.

    +
    • Drop Table: This permission allows you to delete tables in a database.
    • Select Table: This permission allows you to query data of the current table.
    • View Table Information: This permission allows you to view information about the current table.
    • Insert: This permission allows you to insert data into the current table.
    • Add Column: This permission allows you to add columns to the current table.
    • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
    • Grant Permission: This permission allows you to grant database permissions to other users or projects.
    • Revoke Permission: This permission allows you to revoke the permissions of the database that other users have but cannot revoke the database owner's permissions.
    • Add Partition to Partition Table: This permission allows you to add a partition to a partition table.
    • Delete Partition from Partition Table: This permission allows you to delete existing partitions from a partition table.
    • Configure Path for Partition: This permission allows you to set the path of a partition in a partition table to a specified OBS path.
    • Rename Table Partition: This permission allows you to rename partitions in a partition table.
    • Rename Table: This permission allows you to rename tables.
    • Restore Table Partition: This permission allows you to export partition information from the file system and save the information to metadata.
    • View All Partitions: This permission allows you to view all partitions in a partition table.
    • View Other Users' Permissions: This permission allows you to query other users' permission on the current database.
    +
    +
    +
  3. Click OK.
+
+
+

Modifying Permissions for an Existing User or Project

For a user or project that has some permissions on the database, you can revoke the existing permissions or grant new ones.

+

If the options in Set Permission are gray, the corresponding account does not have the permission to modify the database. You can apply to the administrator, database owner, or other authorized users for granting and revoking permissions of databases.

+
+
  1. In the User Permission Info list, find the user whose permission needs to be set.
    • If the user is a sub-user, you can set permissions for it.
    • If the user is already an administrator, you can only view the permissions information.
    +

    In the Project Permission Info list, locate the project for which you want to set permissions and click Set Permission.

    +
  2. In the Operation column of the sub-user or project, click Set Permission. The Set Permission dialog box is displayed.

    For details about the permissions of database users or projects, see Table 1.

    +
  3. Click OK.
+
+
+

Revoking All Permissions of a User or Project

Revoke all permissions of a user or a project.
  • In the user list under User Permission Info, locate the row where the target sub-user resides and click Revoke Permission in the Operation column. In the displayed dialog box, click OK. In this case, the user has no permissions on the database.

    If a user is an administrator, Revoke Permission is gray, indicating that the user's permission cannot be revoked.

    +
    +
  • In the Project Permission Info area, select the project whose permissions need to be revoked and click Revoke Permission in the Operation column. After you click OK, the project does not have any permissions on the database.
+
+
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0448.html b/docs/dli/umn/dli_01_0448.html new file mode 100644 index 00000000..eb88f5c2 --- /dev/null +++ b/docs/dli/umn/dli_01_0448.html @@ -0,0 +1,145 @@ + + +

Table Permission Management

+

Operation Scenario

+
+

Precautions

+
+

Viewing Table Permissions

  1. On the left of the management console, choose Data Management > Databases and Tables.
  2. Click the database name in the table whose authority is to be set. The Table Management page of the database is displayed.
  3. Locate the row where the target table resides and click Manage Permissions in the Operation column.

    Permissions can be granted to new users or projects, modified for users or projects with existing permissions, or revoked from a user or project.

    +
+
+

Granting Permissions to a New User or a Project

Here, the new user or project refers to a user or a project that does not have permissions on the database.
  1. Click the table you need. In the displayed table permissions page, click Grant Permission in the upper right corner.
  2. In the displayed Grant Permission dialog box, select the required permissions.
    • For details about the DLI table permissions, see Table 1.

      + +
      + + + + + + + + + + + + + +
      Table 1 Parameter description

      Parameter

      +

      Description

      +

      Authorization Object

      +

      Select User or Project.

      +

      Username/Project

      +
      • If you select User, enter the IAM username when granting table permissions to the user.
        NOTE:

        The username is an existing IAM user name and has logged in to the DLI management console.

        +
        +
      • If you select Project, select the project to be authorized in the current region.
        NOTE:

        If you select Project, you can only view information about the authorized tables and their databases.

        +
        +
      +

      Non-inheritable Permissions

      +

      Select a permission to grant it to the user, or deselect a permission to revoke it.

      +
      • The following permissions are applicable to both user and project authorization:
        • Select Table: This permission allows you to query data of the current table.
        • View Table Information: This permission allows you to view information about the current table.
        • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
        • Drop Table: This permission allows you to delete the current table.
        • Rename Table: Rename the current table.
        • Insert: This permission allows you to insert data into the current table.
        • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
        • Add Column: Add columns to the current table.
        • Grant Permission: The current user can grant table permissions to other users.
        • Revoke Permission: The current user can revoke the table's permissions that other users have but cannot revoke the table owner's permissions.
        • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
        +

        The partition table also has the following permissions:

        +
        • Delete Partition: This permission allows you to delete existing partitions from a partition table.
        • View All Partitions: This permission allows you to view all partitions in a partition table.
        +
      +
      • The following permissions can only be granted to users:
        • View Table: This permission allows you to display the current table.
        +
      +
      +
      +
    • For details about the OBS table permissions, see Table 2.

      + +
      + + + + + + + + + + + + + +
      Table 2 Parameter description

      Parameter

      +

      Description

      +

      Authorization Object

      +

      Select User or Project.

      +

      Username/Project

      +
      • If you select User, enter the IAM username when granting table permissions to the user.
        NOTE:

        The username is an existing IAM user name and has logged in to the DLI management console.

        +
        +
      • If you select Project, select the project to be authorized in the current region.
        NOTE:

        If you select Project, you can only view information about the authorized tables and their databases.

        +
        +
      +

      Non-inheritable Permissions

      +
      Select a permission to grant it to the user, or deselect a permission to revoke it.
      • The following permissions are applicable to both user and project authorization:
        • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
        • View Table Information: This permission allows you to view information about the current table.
        • Select Table: This permission allows you to query data of the current table.
        • Drop Table: This permission allows you to delete the current table.
        • Rename Table: Rename the current table.
        • Insert: This permission allows you to insert data into the current table.
        • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
        • Add Column: This permission allows you to add columns to the current table.
        • Grant Permission: This permission allows you to grant table permissions to other users or projects.
        • Revoke Permission: This permission allows you to revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
        • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
        +
        The partition table also has the following permissions:
        • Add Partition: This permission allows you to add a partition to a partition table.
        • Delete Partition: This permission allows you to delete existing partitions from a partition table.
        • Configure Path for Partition: This permission allows you to set the path of a partition in a partition table to a specified OBS path.
        • Rename Table Partition: This permission allows you to rename partitions in a partition table.
        • Restore Table Partition: This permission allows you to export partition information from the file system and save the information to metadata.
        • View All Partitions: This permission allows you to view all partitions in a partition table.
        +
        +
      +
      +
      • The following permissions can only be granted to users:
        • View Table: This permission allows you to view the current table.
        +
      +
      +
      +
    • For details about the view permissions, see Table 3.

      A view can be created only by using SQL statements. You cannot create a view on the Create Table page.

      +
      + +
      + + + + + + + + + + + + + +
      Table 3 Parameter description

      Parameter

      +

      Description

      +

      Authorization Object

      +

      Select User or Project.

      +

      Username/Project

      +
      • If you select User, enter the IAM username when adding a user to the database.
        NOTE:

        The username is an existing IAM user name and has logged in to the DLI management console.

        +
        +
      • If you select Project, select the project to be authorized in the current region.
        NOTE:

        If you select Project, you can only view information about the authorized tables and their databases.

        +
        +
      +

      Non-inheritable Permissions

      +
      Select a permission to grant it to the user, or deselect a permission to revoke it.
      • The following permissions are applicable to both user and project authorization:
        • View Table Information: This permission allows you to view information about the current table.
        • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
        • Drop Table: This permission allows you to delete the current table.
        • Select Table: This permission allows you to query data of the current table.
        • Rename Table: Rename the current table.
        • Grant Permission: The current user or project can grant table permissions to other users or projects.
        • Revoke Permission: The current user or project can revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
        • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
        +
      • Only applicable to
        • View Table: This permission allows you to view the current table.
        +
      +
      +
      +
      +
    +
  3. Click OK.
+
+
+

Modifying Permissions for an Existing User or Project

For a user or project that has some permissions on the database, you can revoke the existing permissions or grant new ones.

+

If all options under Set Permission are gray, you are not allowed to change permissions on this table. You can apply to the administrator, table owner, or other authorized users for granting and revoking table permissions.

+
+
  1. In the User Permission Info list, find the user whose permission needs to be set.
    • If the user is a sub-user and is not the owner of the table, you can set permissions.
    • If the user is an administrator or table owner, you can only view permissions.
    +

    In the Project Permission Info list, locate the project for which you want to set permissions and click Set Permission.

    +
  2. In the Operation column of the sub-user or project, click Set Permission. The Set Permission dialog box is displayed.
    • For details about DLI table user or project permissions, see Table 1.
    • For details about OBS table user or project permissions, see Table 2.
    • For details about View table user or project permissions, see Table 3.
    +
  3. Click OK.
+
+
+

Revoking All Permissions of a User or Project

Revoke all permissions of a user or a project.

+ +
+
+
+
+
Parent topic: Databases and Tables
+
+
+ diff --git a/docs/dli/umn/dli_01_0451.html b/docs/dli/umn/dli_01_0451.html new file mode 100644 index 00000000..616c8ed3 --- /dev/null +++ b/docs/dli/umn/dli_01_0451.html @@ -0,0 +1,316 @@ + + +

Creating a Custom Policy

+

Custom policies can be created as a supplement to the system policies of DLI. You can add actions to custom policies. For the actions supported for custom policies, see "Permissions Policies and Supported Actions" in the Elastic Volume Service API Reference.

+

You can create custom policies in either of the following two ways:

+ +

. This section describes common DLI custom policies.

+

Policy Field Description

The following example assumes that the authorized user has the permission to create tables in all databases in all regions:

+
{
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "dli:database:create_table"
+            ],
+            "Resource": [
+                "dli:*:*:database:*"
+            ]
+        }
+    ]
+}
+ +
+

Creating a Custom Policy

You can set actions and resources of different levels based on scenarios.

+
  1. Define an action.

    The format is Service name:Resource type:Action. The wildcard is *. Example:

    + +
    + + + + + + + + + + + + + +
    Table 2 Action

    Action

    +

    Description

    +

    dli:queue:submit_job

    +

    Submission operations on a DLI queue

    +

    dli:queue:*

    +

    All operations on a DLI queue

    +

    dli:*:*

    +

    All operations on all DLI resource types

    +
    +
    +

    For more information about the relationship between operations and system permissions, see Common Operations Supported by DLI System Policy.

    +
  2. Define a resource.

    The format is Service name:Region:Domain ID:Resource type:Resource path. The wildcard (*) indicates all resources. The five fields can be flexibly set. Different levels of permission control can be set for resource paths based on scenario requirements. If you need to set all resources of the service, you do not need to specify this field. For details about the definition of Resource, see Table 3. For details about the resource types and paths in Resource, see Table 4.

    + +
    + + + + + + + + + + + + + + + + +
    Table 3 Resource

    Resource

    +

    Description

    +

    DLI:*:*:table:databases.dbname.tables.*

    +

    DLI, any region, any account ID, all table resources of database dbname

    +

    DLI:*:*:database:databases.dbname

    +

    DLI, any region, any account ID, queue of database dbname

    +

    DLI:*:*:queue:queues.*

    +

    DLI, any region, any account ID, any queue resource

    +

    DLI:*:*:jobs:jobs.flink.1

    +

    DLI, any region, any account ID, Flink job whose ID is 1

    +
    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 4 DLI resources and their paths

    Resource Type

    +

    Resource Names

    +

    Path

    +

    queue

    +

    DLI queue

    +

    queues.queuename

    +

    database

    +

    DLI database

    +

    databases.dbname

    +

    table

    +

    DLI table

    +

    databases.dbname.tables.tbname

    +

    column

    +

    DLI column

    +

    databases.dbname.tables.tbname.columns.colname

    +

    jobs

    +

    DLI Flink job

    +

    jobs.flink.jobid

    +

    resource

    +

    DLI package

    +

    resources.resourcename

    +

    group

    +

    DLI package group

    +

    groups.groupname

    +

    datasourceauth

    +

    DLI cross-source authentication information

    +

    datasourceauth.name

    +
    +
    + +
  3. Combine all the preceding fields into a JSON file to form a complete policy. You can set multiple actions and resources. You can also create a policy on the visualized page provided by IAM. For example:

    The authorized user has the permission to create and delete any database, submit jobs for any queue, and delete any table under any account ID in any region of DLI.

    +
    {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": " Allow",
+            "Action": [
+                  "dli:database:create_database",
+                "dli:database:drop_database",
+                  "dli:queue:submit_job",
+                  "dli:table:drop_table"
+            ],
+            "Resource": [
+                  "dli:*:*:database:*",
+                "dli:*:*:queue:*",
+                "dli:*:*:table:*"
+            ]
+        }
+    ]
+}
    +
+
+

Example Custom Policies

+
+
+
+
+
Parent topic: Permissions Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0454.html b/docs/dli/umn/dli_01_0454.html new file mode 100644 index 00000000..ad3e3c99 --- /dev/null +++ b/docs/dli/umn/dli_01_0454.html @@ -0,0 +1,28 @@ + + +

Preparing Flink Job Data

+

To create a Flink job, you need to enter the data source and data output channel, that is, source and sink. To use another service as the source or sink stream, you need to apply for the service first.

+

Flink jobs support the following data sources and output channels:

+ +
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0455.html b/docs/dli/umn/dli_01_0455.html new file mode 100644 index 00000000..35e9d2cb --- /dev/null +++ b/docs/dli/umn/dli_01_0455.html @@ -0,0 +1,168 @@ + + +

Creating a Flink SQL job

+

This section describes how to create a Flink SQL job. You can use Flink SQLs to develop jobs to meet your service requirements. Using SQL statements simplifies logic implementation. You can edit Flink SQL statements for your job in the DLI SQL editor. This section describes how to use the SQL editor to write Flink SQL statements.

+

DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the community. In addition, Redis, DWS(GaussDB), and DIS data source types are added based on the community connector.

+

Prerequisites

+
+

Creating a Flink SQL Job

  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. In the upper right corner of the Flink Jobs page, click Create Job.
  3. Specify job parameters.

    +

    + + + + + + + + + + + + + + + + +
    Table 1 Job configuration information

    Parameter

    +

    Description

    +

    Type

    +

    Set Type to Flink SQL. You will need to rewrite SQL statements to start the job.

    +

    Name

    +

    Name of a job. Enter 1 to 57 characters. Only letters, digits, hyphens (-), and underscores (_) are allowed.

    +
    NOTE:

    The job name must be globally unique.

    +
    +

    Description

    +

    Description of a job. It can contain up to 512 characters.

    +

    Template Name

    +

    You can select a sample template or a custom job template. For details about templates, see Flink Template Management.

    +
    +
    +

  4. Click OK to enter the Edit page.
  5. Edit a Flink SQL job.

    Enter details SQL statements in the statement editing area. For details about SQL syntax, see the Data Lake Insight SQL Syntax Reference.

    +

  6. Click Check Semantics.

    • You can Debug or Start a job only after the semantic verification is successful.
    • If verification is successful, the message "The SQL semantic verification is complete. No error." will be displayed.
    • If verification fails, a red "X" mark will be displayed in front of each SQL statement that produced an error. You can move the cursor to the "X" mark to view error details and change the SQL statement as prompted.
    +

  7. Set job running parameters.

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 2 Running parameters

    Parameter

    +

    Description

    +

    Queue

    +

    A shared queue is selected by default. You can select a custom queue as needed.

    +
    NOTE:
    • During job creation, a sub-user can only select a queue that has been allocated to the user.
    • If the remaining capacity of the selected queue cannot meet the job requirements, the system automatically scales up the capacity. When a queue is idle, the system automatically scales in the queue.
    +
    +

    UDF Jar

    +

    If you selected custom queues, you need to configure this parameter.

    +

    You can customize a UDF Jar file. Before you select a JAR file, upload the corresponding JAR package to the OBS bucket and choose Data Management > Package Management to create a package. For details, see Creating a Package.

    +

    In SQL, you can call a user-defined function that is inserted into a JAR file.

    +

    CUs

    +

    Sum of the number of compute units and job manager CUs of DLI. One CU equals one vCPU and 4 GB.

    +

    The configured number of CUs is the number of CUs required for job running and cannot exceed the number of CUs in the bound queue.

    +

    Job Manager CUs

    +

    Number of CUs of the management unit.

    +

    Parallelism

    +

    Number of Flink SQL jobs that run at the same time Properly increasing the number of parallel threads improves the overall computing capability of the job. However, the switchover overhead caused by the increase of threads must be considered.

    +
    NOTE:
    • This value cannot be greater than four times the compute units (number of CUs minus the number of job manager CUs).
    • The priority of the number of parallel tasks on this page is lower than that set in the code.
    +
    +

    Task Manager Configuration

    +

    Whether to set Task Manager resource parameters

    +

    If this option is selected, you need to set the following parameters:

    +
    • CU(s) per TM: Number of resources occupied by each Task Manager.
    • Slot(s) per TM: Number of slots contained in each Task Manager.
    +

    OBS Bucket

    +

    OBS bucket to store job logs and checkpoint information. If the selected OBS bucket is not authorized, click Authorize.

    +
    NOTE:

    If both Enable Checkpointing and Save Job Log are selected, you only need to authorize OBS once.

    +
    +

    Save Job Log

    +

    Whether to save the job running logs to OBS The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID. To go to this path, go to the job list and click the job name. On the Run Log tab page, click the provided OBS link.

    +
    CAUTION:

    You are advised to select this parameter. Otherwise, no run log is generated after the job is executed. If the job is abnormal, the run log cannot be obtained for fault locating.

    +
    +

    If this option is selected, you need to set the following parameters:

    +
    OBS Bucket: Select an OBS bucket to store user job logs. If the selected OBS bucket is not authorized, click Authorize.
    NOTE:

    If both Enable Checkpointing and Save Job Log are selected, you only need to authorize OBS once.

    +
    +
    +

    Alarm Generation upon Job Exception

    +

    Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email

    +

    If this option is selected, you need to set the following parameters:

    +

    SMN Topic

    +

    Select a user-defined SMN topic. For details about how to customize SMN topics, see Creating a Topic in the Simple Message Notification User Guide.

    +

    Enable Checkpointing

    +

    Whether to enable job snapshots. If this function is enabled, jobs can be restored based on the checkpoints.

    +
    If this option is selected, you need to set the following parameters:
    • Checkpoint Interval indicates the interval for creating checkpoints. The value ranges from 1 to 999999, and the default value is 30.
    • Checkpoint Mode can be set to either of the following values:
      • At least once: Events are processed at least once.
      • Exactly once: Events are processed only once.
      +
    +
    • OBS Bucket: Select an OBS bucket to store your checkpoints. If the selected OBS bucket is not authorized, click Authorize.
      The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
      NOTE:

      You only need to authorize OBS once for both Enable Checkpointing and Save Job Log.

      +
      +
      +
    +
    +

    Auto Restart upon Exception

    +

    Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

    +

    If this option is selected, you need to set the following parameters:

    +
    • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is Times/hour.
      • Unlimited: The number of retries is unlimited.
      • Limited: The number of retries is user-defined.
      +
    • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.
    +

    Idle State Retention Time

    +

    Defines for how long the state of a key is retained without being updated before it is removed in GroupBy or Window. The default value is 1 hour.

    +

    Dirty Data Policy

    +

    Select a policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

    +
    NOTE:

    Save indicates that the dirty data is stored to the OBS bucket selected above.

    +
    +

    Dirty Data Dump Address

    +

    Set this parameter when Dirty Data Policy is set to Save. Click the address box to select the OBS path for storing dirty data.

    +
    +
    +

  8. (Optional) Debug parameters as required. The job debugging function is used only to verify the SQL logic and does not involve data write operations. For details, see Debugging a Flink Job.
  9. (Optional) Set the runtime configuration as required. Set Custom Configuration to User-defined.
  10. Click Save.
  11. Click Start. On the displayed Start Flink Jobs page, confirm the job specifications, and click Start Now to start the job.

    After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, the job status will change from Submitting to Running. After the execution is complete, the message Completed is displayed.

    +

    If the job status is Submission failed or Running exception, the job submission failed or the job did not execute successfully. In this case, you can move the cursor over the status icon in the Status column of the job list to view the error details. You can click to copy error information. After handling the fault based on the provided information, resubmit the job.

    +

    Other available buttons are as follows:

    +
    • Save As: Save the created job as a new job.
    • Debug: Perform job debugging. For details, see Debugging a Flink Job.
    • Format: Format the SQL statements in the editing box.
    • Set as Template: Set the created SQL statements as a job template.
    • Theme Settings: Set the theme related parameters, including Font Size, Wrap, and Page Style.
    +
    +

+
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0457.html b/docs/dli/umn/dli_01_0457.html new file mode 100644 index 00000000..5106d75d --- /dev/null +++ b/docs/dli/umn/dli_01_0457.html @@ -0,0 +1,171 @@ + + +

Creating a Flink Jar Job

+

This section describes how to create a Flink Jar job. You can perform secondary development based on Flink APIs, build your own JAR file, and submit the JAR file to DLI queues. DLI is fully compatible with open-source community APIs. To create a custom Flink job, you need to compile and build application JAR files. You must have a certain understanding of Flink secondary development and have high requirements related to stream computing complexity.

+

Prerequisites

+
+

Creating a Flink Jar Job

  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. In the upper right corner of the Flink Jobs page, click Create Job.
  3. Specify job parameters.

    +

    + + + + + + + + + + + + + +
    Table 1 Job configuration information

    Parameter

    +

    Description

    +

    Type

    +

    Select Flink Jar.

    +

    Name

    +

    Name of a job. Enter 1 to 57 characters. Only letters, digits, hyphens (-), and underscores (_) are allowed.

    +
    NOTE:

    The job name must be globally unique.

    +
    +

    Description

    +

    Description of a job. It can be up to 512 characters long.

    +
    +
    +

  4. Click OK to enter the page for job information editing.
  5. Select a queue. Flink Jar jobs can run only on general queues.

    • A Flink Jar job can run only on a pre-created dedicated queue.
    • If no dedicated queue is available in the Queue drop-down list, create a dedicated queue and bind it to the current user.
    +
    +

  6. Configure Flink Jar job parameters.

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 2 Parameter description

    Name

    +

    Description

    +

    Application

    +

    User-defined package. Before selecting a JAR file to be inserted, upload the corresponding JAR file to the OBS bucket and choose Data Management > Package Management to create a package. For details, see Creating a Package. For details about the built-in dependencies, see Flink 1.7.2 Dependencies and Flink 1.10 Dependencies.

    +

    Main Class

    +

    The name of the JAR package to be loaded, for example, KafkaMessageStreaming.

    +
    • Default: Specified based on the Manifest file in the JAR package.
    • Manually assign: You must enter the class name and confirm the class arguments (separate arguments with spaces).
    +
    NOTE:

    When a class belongs to a package, the main class path must contain the complete package path, for example, packagePath.KafkaMessageStreaming.

    +
    +

    Class Arguments

    +

    List of arguments of a specified class. The arguments are separated by spaces.

    +

    Flink parameters support replacement of non-sensitive global variables. For example, if you add the global variable windowsize in Global Configuration > Global Variables, you can add the -windowsSize {{windowsize}} parameter for the Flink Jar job.

    +

    JAR Package Dependencies

    +

    Select a user-defined package dependency. The dependent program packages are stored in the classpath directory of the cluster.

    +

    Before selecting a JAR file to be inserted, upload the corresponding JAR file to the OBS bucket and choose Data Management > Package Management to create a package. Select JAR as the package type. For details, see Creating a Package.

    +

    For details about the built-in dependencies, see Flink 1.7.2 Dependencies and Flink 1.10 Dependencies.

    +

    Other Dependencies

    +

    User-defined dependency files. Other dependency files need to be referenced in the code.

    +

    Before selecting a dependency file, upload the file to the OBS bucket and choose Data Management > Package Management to create a package. The package type is not limited. For details, see Creating a Package.

    +

    You can add the following command to the application to access the corresponding dependency file. In the command, fileName indicates the name of the file to be accessed, and ClassName indicates the name of the class that needs to access the file.

    +
    ClassName.class.getClassLoader().getResource("userData/fileName")
    +

    Flink Version

    +

    Before selecting a Flink version, you need to select the queue to which the Flink version belongs. Currently, the following versions are supported: 1.10.

    +

    Runtime Configuration

    +

    User-defined optimization parameters. The parameter format is key=value.

    +

    Flink optimization parameters support replacement non-sensitive global variable. For example, if you create global variable phase in Global Configuration > Global Variables, optimization parameter table.optimizer.agg-phase.strategy={{phase}} can be added to the Flink Jar job.

    +
    +
    +
    +

  7. Configure job parameters.

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 3 Parameter description

    Name

    +

    Description

    +

    CUs

    +

    One CU has one vCPU and 4-GB memory. The number of CUs ranges from 2 to 400.

    +

    Job Manager CUs

    +

    Set the number of CUs on a management unit. The value ranges from 1 to 4. The default value is 1.

    +

    Parallelism

    +

    Maximum number of parallel operators in a job.

    +
    NOTE:
    • The value must be less than or equal to four times the number of compute units (CUs minus the number of job manager CUs).
    • You are advised to set this parameter to a value greater than that configured in the code. Otherwise, job submission may fail.
    +
    +

    Task Manager Configuration

    +

    Whether to set Task Manager resource parameters

    +

    If this option is selected, you need to set the following parameters:

    +
    • CU(s) per TM: Number of resources occupied by each Task Manager.
    • Slot(s) per TM: Number of slots contained in each Task Manager.
    +

    Save Job Log

    +

    Whether to save the job running logs to the OBS bucket.

    +
    CAUTION:

    You are advised to select this parameter. Otherwise, no run log is generated after the job is executed. If the job is abnormal, the run log cannot be obtained for fault locating.

    +
    +

    If this option is selected, you need to set the following parameters:

    +

    OBS Bucket: Select an OBS bucket to store job logs. If the selected OBS bucket is not authorized, click Authorize.

    +

    Alarm Generation upon Job Exception

    +

    Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email

    +

    If this option is selected, you need to set the following parameters:

    +

    SMN Topic

    +

    Select a user-defined SMN topic. For details about how to customize SMN topics, see Creating a Topic in the Simple Message Notification User Guide.

    +

    Auto Restart upon Exception

    +

    Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

    +

    If this option is selected, you need to set the following parameters:

    +
    • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is Times/hour.
      • Unlimited: The number of retries is unlimited.
      • Limited: The number of retries is user-defined.
      +
    • Restore Job from Checkpoint: Restore the job from the saved checkpoint.

      If you select this parameter, you also need to set Checkpoint Path.

      +

      Checkpoint Path: Select the checkpoint saving path. The checkpoint path must be the same as that you set in the application package. Note that the checkpoint path for each job must be unique. Otherwise, the checkpoint cannot be obtained.

      +
    +
    +
    +

  8. Click Save on the upper right of the page.
  9. Click Start on the upper right side of the page. On the displayed Start Flink Job page, confirm the job specifications, and click Start Now to start the job.

    After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, the job status will change from Submitting to Running. After the execution is complete, the message Completed is displayed.

    +

    If the job status is Submission failed or Running exception, the job submission failed or the job did not execute successfully. In this case, you can move the cursor over the status icon in the Status column of the job list to view the error details. You can click to copy these details. After handling the fault based on the provided information, resubmit the job.

    +

    Other available buttons are as follows:

    +

    Save As: Save the created job as a new job.

    +
    +

+
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0458.html b/docs/dli/umn/dli_01_0458.html new file mode 100644 index 00000000..3f454911 --- /dev/null +++ b/docs/dli/umn/dli_01_0458.html @@ -0,0 +1,24 @@ + + +

Debugging a Flink Job

+

The job debugging function helps you check the logic correctness of your compiled SQL statements before running a job.

+
  • Currently, only Flink SQL jobs support this function.
  • The job debugging function is used only to verify the SQL logic and does not involve data write operations.
+
+

Procedures

  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. In the Operation column of the created Flink SQL job, click Edit. The page for editing the Flink SQL job is displayed.

    For a job that is being created, you can debug the job on the editing page.

    +

  3. Click Debug above the SQL editing box to parse the edited SQL statements. The Debugging Parameters tab is displayed on the right of the page.

    +
    • Dump Bucket: Select an OBS bucket to save debugging logs. If you select an unauthorized OBS bucket, click Authorize.
    • Data Input Mode: You can select CSV data stored in the OBS bucket or manually enter the data.
      • OBS (CSV)

        If you select this value, prepare OBS data first before using DLI. For details, see Preparing Flink Job Data. OBS data is stored in CSV format, where multiple records are separated by line breaks and different fields in a single record are separated by commas (,). In addition, you need to select a specific object in OBS as the input source data.

        +

        +
      • Manual typing

        If you select this value, compile SQL statements as data sources. In this mode, you need to enter the value of each field in a single record.

        +

        +
      +
    +

  4. Click Start Debugging. Once debugging is complete, the Debugging Result page appears.

    • If the debugging result meets the expectation, the job is running properly.
    • If the debugging result does not meet the expectation, business logic errors may have occurred. In this case, modify SQL statements and conduct debugging again.
    +

+
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0461.html b/docs/dli/umn/dli_01_0461.html new file mode 100644 index 00000000..f73d2853 --- /dev/null +++ b/docs/dli/umn/dli_01_0461.html @@ -0,0 +1,107 @@ + + +

Performing Operations on a Flink Job

+

After a job is created, you can perform operations on the job as required.

+ +

Editing a Job

You can edit a created job, for example, by modifying the SQL statement, job name, job description, or job configurations.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. In the row where the job you want to edit locates, click Edit in the Operation column to switch to the Edit page.
  3. Edit the job as required.

    For details about how to edit a Flink SQL job, see Step 5 to Step 7 in Creating a Flink SQL Job.

    +

    For details about how to edit a user-defined Flink job, see Step 5 to Step 7 in Creating a Flink Jar Job.

    +

+
+

Starting a Job

You can start a saved or stopped job.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Use either of the following methods to start jobs:

    • Starting a single job

      Select a job and click Start in the Operation column.

      +

      Alternatively, you can select the row where the job you want to start locates and click Start in the upper left of the job list.

      +
    • Batch starting jobs

      Select the rows where the jobs you want to start locate and click Start in the upper left of the job list.

      +
    +

    After you click Start, the Start Flink Jobs page is displayed.

    +

  1. On the Start Flink Jobs page, confirm the job information. If they are correct, click Start Now.

    After a job is started, you can view the job execution result in the Status column.

    +

+
+

Stopping a Job

You can stop a job in the Running or Submitting state.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Stop a job using either of the following methods:

    • Stopping a job

      Locate the row that contains the job to be stopped, click More in the Operation column, and select Stop.

      +

      Alternatively, you can select the row where the job you want to stop locates and click Stop in the upper left of the job list.

      +
    • Batch stopping jobs

      Locate the rows containing the jobs you want to stop and click Stop in the upper left of the job list.

      +
    +

  3. In the displayed Stop Job dialog box, click OK to stop the job.

    • Before stopping a job, you can trigger a savepoint to save the job status information. When you start the job again, you can choose whether to restore the job from the savepoint.
    • If you select Trigger savepoint, a savepoint is created. If Trigger savepoint is not selected, no savepoint is created. By default, the savepoint function is disabled.
    • The lifecycle of a savepoint starts when the savepoint is triggered and stops the job, and ends when the job is restarted. The savepoint is automatically deleted after the job is restarted.
    +
    +
    +
    When a job is being stopped, the job status is displayed in the Status column of the job list. The details are as follows:
    • Stopping: indicates that the job is being stopped.
    • Stopped: indicates that the job is stopped successfully.
    • Stop failed: indicates that the job failed to be stopped.
    +
    +

+
+

Deleting a Job

If you do not need to use a job, perform the following operations to delete it. A deleted job cannot be restored. Therefore, exercise caution when deleting a job.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  1. Perform either of the following methods to delete jobs:

    • Deleting a single job

      Locate the row containing the job you want to delete and click More > Delete in the Operation column.

      +

      Alternatively, you can select the row containing the job you want to delete and click Delete in the upper left of the job list.

      +

      +
    • Deleting jobs in batches

      Select the rows containing the jobs you want to delete and click Delete in the upper left of the job list.

      +
    +

  2. Click Yes.
+
+

Exporting a Job

You can export the created Flink jobs to an OBS bucket.

+

This mode is applicable to the scenario where a large number of jobs need to be created when you switch to another region, project, or user. In this case, you do not need to create a job. You only need to export the original job, log in to the system in a new region or project, or use a new user to import the job.

+

When switching to another project or user, you need to grant permissions to the new project or user. For details, see Managing Flink Job Permissions.

+
+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  1. Click Export Job in the upper right corner. The Export Job dialog box is displayed.

    +

  2. Select the OBS bucket where the job is stored. Click Next.
  3. Select job information you want to export.

    By default, configurations of all jobs are exported. You can enable the Custom Export function to export configurations of the desired jobs.

    +

  4. Click Confirm to export the job.
+
+

Importing a Job

You can import the Flink job configuration file stored in the OBS bucket to the Flink Jobs page of DLI.

+

This mode is applicable to the scenario where a large number of jobs need to be created when you switch to another region, project, or user. In this case, you do not need to create a job. You only need to export the original job, log in to the system in a new region or project, or use a new user to import the job.

+

If you need to import a self-created job, you are advised to use the job creation function. For details, see Creating a Flink SQL job, Creating a Flink Jar Job.

+
  • When switching to another project or user, you need to grant permissions to the new project or user. For details, see Managing Flink Job Permissions.
  • Only jobs whose data format is the same as that of Flink jobs exported from DLI can be imported.
+
+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  1. Click Import Job in the upper right corner. The Import Job dialog box is displayed.

    +

  2. Select the complete OBS path of the job configuration file to be imported. Click Next.
  3. Configure the same-name job policy and click next. Click Next.

    • Select Overwrite job of the same name. If the name of the job to be imported already exists, the existing job configuration will be overwritten and the job status switches to Draft.
    • If Overwrite job of the same name is not selected and the name of the job to be imported already exists, the job will not be imported.
    +

  4. Ensure that Config File and Overwrite Same-Name Job are correctly configured. Click Confirm to export the job.
+
+

Modifying Name and Description

You can change the job name and description as required.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. In the Operation column of the job whose name and description need to be modified, choose More > Modify Name and Description. The Modify Name and Description dialog box is displayed. Change the name or modify the description of a job.
  3. Click OK.
+
+

+

Triggering a Savepoint

You can click Trigger Savepoint for jobs in the Running status to save the job status.

+
+

Importing to a Savepoint

You can import a savepoint to restore the job status. For details about the savepoint, see Checkpointing at the official website of Flink.

+

You need to select the OBS path of the save point.

+
+

Runtime Configuration

You can select Runtime Configuration to configure job exception alarms and restart options.

+

Flink SQL jobs and Flink Jar jobs are supported.

+
+
  1. In the Operation column of the Flink job, choose More > Runtime Configuration.
  2. In the Runtime Configuration dialog box, set the following parameters: +
    + + + + + + + + + + +
    Table 1 Running parameters

    Parameter

    +

    Description

    +

    Alarm Generation upon Job Exception

    +

    Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email.

    +

    If this option is selected, you need to set the following parameters:

    +

    SMN Topic

    +

    Select a user-defined SMN topic. For details about how to customize SMN topics, see Creating a Topic in the Simple Message Notification User Guide.

    +

    Auto Restart upon Exception

    +

    Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

    +

    If this option is selected, you need to set the following parameters:

    +
    • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is times/hour.
      • Unlimited: The number of retries is unlimited.
      • Limited: The number of retries is user-defined.
      +
    • Restore Job from Checkpoint: Restore the job from the saved checkpoint.
      NOTE:

      For Flink streaming SQL jobs, you need to select Enable Checkpoint on the job editing page before configuring this parameter.

      +
      +

      If this parameter is selected, you need to set Checkpoint Path for Flink Jar jobs.

      +

      Checkpoint Path: Select the checkpoint saving path. The checkpoint path must be the same as that you set in the application package. Note that the checkpoint path for each job must be unique. Otherwise, the checkpoint cannot be obtained.

      +
    +
    +
    +
+
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0462.html b/docs/dli/umn/dli_01_0462.html new file mode 100644 index 00000000..dd2063a3 --- /dev/null +++ b/docs/dli/umn/dli_01_0462.html @@ -0,0 +1,358 @@ + + +

Flink Job Details

+

After creating a job, you can view the job details to learn about the following information:

+ +

Viewing Job Details

This section describes how to view job details. After you create and save a job, you can click the job name to view job details, including SQL statements and parameter settings. For a Jar job, you can only view its parameter settings.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Click the name of the job to be viewed. The Job Detail tab is displayed.

    In the Job Details tab, you can view SQL statements, configured parameters.

    +
    The following uses a Flink SQL job as an example. +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 1 Description

    Parameter

    +

    Description

    +

    Type

    +

    Job type, for example, Flink SQL

    +

    Name

    +

    Flink job name

    +

    Description

    +

    Description of a Flink job

    +

    Status

    +

    Running status of a job

    +

    Running Mode

    +

    If your job runs on a shared queue, this parameter is Shared.

    +

    If your job runs on a custom queue with dedicated resources, this parameter is Exclusive.

    +

    Queue

    +

    If the queue where the job runs is a shared queue, the shared queue is displayed.

    +

    If the queue where the job runs is a custom queue with dedicated resources, the queue name is displayed.

    +

    UDF Jar

    +

    This parameter is displayed when a non-shared queue is selected for the job and UDF Jar is configured.

    +

    Runtime Configuration

    +

    Displayed when a user-defined parameter is added to a job

    +

    CUs

    +

    Number of CUs configured for a job

    +

    Job Manager CUs

    +

    Number of job manager CUs configured for a job.

    +

    Parallelism

    +

    Number of jobs that can be concurrently executed by a Flink job

    +

    CU(s) per TM

    +

    Number of CUs occupied by each Task Manager configured for a job

    +

    Slot(s) per TM

    +

    Number of Task Manager slots configured for a job

    +

    OBS Bucket

    +

    OBS bucket name. After Enable Checkpointing and Save Job Log are enabled, checkpoints and job logs are saved in this bucket.

    +

    Save Job Log

    +

    Whether the job running logs are saved to OBS

    +

    Alarm Generation upon Job Exception

    +

    Whether job exceptions are reported

    +

    SMN Topic

    +

    Name of the SMN topic. This parameter is displayed when Alarm Generation upon Job Exception is enabled.

    +

    Auto Restart upon Exception

    +

    Whether automatic restart is enabled.

    +

    Max. Retry Attempts

    +

    Maximum number of retry times upon an exception. Unlimited means the number is not limited.

    +

    Restore Job from Checkpoint

    +

    Whether the job can be restored from a checkpoint

    +

    ID

    +

    Job ID

    +

    Savepoint

    +

    OBS path of the savepoint

    +

    Enable Checkpointing

    +

    Whether checkpointing is enabled

    +

    Checkpoint Interval

    +

    Interval between storing intermediate job running results to OBS. The unit is second.

    +

    Checkpoint Mode

    +

    Checkpoint mode. Available values are as follows:

    +
    • At least once: Events are processed at least once.
    • Exactly once: Events are processed only once.
    +

    Idle State Retention Time

    +

    Defines for how long the state of a key is retained without being updated before it is removed in GroupBy or Window.

    +

    Dirty Data Policy

    +

    Policy for processing dirty data. The value is displayed only when there is a dirty data policy. Available values are as follows:

    +

    Ignore

    +

    Trigger a job exception

    +

    Save

    +

    Dirty Data Dump Address

    +

    OBS path for storing dirty data when Dirty Data Policy is set to Save.

    +

    Created

    +

    Time when a job is created

    +

    Updated

    +

    Time when a job was last updated

    +
    +
    +
    +

+
+

Checking the Job Monitoring Information

You can use Cloud Eye to view details about job data input and output.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Click the name of the job you want. The job details are displayed.

    Click Job Monitoring in the upper right corner of the page to switch to the Cloud Eye console.

    +

    The following table describes monitoring metrics related to Flink jobs.

    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 2 Monitoring metrics related to Flink jobs

    Name

    +

    Description

    +

    Flink Job Data Read Rate

    +

    Displays the data input rate of a Flink job for monitoring and debugging. Unit: record/s.

    +

    Flink Job Data Write Rate

    +

    Displays the data output rate of a Flink job for monitoring and debugging. Unit: record/s.

    +

    Flink Job Total Data Read

    +

    Displays the total number of data inputs of a Flink job for monitoring and debugging. Unit: records

    +

    Flink Job Total Data Write

    +

    Displays the total number of output data records of a Flink job for monitoring and debugging. Unit: records

    +

    Flink Job Byte Read Rate

    +

    Displays the number of input bytes per second of a Flink job. Unit: byte/s

    +

    Flink Job Byte Write Rate

    +

    Displays the number of output bytes per second of a Flink job. Unit: byte/s

    +

    Flink Job Total Read Byte

    +

    Displays the total number of input bytes of a Flink job. Unit: byte

    +

    Flink Job Total Write Byte

    +

    Displays the total number of output bytes of a Flink job. Unit: byte

    +

    Flink Job CPU Usage

    +

    Displays the CPU usage of Flink jobs. Unit: %

    +

    Flink Job Memory Usage

    +

    Displays the memory usage of Flink jobs. Unit: %

    +

    Flink Job Max Operator Latency

    +

    Displays the maximum operator delay of a Flink job. The unit is ms.

    +

    Flink Job Maximum Operator Backpressure

    +

    Displays the maximum operator backpressure value of a Flink job. A larger value indicates severer backpressure.

    +

    0: OK

    +

    50: low

    +

    100: high

    +
    +
    +

+
+

Viewing the Task List of a Job

You can view details about each task running on a job, including the task start time, number of received and transmitted bytes, and running duration.

+

If the value is 0, no data is received from the data source.

+
+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Click the name of the job you want. The job details are displayed.
  3. On Task List and view the node information about the task.

    View the operator task list. The following table describes the task parameters. +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 3 Parameter description

    Parameter

    +

    Description

    +

    Name

    +

    Name of an operator.

    +

    Duration

    +

    Running duration of an operator.

    +

    Max Concurrent Jobs

    +

    Number of parallel tasks in an operator.

    +

    Task

    +

    Operator tasks are categorized as follows:

    +
    • The digit in red indicates the number of failed tasks.
    • The digit in light gray indicates the number of canceled tasks.
    • The digit in yellow indicates the number of tasks that are being canceled.
    • The digit in green indicates the number of finished tasks.
    • The digit in blue indicates the number of running tasks.
    • The digit in sky blue indicates the number of tasks that are being deployed.
    • The digit in dark gray indicates the number of tasks in a queue.
    +

    Status

    +

    Status of an operator task.

    +

    Back Pressure Status

    +

    Working load status of an operator. Available options are as follows:

    +
    • OK: indicates that the operator is in normal working load.
    • LOW: indicates that the operator is in slightly high working load. DLI processes data quickly.
    • HIGH: indicates that the operator is in high working load. The data input speed at the source end is slow.
    +

    Delay

    +

    Duration from the time when source data starts being processed to the time when data reaches the current operator. The unit is millisecond.

    +

    Sent Records

    +

    Number of data records sent by an operator.

    +

    Sent Bytes

    +

    Number of bytes sent by an operator.

    +

    Received Bytes

    +

    Number of bytes received by an operator.

    +

    Received Records

    +

    Number of data records received by an operator.

    +

    Started

    +

    Time when an operator starts running.

    +

    Ended

    +

    Time when an operator stops running.

    +
    +
    +
    +

+
+

Viewing the Job Execution Plan

You can view the execution plan to understand the operator stream information about the running job.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Click the name of the job you want. The job details are displayed.
  3. Click the Execution Plan tab to view the operator flow direction.

    Click a node. The corresponding information is displayed on the right of the page.
    • Scroll the mouse wheel to zoom in or out.
    • The stream diagram displays the operator stream information about the running job in real time.
    +
    +

+
+

Viewing Job Submission Logs

You can view the submission logs to locate the fault.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Click the name of the job you want. The job details are displayed.
  3. In the Commit Logs tab, view the information about the job submission process.
+
+

Viewing Job Running Logs

You can view the run logs to locate the faults occurring during job running.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. Click the name of the job you want. The job details are displayed.
  3. On the Run Log tab page, you can view the Job Manager and Task Manager information of the running job.

    Information about JobManager and TaskManager is updated every minute. Only run logs of the last minute are displayed by default.

    +

    If you select an OBS bucket for saving job logs during the job configuration, you can switch to the OBS bucket and download log files to view more historical logs.

    +

    If the job is not running, information on the Task Manager page cannot be viewed.

    +

+
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0464.html b/docs/dli/umn/dli_01_0464.html new file mode 100644 index 00000000..c02d38dd --- /dev/null +++ b/docs/dli/umn/dli_01_0464.html @@ -0,0 +1,171 @@ + + +

Flink Template Management

+

Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for editing SQL statements. You can also customize a job template based on your habits and methods so that you can directly invoke or modify the template in later jobs.

+

Flink template management provides the following functions:

+ +

Flink SQL Sample Template

The template list displays existing sample templates for Flink SQL jobs. Table 1 describes the parameters in the template list.

+ +
+ + + + + + + + + + + + + +
Table 1 Parameters in the Flink SQL sample template list

Parameter

+

Description

+

Name

+

Name of a template, which has 1 to 64 characters and only contains letters, digits, hyphens (-), and underlines (_).

+

Description

+

Description of a template. It contains 0 to 512 characters.

+

Operation

+

Create Job: Create a job directly by using the template. After a job is created, the system switches to the Edit page under Job Management.

+
+
+

The existing sample templates apply to the following scenarios:

+ +
+

Custom Templates

The custom template list displays all Jar job templates. Table 1 describes parameters in the custom template list.

+ +
+ + + + + + + + + + + + + + + + + + + +
Table 2 Parameters in the custom template list

Parameter

+

Description

+

Name

+

Name of a template, which has 1 to 64 characters and only contains letters, digits, hyphens (-), and underlines (_).

+

Description

+

Description of a template. It contains 0 to 512 characters.

+

Created

+

Time when a template is created.

+

Updated

+

Latest time when a template is modified.

+

Operation

+
  • Edit: Modify a template that has been created.
  • Create Job: Create a job directly by using the template. After a job is created, the system switches to the Edit page under Job Management.
  • More:
    • Delete: Delete a created template.
    +
+
+
+
+

Creating a Template

You can create a template using any of the following methods:

+ + +
+

Creating a Job Based on a Template

You can create jobs based on sample templates or custom templates.

+
  1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates.
  2. In the sample template list, click Create Job in the Operation column of the target template. For details about how to create a job, see Creating a Flink SQL Job and Creating a Flink Jar Job.
+
+

Modifying a Template

After creating a custom template, you can modify it as required. The sample template cannot be modified, but you can view the template details.

+
  1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
  2. In the row where the template you want to modify is located in the custom template list, click Edit in the Operation column to enter the Edit page.
  3. In the SQL statement editing area, modify the SQL statements as required.
  4. Set Saving Mode to Save Here.
  5. Click Save in the upper right corner to save the modification.
+
+

Deleting a Template

You can delete a custom template as required. The sample templates cannot be deleted. Deleted templates cannot be restored. Exercise caution when performing this operation.

+
  1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
  2. In the custom template list, select the templates you want to delete and click Delete in the upper left of the custom template list.

    Alternatively, you can delete a template by performing the following operations: In the custom template list, locate the row where the template you want to delete resides, and click More > Delete in the Operation column.

    +
  3. In the displayed dialog box, click Yes.
+
+
+
+
+
Parent topic: Job Templates
+
+
+ diff --git a/docs/dli/umn/dli_01_0465.html b/docs/dli/umn/dli_01_0465.html new file mode 100644 index 00000000..f5d625a1 --- /dev/null +++ b/docs/dli/umn/dli_01_0465.html @@ -0,0 +1,17 @@ + + +

Spark Job Management

+
+
+ + +
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0475.html b/docs/dli/umn/dli_01_0475.html new file mode 100644 index 00000000..30bd8bb2 --- /dev/null +++ b/docs/dli/umn/dli_01_0475.html @@ -0,0 +1,68 @@ + + +

DLI Request Conditions

+

Request conditions are useful in determining when a custom policy takes effect. A request condition consists of a condition key and operator. Condition keys are either global or service-level and are used in the Condition element of a policy statement. Global condition keys (starting with g:) are available for operations of all services, while service-level condition keys (starting with a service name such as dli) are available only for operations of a specific service. An operator is used together with a condition key to form a complete condition statement.

+

IAM provides a set of DLI predefined condition keys. The following table lists the predefined condition keys of DLI.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 DLI request conditions

Condition Key

+

Operator

+

Description

+

g:CurrentTime

+

Date and time

+

Time when an authentication request is received

+
NOTE:

The time is expressed in the format defined by ISO 8601, for example, 2012-11-11T23:59:59Z.

+
+

g:MFAPresent

+

Boolean

+

Whether multi-factor authentication is used during user login

+

g:UserId

+

String

+

ID of the current login user

+

g:UserName

+

String

+

Current login user

+

g:ProjectName

+

String

+

Project that you have logged in to

+

g:DomainName

+

String

+

Domain that you have logged in to

+
+
+
+
+
+
Parent topic: Permissions Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0476.html b/docs/dli/umn/dli_01_0476.html new file mode 100644 index 00000000..6965b13d --- /dev/null +++ b/docs/dli/umn/dli_01_0476.html @@ -0,0 +1,51 @@ + + +

Global Variables

+

Scenario

Global variables can be used to simplify complex parameters. For example, long and difficult variables can be replaced to improve the readability of SQL statements.

+
+

Creating Variables

  1. In the navigation pane of the DLI console, choose Global Configuration > Global Variables.
  2. On the Global Variables page, click Create in the upper right corner to create a global variable. +
    + + + + + + + + + + + + + +
    Table 1 Parameters description

    Parameter

    +

    Description

    +

    Variable

    +

    Name of the created global variable.

    +

    Sensitive

    +

    If the value is sensitive information, such as passwords, set this parameter to Yes. Otherwise, set it to No.

    +

    Value

    +

    Global variable value.

    +
    +
    +
  3. After creating a global variable, use {{xxxx}} in the SQL statement to replace the parameter value set as the global variable. xxxx indicates the variable name. For example, if you set global variable abc to represent the table name, replace the actual table name with {{abc}} in the table creation statement.
    create table {{table_name}} (String1 String, int4 int, varchar1 varchar(10))
+  partitioned by (int1 int,int2 int,int3 int)
    +
    • Only the user who creates a global variable can use the variable.
    • Do not use global variables in OPTIONS of the table creation statements.
    +
    +
+
+

Modifying Variables

On the Global Variables page, click Modify in the Operation column of a variable to modify the variable value.

+

Only the user who creates a global variable can modify the variable.

+
+
+

Deleting Variables

On the Global Variables page, click Delete in the Operation column of a variable to delete the variable value.

+
  • Only the user who creates a global variable can delete the variable.
  • After a variable is deleted, the variable cannot be used in SQL statements.
+
+
+
+
+
+
Parent topic: Global Configuration
+
+
+ diff --git a/docs/dli/umn/dli_01_0477.html b/docs/dli/umn/dli_01_0477.html new file mode 100644 index 00000000..5e2d31f2 --- /dev/null +++ b/docs/dli/umn/dli_01_0477.html @@ -0,0 +1,72 @@ + + +

Managing Permissions on Packages and Package Groups

+

Scenarios

+
+

On the Package Management page, click Manage Permissions in the Operation column of the target package. On the displayed User Permission Info page, you can grant permissions for the package group or package, set and revoke user permissions.

+
  • If you select a group when creating a package, you can manage permissions of the corresponding program package group.
  • If you select No grouping when creating a package, you can manage permissions of the corresponding package.
+
+

Granting Permissions on Package Groups/Packages

Click Grant Permission in the upper right corner of the page.

+ +
+

Setting Permissions on Package Groups and Packages

Click Set Permission in the Operation column of the sub-user to modify the permission of the user. Table 1 and Table 2 list the detailed permission descriptions.

+

If the Set Permission button is gray, you do not have the permission to modify the package group or package. You can apply to the administrator, group owner, or other users who have the permissions on granting and revoking permissions of package groups or packages.

+
+

Revoking Permissions on Package Groups and Packages

Click Revoke Permission in the Operation column of a sub-user to revoke the user's permissions. After the operation, the sub-user does not have any permission on the package group or package.

+
+

Permissions Description

+
+
+
+
+
Parent topic: Package Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0478.html b/docs/dli/umn/dli_01_0478.html new file mode 100644 index 00000000..4fa1f05e --- /dev/null +++ b/docs/dli/umn/dli_01_0478.html @@ -0,0 +1,44 @@ + + +

Modifying the Owner

+

To change the owner of a package, click More > Modify Owner in the Operation column of a package on the Package Management page.

+ + +
+ + + + + + + + + + + + + + + + +
Table 1 Description

Parameter

+

Description

+

Group Name

+
  • If you select a group when creating a package, the name of the group is displayed.
  • If no group is selected when creating a package, this parameter is not displayed.
+

Name

+

Name of a package.

+

Select Type

+
  • If you select a group when creating a package, you can change the owner of the group or package.
  • If no group is selected when creating a package, this parameter is not displayed.
+

Username

+

Name of the package owner.

+
NOTE:

The username is the name of an existing IAM user.

+
+
+
+
+
+
+
Parent topic: Package Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0479.html b/docs/dli/umn/dli_01_0479.html new file mode 100644 index 00000000..0e381439 --- /dev/null +++ b/docs/dli/umn/dli_01_0479.html @@ -0,0 +1,55 @@ + + +

Managing Flink Job Permissions

+

Scenarios

+
+

Flink Job Permission Operations

  1. On the left of the DLI management console, choose Job Management > Flink Jobs.
  2. Select the job to be configured and choose More > Permissions in the Operation column. The User Permissions area displays the list of users who have permissions on the job.

    +

    You can assign queue permissions to new users, modify permissions for users who have some permissions of a queue, and revoke all permissions of a user on a queue.

    +
    • Assign permissions to a new user.
      A new user does not have permissions on the job.
      1. Click Grant Permission on the right of User Permissions page. The Grant Permission dialog box is displayed.
      2. Specify Username and select corresponding permissions.
      3. Click OK.

        Table 1 describes the related parameters.

        + +
        + + + + + + + + + + +
        Table 1 Permission parameters

        Parameter

        +

        Description

        +

        Username

        +

        Name of the user you want to grant permissions to.

        +
        NOTE:

        The username is the name of an existing IAM user. In addition, the user can perform authorization operations only after logging in to the platform.

        +
        +

        Permissions to be granted to the user

        +
        • View Job Details: This permission allows you to view the job details.
        • Modify Job: This permission allows you to modify the job.
        • Delete Job: This permission allows you to delete the job.
        • Start Job: This permission allows you to start the job.
        • Stop Job: This permission allows you to stop the job.
        • Export Job: This permission allows you to export the job.
        • Grant Permission: This permission allows you to grant job permissions to other users.
        • Revoke Permission: This permission allows you to revoke the job permissions that other users have but cannot revoke the job creator's permissions.
        • View Other User's Permissions: This permission allows you to view the job permissions of other users.
        +
        +
        +
      +
      +
    • To assign or revoke permissions of a user who has some permissions on the job, perform the following steps:
      1. In the list under User Permissions for a job, select the user whose permissions need to be modified and click Set Permission in the Operation column.
      2. In the displayed Set Permission dialog box, modify the permissions of the current user. Table 1 lists the detailed permission descriptions.

        If all options under Set Permission are gray, you are not allowed to change permissions on this job. You can apply to the administrator, job creator, or other authorized users for job permission granting and revoking.

        +
      3. Click OK.
      +
    • To revoke all permissions of a user on a job, perform the following steps:

      In the list under User Permissions for a job, locate the user whose permissions need to be revoked, click Revoke Permission in the Operation column, and click Yes. After this operation, the user does not have any permission on the job.

      +
    +
+
+

Flink Job Permissions

+
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0480.html b/docs/dli/umn/dli_01_0480.html new file mode 100644 index 00000000..e4a57f12 --- /dev/null +++ b/docs/dli/umn/dli_01_0480.html @@ -0,0 +1,42 @@ + + +

Managing Datasource Connection Permissions

+

Scenarios

+

On the Datasource Authentication tab page, click Manage Permissions in the Operation column of the row that contains the authentication to be modified. On the User Permission Info page that is displayed, you can grant, set, and revoke permissions of the datasource connection.

+
+

Granting Permissions on Datasource Connections

Log in to the DLI management console, choose Datasource Connections. Click the Datasource Authentication tab, locate the target authentication certificate, click Permission Management in the Operation column. Click Grant Permission in the upper right corner of the page. +
+ + + + + + + + + + +
Table 1 Permission granting parameters

Parameter

+

Description

+

Username

+

Name of the authorized IAM user.

+
NOTE:

The username is the name of an existing IAM user.

+
+

Select the permissions to be granted to the user

+
  • Access: This permission allows you to access the datasource connection.
  • Update: This permission allows you to update the datasource connection.
  • Delete: This permission allows you to delete the datasource connection.
  • Grant Permission: This permission allows you to grant the datasource connection permission to other users.
  • Grant Permission: This permission allows you to revoke the datasource connection permission to other users. However, you cannot revoke the permissions of the datasource connection owner.
  • View Other User's Permissions: This permission allows you to view the datasource connection permissions of other users.
+
+
+
+
+

Setting Permissions

Click Set Permission in the Operation column of the sub-user to modify the permission of the user. Table 1 lists the detailed permission descriptions.

+

If all options under Set Permission are gray, you are not allowed to change permissions on this datasource connection. You can apply for the granting and revoking permissions from administrators, group owners, and other users who have the permission to grant permissions.

+
+

Revoking Permissions

Click Revoke Permission in the Operation column of a sub-user to revoke the user's permissions. After this operation, the sub-user does not have any permission of the datasource connection.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_01_0485.html b/docs/dli/umn/dli_01_0485.html new file mode 100644 index 00000000..82ca4020 --- /dev/null +++ b/docs/dli/umn/dli_01_0485.html @@ -0,0 +1,17 @@ + + + +

Global Configuration

+ +

+
+ +
+ +
+ diff --git a/docs/dli/umn/dli_01_0486.html b/docs/dli/umn/dli_01_0486.html new file mode 100644 index 00000000..14c56b32 --- /dev/null +++ b/docs/dli/umn/dli_01_0486.html @@ -0,0 +1,64 @@ + + +

Service Authorization

+

Prerequisites

Only the tenant account or a subaccount of user group admin can authorize access.

+
+

Procedure

After entering the DLI management console, you are advised to set agency permissions to ensure that DLI can be used properly.

+

If you need to adjust the agency permissions, modify them on the Service Authorization page. For details about the required agency permissions, see Table 1.

+
  1. Select required agency permissions and click Update Authorization. Only the tenant account or a subaccount of user group admin can authorize access. If the message "Agency permissions updated" is displayed, the update is successful.
  2. Once service authorization has succeeded, an agency named dli_admin_agency on IAM will be created. Go to the agency list to view the details. Do not delete dli_admin_agency.

    +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 DLI agency permissions

Permission

+

Details

+

Remarks

+

Tenant Administrator (global service)

+

Tenant Administrator permissions are required to access data from OBS to execute Flink jobs on DLI, for example, obtaining OBS/DWS data sources, log dump (including bucket authorization), checkpointing enabling, and job import and export.

+

Due to cloud service cache differences, permission setting operations require about 60 minutes to take effect.

+

DIS Administrator

+

DIS Administrator permissions are required to use DIS data as the data source of DLI Flink jobs.

+

Due to cloud service cache differences, permission setting operations require about 30 minutes to take effect.

+

VPC Administrator

+

VPC Administrator permissions are required to use the VPC, subnet, route, VPC peering connection, and port for DLI datasource connections.

+

Due to cloud service cache differences, permission setting operations require about 3 minutes to take effect.

+

SMN Administrator

+

To receive notifications when a DLI job fails, SMN Administrator permissions are required.

+

Due to cloud service cache differences, permission setting operations require about 3 minutes to take effect.

+

Tenant Administrator (project-level)

+

Tenant Administrator (Project-level) permission is needed if you use services that can run only with this permission.

+

Due to cloud service cache differences, permission setting operations require about 3 minutes to take effect.

+
+
+
+
+
+
+
Parent topic: Global Configuration
+
+
+ diff --git a/docs/dli/umn/dli_01_0487.html b/docs/dli/umn/dli_01_0487.html new file mode 100644 index 00000000..679943b9 --- /dev/null +++ b/docs/dli/umn/dli_01_0487.html @@ -0,0 +1,26 @@ + + +

Elastic Scaling

+

Prerequisites

Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.

+
+

Precautions

+
+

Scaling Out

If the current queue specifications do not meet service requirements, you can add the number of CUs to scale out the queue.

+

Scale-out is time-consuming. After you perform scale-out on the Elastic Scaling page of DLI, wait for about 10 minutes. The duration is related to the CU amount to add. After a period of time, refresh the Queue Management page and check whether values of Specifications and Actual CUs are the same to determine whether the scale-out is successful. Alternatively, on the Job Management page, check the status of the SCALE_QUEUE SQL job. If the job status is Scaling, the queue is being scaled out.

+
+

The procedure is as follows:

+
  1. On the left of the DLI management console, click Resources > Queue Management.
  2. Select the queue to be scaled out and choose More > Elastic Scaling in the Operation column.
  3. On the displayed page, select Scale-out for Operation and set the scale-out amount.
  4. Click .
+
+

Scaling In

If the current queue specifications are too much for your computing service, you can reduce the number of CUs to scale in the queue.

+
  • Scale-in is time-consuming. After you perform scale-in on the Elastic Scaling page of DLI, wait for about 10 minutes. The duration is related to the CU amount to reduce. After a period of time, refresh the Queue Management page and check whether values of Specifications and Actual CUs are the same to determine whether the scale-in is successful. Alternatively, on the Job Management page, check the status of the SCALE_QUEUE SQL job. If the job status is Scaling, the queue is being scaled in.
  • The system may not fully scale in the queue to the target size. If the current queue is in use or the service volume of the queue is large, the scale-in may fail or only partial specifications may be reduced.
  • By default, the minimum number of CUs is 16. That is, when the queue specifications are 16 CUs, you cannot scale in the queue.
+
+

The procedure is as follows:

+
  1. On the left of the DLI management console, click Resources > Queue Management.
  2. Select the queue to be scaled out and choose More > Elastic Scaling in the Operation column.
  3. On the displayed page, select Scale-in for Operation and set the scale-in amount.
  4. Click .
+
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0488.html b/docs/dli/umn/dli_01_0488.html new file mode 100644 index 00000000..b182f088 --- /dev/null +++ b/docs/dli/umn/dli_01_0488.html @@ -0,0 +1,80 @@ + + +

Scheduling CU Changes

+

Scenario

When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. If the purchased queue specifications are small, resources may be insufficient during peak hours. If the queue specifications are large, resources may be wasted.

+

DLI provides scheduled tasks for elastic scale-in and -out in the preceding scenario. You can set different queue sizes (CUs) at different time or in different periods based on your service period or usage and the existing queue specifications to meet your service requirements and reduce costs.

+
+

Precautions

+
+

Creating Periodic Task

+

The procedure is as follows:

+
  1. On the left of the DLI management console, click Resources > Queue Management.
  2. Locate the queue for which you want to schedule a periodic task for elastic scaling, and choose More > Schedule CU Changes in the Operation column.
  3. On the displayed page, click Create Periodic Task in the upper right corner.
  4. On the Create Periodic Task page, set the required parameters. Click OK. +
    + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 1 Parameter description

    Parameter

    +

    Description

    +

    Task Name

    +

    Enter the name of the periodic task.

    +
    • The task name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_) or be left unspecified.
    • The name can contain a maximum of 128 characters.
    +

    Enable Task

    +

    Whether to enable periodic elastic scaling. The task is enabled by default. If disabled, the task will not be triggered on time.

    +

    Validity Period

    +

    Set the time segment for executing the periodic task, including Date and Time. If there is no time segment restriction, leave this parameter empty, indicating that the task takes effect permanently. If you need to specify the time segment for the task to take effect, set this parameter based on the service requirements.

    +
    NOTE:
    • The start time of the Validity Period must be later than the current system time.
    • If only scale-out is configured, the system does not automatically scale in after the Validity Period expires. You need to manually modify or configure a periodic scale-in task. Similarly, if only scale-in is configured, you need to manually modify or configure a periodic scale-out task. That is, a scheduled scaling task is executed at a time.
    • If both scale-out and scale-in are configured, the system scales in or out resources based on the configured queue specifications within the validity period. After the validity period expires, the system retains the last configured queue specifications.
    +
    +

    Actual CUs

    +

    Queue specifications before scale-in or scale-out.

    +

    Final CUs

    +

    Specifications after the queue is scaled in or out.

    +
    NOTE:
    • By default, the maximum specifications of a queue are 512 CUs.
    • The minimum queue specifications for scheduled scaling are 64 CUs. That is, only when Actual CUs are more than 64 CUs, the scheduled scaling can be performed.
    • The value of Actual CUs must be a multiple of 16.
    +
    +

    Repeat

    +
    Time when scheduled scale-out or scale-in is repeat. Scheduled tasks can be scheduled by week in Repeat.
    • By default, this parameter is not configured, indicating that the task is executed only once at the time specified by Executed.
    • If you select all, the plan is executed every day.
    • If you select some options of Repeat, the plan is executed once a week at all specified days.
    +
    +
    NOTE:
    • You do not need to set this parameter if you only need to perform scale-in or scale-out once.
    • If you have set scaling, you can set Repeat as required. You can also set the repeat period together with the validity period.
    +
    +

    Executed

    +

    Time when scheduled scale-out or scale-in is performed

    +
    • When each scheduled task starts, the actual start time of the specification change has a deviation of 5 minutes. It is recommended that the task start time be at least 20 minutes earlier than the time when the queue is actually used.
    • The interval between two scheduled tasks must be at least 2 hours.
    +
    +
    +

    After a periodic task is created, you can view the specification change of the current queue and the latest execution time on the page for scheduling CU changes.

    +

    Alternatively, on the Queue Management page, check whether the Specifications change to determine whether the scaling is successful.

    +

    You can also go to the Job Management page and check the status of the SCALE_QUEUE job. If the job status is Scaling, the queue is being scaled in or out.

    +
+
+

Modifying a Scheduled Task

If the configured scheduled task does not meet service requirements, you can click Modify in the Operation column of the task on the Schedule CU Changes page to modify the task configuration.

+
+

Deleting a Scheduled Task

If you do not need a scheduled task for queue scaling, click Delete in the Operation column of the task on the Schedule CU Changes page. In the displayed Delete Periodic Task dialog box, click Yes.

+
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0489.html b/docs/dli/umn/dli_01_0489.html new file mode 100644 index 00000000..52f1f5c4 --- /dev/null +++ b/docs/dli/umn/dli_01_0489.html @@ -0,0 +1,14 @@ + + +

Testing Address Connectivity

+

It can be used to test the connectivity between the DLI queue and the peer IP address specified by the user in common scenarios, or the connectivity between the DLI queue and the peer IP address bound to the datasource connection in cross-source connection scenarios. The operation is as follows:

+
  1. On the Queue Management page, locate the row that contains the target queue and choose More > Test Address Connectivity in the Operation column.
  2. On the Test Address Connectivity page, enter the address to be tested. The domain name and IP address are supported, and the port number can be specified.
  3. Click Test.

    If the test address is reachable, a message is displayed on the page, indicating that the address is reachable.

    +

    If the test address is unreachable, the system displays a message indicating that the address is unreachable. Check the network configurations and try again. Network configurations include the VPC peering and the datasource connection. Check whether they have been activated.

    +
+
+
+
+
Parent topic: Queue Management
+
+
+ diff --git a/docs/dli/umn/dli_01_0498.html b/docs/dli/umn/dli_01_0498.html new file mode 100644 index 00000000..3d9bb2e2 --- /dev/null +++ b/docs/dli/umn/dli_01_0498.html @@ -0,0 +1,166 @@ + + +

(Recommended) Creating a Flink OpenSource SQL Job

+

This section describes how to create a Flink OpenSource SQL job. DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the community. In addition, Redis, DWS(GaussDB), and DIS data source types are added based on the community connector. For details about the syntax and restrictions of Flink SQL DDL, DML, and functions, see Table API & SQL.

+

Prerequisites

+
+

Creating a Flink OpenSource SQL Job

  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. In the upper right corner of the Flink Jobs page, click Create Job.

    +

  3. Specify job parameters.

    +

    + + + + + + + + + + + + + + + + +
    Table 1 Job configuration information

    Parameter

    +

    Description

    +

    Type

    +

    Set Type to Flink OpenSource SQL. You will need to start jobs by compiling SQL statements.

    +

    Name

    +

    Name of a job. Enter 1 to 57 characters. Only letters, digits, hyphens (-), and underscores (_) are allowed.

    +
    NOTE:

    The job name must be globally unique.

    +
    +

    Description

    +

    Description of a job. It can be up to 512 characters long.

    +

    Template Name

    +

    You can select a sample template or a custom job template. For details about templates, see Flink Template Management.

    +
    +
    +

  4. Click OK to enter the Edit page.
  5. Edit an OpenSource SQL job.

    Enter details SQL statements in the SQL statement edit area. For details about SQL statements, see the Data Lake Insight Flink OpenSource SQL Syntax Reference.

    +

  6. Click Check Semantics.

    • You can Start a job only after the semantic verification is successful.
    • If verification is successful, the message "The SQL semantic verification is complete. No error." will be displayed.
    • If verification fails, a red "X" mark will be displayed in front of each SQL statement that produced an error. You can move the cursor to the "X" mark to view error details and change the SQL statement as prompted.
    +

  7. Set job running parameters.

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 2 Running parameters

    Parameter

    +

    Description

    +

    Queue

    +

    A shared queue is selected by default. Select a CCE queue with dedicated resources and configure the following parameters:

    +

    UDF Jar: You can customize a UDF Jar file. Before selecting a JAR file to be inserted, upload the corresponding JAR package to the OBS bucket and choose Data Management > Package Management to create a package. For details, see Creating a Package.

    +

    In SQL, you can call a user-defined function that is inserted into a JAR file.

    +
    NOTE:

    During job creation, a sub-user can only select a queue that has been allocated to the user.

    +

    If the remaining capacity of the selected queue cannot meet the job requirements, the system automatically scales up the capacity and you will be billed based on the increased capacity. When a queue is idle, the system automatically scales in the queue.

    +
    +

    CUs

    +

    Sum of the number of compute units and job manager CUs of DLI. One CU equals one vCPU and 4 GB.

    +

    The value is the number of CUs required for job running and cannot exceed the number of CUs in the bound queue.

    +

    Job Manager CUs

    +

    Number of CUs of the management unit.

    +

    Parallelism

    +

    Number of Flink OpenSource SQL jobs that run at the same time

    +
    NOTE:

    This value cannot be greater than four times the compute units (number of CUs minus the number of job manager CUs).

    +
    +

    Task Manager Configuration

    +

    Whether to set Task Manager resource parameters.

    +

    If this option is selected, you need to set the following parameters:

    +
    • CU(s) per TM: Number of resources occupied by each Task Manager.
    • Slot(s) per TM: Number of slots contained in each Task Manager.
    +

    OBS Bucket

    +

    OBS bucket to store job logs and checkpoint information. If the selected OBS bucket is not authorized, click Authorize.

    +

    Save Job Log

    +

    Whether to save the job running logs to OBS. The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID.

    +
    CAUTION:

    You are advised to configure this parameter. Otherwise, no run log is generated after the job is executed. If the job fails, the run log cannot be obtained for fault locating.

    +
    +

    If this option is selected, you need to set the following parameters:

    +
    OBS Bucket: Select an OBS bucket to store user job logs. If the selected OBS bucket is not authorized, click Authorize.
    NOTE:

    If both Enable Checkpointing and Save Job Log are selected, you only need to authorize OBS once.

    +
    +
    +

    Alarm Generation upon Job Exception

    +

    Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email.

    +

    If this option is selected, you need to set the following parameters:

    +

    SMN Topic

    +

    Select a user-defined SMN topic. For details about how to customize SMN topics, see Creating a Topic in the Simple Message Notification User Guide.

    +

    Enable Checkpointing

    +

    Whether to enable job snapshots. If this function is enabled, jobs can be restored based on the checkpoints.

    +
    If this option is selected, you need to set the following parameters:
    • Checkpoint Interval (s) indicates the interval for creating checkpoints. The value ranges from 1 to 999999, and the default value is 30.
    • Checkpoint Mode can be set to either of the following values:
      • At least once: Events are processed at least once.
      • Exactly once: Events are processed only once.
      +
    +
    • OBS Bucket: Select an OBS bucket to store your checkpoints. If the selected OBS bucket is not authorized, click Authorize.
      The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
      NOTE:

      If both Enable Checkpointing and Save Job Log are selected, you only need to authorize OBS once.

      +
      +
      +
    +
    +

    Auto Restart upon Exception

    +

    Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

    +

    If this option is selected, you need to set the following parameters:

    +
    • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is Times/hour.
      • Unlimited: The number of retries is unlimited.
      • Limited: The number of retries is user-defined.
      +
    • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.
    +

    Idle State Retention Time

    +

    How long the state of a key is retained without being updated before it is removed in GroupBy or Window. The default value is 1 hour.

    +

    Dirty Data Policy

    +

    Policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

    +

    If you set this filed to Save, the Dirty Data Dump Address must be set. Click the address box to select the OBS path for storing dirty data.

    +
    +
    +

  8. (Optional) Set the runtime configuration as needed.
  9. Click Save.
  10. Click Start. On the displayed Start Flink Jobs page, confirm the job specifications, and click Start Now to start the job.

    After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, the job status will change from Submitting to Running. After the execution is complete, the message Completed is displayed.

    +

    If the job status is Submission failed or Running exception, the job submission failed or the job did not execute successfully. In this case, you can move the cursor over the status icon in the Status column of the job list to view the error details. You can click to copy these details. After handling the fault based on the provided information, resubmit the job.

    +

    Other buttons are as follows:

    +
    • Save As: Save the created job as a new job.
    • Static Stream Graph: Provide the static concurrency estimation function and stream graph display function.
    • Simplified Stream Graph: Display the data processing flow from the source to the sink.
    • Format: Format the SQL statements in the editing box.
    • Set as Template: Set the created SQL statements as a job template.
    • Theme Settings: Set the theme related parameters, including Font Size, Wrap, and Page Style.
    +
    +

+
+

Simplified Stream Graph

On the OpenSource SQL job editing page, click Simplified Stream Graph.

+
+

Static stream graph

On the OpenSource SQL job editing page, click Static Stream Graph.

+

The Static Stream Graph page also allows you to:

+ +
+
+
+
+
Parent topic: Flink Job Management
+
+
+ diff --git a/docs/dli/umn/dli_01_05110.html b/docs/dli/umn/dli_01_05110.html new file mode 100644 index 00000000..b36ca0f8 --- /dev/null +++ b/docs/dli/umn/dli_01_05110.html @@ -0,0 +1,19 @@ + + + +

Appendix

+ +

+
+ +
+ + +
+
Parent topic: Job Templates
+
+
+ diff --git a/docs/dli/umn/dli_01_05111.html b/docs/dli/umn/dli_01_05111.html new file mode 100644 index 00000000..9c3c64f4 --- /dev/null +++ b/docs/dli/umn/dli_01_05111.html @@ -0,0 +1,30 @@ + + +

TPC-H Sample Data in the SQL Template

+

TPC-H Sample Data

TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and industry to evaluate the performance of decision-making support technology. This business test has higher requirements on vendors, because it can comprehensively evaluate the overall business computing capability. With universal business significance, is widely used in analysis of bank credit, credit card, telecom operation, tax, as well as tobacco industry decision-making analysis.

+

The TPC-H benchmark test is developed from TPC-D (a standard specified by TPC in 1994 and used as the test benchmark for decision-making support systems). TPC-H implements a 3NF data warehouse that contains eight basic relationships, with a data volume range from 1 GB to 3 TB. The TPC-H benchmark test includes 22 queries (Q1 to Q22). The main evaluation indicator is the response time of each query (from submission to result return). The unit of the TPC-H benchmark test is the query number per hour (QphH@size). H indicates the average number of complex queries per hour. size indicates the size of database, which reflects the query processing capability of the system. TPC-H can evaluate key performance parameters that other tests cannot evaluate, because it is modeled based on the actual production and operation environment. In a word, the TPC-H standard by TPC meets the test requirements of data warehouse and motivate vendors and research institutes to stretch the limit of this technology.

+

In this example, DLI directly queries the TPC-H dataset on OBS. DLI has generated a standard TPC-H-2.18 dataset of 100 MB which is uploaded to the tpch folder on OBS. The read-only permission is granted to you to facilitate query operations.

+
+

TPC-H Test and Metrics

TPC-H test is divided into three sub-tests: data loading test, Power test, and Throughput test. Data loading indicates the process of setting up a test database, and the loading test is to test the data loading ability of DBMS. The first test is data loading test that tests data loading time, which is time-consuming. The second test is Power test, also called raw query. After data loading test is complete, the database is in the initial state without any other operation, especially the data in the buffer is not tested. Power test requires that the 22 queries be executed once in sequence and a pair of RF1 and RF2 operations be executed at the same time. The third test is Throughput test, the core and most complex test, more similar to the actual application environment. With multiple query statement groups and a pair of RF1 and RF2 update flows, Throughput test pose greater pressure on the SUT system than Power test does.

+
+

The basic data in the test is related to the execution time (the time of each data loading step, each query execution, and each update execution), based on which you can calculate the data loading time, Power@Size, Throughput@Size, qphH@Size and $/QphH@Size.

+

Power@Size is the result of the Power test, which is defined as the reciprocal of the geometric average value of the query time and change time. The formula is as follows:

+

+

Size indicates the data size. SF is the scaling factor of data scale. QI (i, 0) indicates the time of the ith query, in seconds. R (I j, 0) is the update time of RFj, in seconds.

+

Throughput@Size is the Throughput test result, which is defined as the reciprocal of the average value of all query execution time. The formula is as follows:

+

+

Service Scenario

You can use the built-in TPC-H test suite of DLI to perform interactive query without uploading data.

+
+

Advantages of DLI Built-in TPC-H

+
+

Precautions

When a sub-account uses the TPC-H test suite, the main account needs to grant the sub-account the OBS access permission and the permission to view the main account table. If the master account has not logged in to DLI, the sub-account needs to have the permissions to create databases and tables in addition to the preceding permissions.

+
+

Operation Description

For details, see SQL Template Management.

+
+
+
+
+
Parent topic: Appendix
+
+
+ diff --git a/docs/dli/umn/dli_01_0512.html b/docs/dli/umn/dli_01_0512.html new file mode 100644 index 00000000..85d2730c --- /dev/null +++ b/docs/dli/umn/dli_01_0512.html @@ -0,0 +1,29 @@ + + +

Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template

+

To facilitate SQL operations, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not need to write SQL statements. You can directly perform the SQL operations using the templates.

+

The current system provides various standard TPC-H query statement templates. You can select a template as needed. This example shows how to use a TPC-H template to develop and submit a Spark SQL job.

+

Step 1: Logging in to the Cloud

+

Step 2: Logging In to the DLI Management Console

+

Step 3: Execute the TPC-H Sample Template and View the Result

+

For details about the templates, see SQL Template Management.

+

Step 1: Logging in to the Cloud

  1. Open the DLI homepage.
  2. On the login page, enter the username and password, and click Log In.
+
+

Step 2: Logging In to the DLI Management Console

  1. .
  2. The DLI management console page is displayed.

    You need to perform authorization when accessing the DLI management console for the first time. For details, see Service Authorization.

    +
    +
+
+

Step 3: Execute the TPC-H Sample Template and View the Result

  1. On the DLI management console, choose Job Templates > SQL Templates, and click the Sample Templates tab. Locate the Q1_Price_summary_report_query template under tpchQuery, and click Execute in the Operation column. The SQL Editor page is displayed.

    +
  2. In the upper part of the editing window, set Engine to spark, Queues to default, and Databases to default, and click Execute.

    +
  3. View the query result in the View Result tab in the lower part of the SQL Editor page.

    +
+

This example uses the default queue and database preset in the system as an example. You can also run query statements on a self-created queue and database.

+

For details about how to create a queue, see Creating a Queue. For details about how to create a database, see Creating a Database.

+
+
+
+
+
Parent topic: Getting Started
+
+
+ diff --git a/docs/dli/umn/dli_01_0531.html b/docs/dli/umn/dli_01_0531.html new file mode 100644 index 00000000..9815fa5e --- /dev/null +++ b/docs/dli/umn/dli_01_0531.html @@ -0,0 +1,141 @@ + + +

Creating and Submitting a Flink OpenSource SQL Job

+

This section describes how to submit a Flink OpenSource SQL job on the DLI console for real-time computing. The general procedure is as follows:

+

Step 1: Log In to the Cloud Platform

+

Step 2: Prepare a Data Source Channel

+

Step 3: Prepare a Data Output Channel

+

Step 4: Create an OBS Bucket to Store Output Data

+

Step 5: Creating a Queue

+

Step 6: Creating an Enhanced Datasource Connection Between DLI and Kafka

+

Step 7: Creating an Enhanced Datasource Connection Between DLI and RDS

+

Step 8: Creating a Flink OpenSource SQL Job

+

You need to create a Flink OpenSource SQL job that has an input stream and an output stream. The input stream reads data from Kafka, and the output stream writes data into RDS.

+

Step 1: Log In to the Cloud Platform

To use DLI, you need to log in to the cloud platform.

+
+
  1. Open the DLI homepage.
  2. On the login page, enter the username and password, and click Log In.
+

Step 2: Prepare a Data Source Channel

DLI Flink jobs allow other services to function as data sources and data output channels. For details, see"Preparing Data".

+

In this example, the job name is JobSample, Kafka is the data source, and DIS is enabled to for data integration. For details, see "Buying an Instance" in the Distributed Message Service Kafka User Guide.

+
  1. Create the dependent Kafka resources.
    Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules.
    • For details about how to create a VPC and subnet, see "Creating a VPC and Subnet" in the Virtual Private Cloud User Guide. For details about how to create and use a subnet in an existing VPC, see "Create a subnet for the VPC" in the Virtual Private Cloud User Guide.
      • The created VPC and the Kafka instance you will create must be in the same region.
      • Retain the default settings unless otherwise specified.
      +
      +
    • For details about how to create a security group, see "Creating a Security Group" in the Virtual Private Cloud User Guide. For details about how to add rules to a security group, see "Creating a Subnet for the VPC" in the Virtual Private Cloud User Guide.
    +
    +

    For more information, see Managing Kafka Premium Instances > in the Distributed Message Service User Guide.

    +
  2. Create a DMS for Kafka Instance for job input streams.
    1. Log in to the DMS for Kafka console.
    2. Select a region in the upper left corner.
    3. On the DMS for Kafka page, click Buy Instance in the upper right corner and set related parameters. The required instance information is as follows:
      • Region: Select the region where DLI is located.
      • Project: Keep the default value.
      • AZ: Keep the default value.
      • Instance Name: kafka-dliflink
      • Enterprise Project: default
      • Version: Keep the default value.
      • CPU Architecture: Keep the default value.
      • Specifications: Select the specifications as needed.
      • Brokers: Keep the default value.
      • Storage Space: Keep the default value.
      • Capacity Threshold Policy: Keep the default value.
      • VPC and Subnet: Select the VPC and subnet created in 1.
      • Security Group: Select the security group created in 1.
      • Manager Username: Enter dliflink (used to log in to the instance management page).
      • Password: **** (The system cannot detect your password.)
      • Confirm Password: ****
      • More Settings: Do not configure this parameter.
      +
    4. Click Buy. The confirmation page is displayed.
    5. Click Submit.
    +
  3. Create a Kafka topic.
    1. Click the name of the created Kafka instance. The basic information page of the instance is displayed.
    2. Choose Topics in the navigation pane on the left. On the displayed page, click Create Topic. Configure the following parameters:
      • Topic Name: For this example, enter testkafkatopic.
      • Partitions: Set the value to 1.
      • Replicas: Set the value to 1.
      +

      Retain default values for other parameters.

      +
    +
+
+

Step 3: Prepare a Data Output Channel

To use RDS as the data output channel, create an RDS MySQL instance. For details, see Getting Started with RDS for MySQL.

+
  1. Log in to the RDS console.
  2. In the upper left corner of the management console, select the target region and project.
  3. Click Buy DB Instance in the upper right corner of the page and set related parameters. Retain the default values for other parameters.
    • Region: Select the region where DLI is located.
    • DB Instance Name: Enter rds-dliflink.
    • DB Engine: Select MySQL.
    • DB Engine Version: Select 8.0.
    • DB Instance Type: Select Primary/Standby.
    • Storage Type: Cloud SSD may be selected by default.
    • Primary AZ: Select a custom AZ.
    • Standby AZ: Select a custom AZ.
    • Time Zone: Keep the default value.
    • Instance Class: Select a class as needed and choose 2 vCPUs | 8 GB.
    • Storage Space (GB): Set it to 40.
    • VPC: Select the VPC and subnet created in 1.
    • Database Port: Enter 3306.
    • Security Group: Select the security group created in 1.
    • Administrator Password: **** (Keep the password secure. The system cannot retrieve your password.)
    • Confirm Password: ****
    • Parameter Template: Choose Default-MySQL-8.0.
    • Read Replica: Select Skip.
    +
  4. Click Next and confirm the specifications.
  5. Click Submit. The RDS DB instance is created.
  6. Log in to the MySQL database and create table orders in database flink.
    Log in to the MySQL instance, click the flink database. On the displayed page, click SQL Window. Enter the following table creation statement in the SQL editing pane to create a table.
    CREATE TABLE `flink`.`orders` (
+	`order_id` VARCHAR(32) NOT NULL,
+	`order_channel` VARCHAR(32) NULL,
+	`order_time` VARCHAR(32) NULL,
+	`pay_amount` DOUBLE UNSIGNED NOT NULL,
+	`real_pay` DOUBLE UNSIGNED NULL,
+	`pay_time` VARCHAR(32) NULL,
+	`user_id` VARCHAR(32) NULL,
+	`user_name` VARCHAR(32) NULL,
+	`area_id` VARCHAR(32) NULL,
+	PRIMARY KEY (`order_id`)
+)	ENGINE = InnoDB
+	DEFAULT CHARACTER SET = utf8mb4
+	COLLATE = utf8mb4_general_ci;
    +
    +
+
+

Step 4: Create an OBS Bucket to Store Output Data

In this example, you need to enable OBS for JobSample to provide DLI Flink jobs with the functions of checkpoint, saving job logs, and commissioning test data.

+

For details about how to create a bucket, see Creating a Bucket in the Object Storage Service Console Operation Guide.

+
  1. In the navigation pane on the OBS management console, choose Object Storage.
  2. In the upper right corner of the page, click Create Bucket and set bucket parameters.
    • Region: Select the region where DLI is located.
    • Bucket Name: Enter a bucket name. For this example, enter obstest.
    • Default Storage Class: Standard
    • Bucket Policy: Private
    • Default Encryption: Do not enable
    • Direct Reading: Do not enable
    • Enterprise Project: default
    • Tags: Leave it blank.
    +
  3. Click Create Now.
+
+

Step 5: Creating a Queue

Flink OpenSource SQL jobs cannot run on the default queue. You need to create a queue, for example, Flinktest. For details, see Creating a Queue.

+
  1. Log in to the DLI management console. On the Overview page, click Buy Queue in the upper right corner.

    If this is your first time to log in to the DLI management console, you need to be authorized to access OBS.

    +
  2. Configure the following parameters:
    • Name: Flinktest
    • Queue Usage: Select For general purpose and enable Dedicated Resource Mode.
    • Specifications: 16 CUs
    • Enterprise Project: default
    • Description: Leave it blank.
    • Advanced Settings: Custom
    • CIDR Block: Set a CIDR block that does not conflict with the Kafka instance's CIDR block.
    +
  3. Click Buy and confirm the configuration.
  4. Submit the request.

    It takes 10 to 15 minutes to bind the queue to a cluster after the queue is created.

    +
+
+

Step 6: Creating an Enhanced Datasource Connection Between DLI and Kafka

You need to create an enhanced datasource connection for the Flink OpenSource SQL job. For details, see "Creating an Enhanced Datasource Connection".

+
  • The CIDR block of the DLI queue bound with a datasource connection cannot overlap with the CIDR block of the data source.
  • Datasource connections cannot be created for the default queue.
  • To access a table across data sources, you need to use a queue bound to a datasource connection.
+
+
  1. Create a Kafka security group rule to allow access from the CIDR block of the DLI queue.

    1. On the Kafka management console, click an instance name on the DMS for Kafka page. Basic information of the Kafka instance is displayed.
    2. In the Connection pane, obtain the Instance Address (Private Network). In the Network pane, obtain the VPC and subnet of the instance.
    3. Click the security group name in the Network pane. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue.

      For example, if the CIDR block of the queue is 10.0.0.0/16, set Priority to 1, Action to Allow, Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.

      +
    +

  2. Create an enhanced datasource connection to Kafka.

    1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
    2. In the displayed dialog box, set the following parameters: For details, see the following section:
      • Connection Name: Name of the enhanced datasource connection For this example, enter dli_kafka.
      • Resource Pool: Select the name of the queue created in Step 5: Creating a Queue.
      • VPC: Select the VPC of the Kafka instance.
      • Subnet: Select the subnet of Kafka instance.
      • Set other parameters as you need.
      +

      Click OK. Click the name of the created datasource connection to view its status. You can perform subsequent steps only after the connection status changes to Active.

      +
    3. Choose Resources > Queue Management and locate the queue created in Step 5: Creating a Queue. In the Operation column, choose More > Test Address Connectivity.
    4. In the displayed dialog box, enter Kafka instance address (private network):port in the Address box and click Test to check whether the instance is reachable. Note that multiple addresses must be tested separately.
    +

+
+

Step 7: Creating an Enhanced Datasource Connection Between DLI and RDS

  1. Create an RDS security group rule to allow access from CIDR block of the DLI queue.

    If the RDS DB instance and Kafka instance are in the same security group of the same VPC, skip this step. Access from the DLI queue has been allowed in step 6-1.
    1. Go to the RDS console, click the name of the target RDS for MySQL DB instance on the Instances page. Basic information of the instance is displayed.
    2. In the Connection Information pane, obtain the floating IP address, database port, VPC, and subnet.
    3. Click the security group name. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue. For example, if the CIDR block of the queue is 10.0.0.0/16, set Priority to 1, Action to Allow, Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.
    +
    +

  2. Create an enhanced datasource connection to RDS.

    If the RDS DB instance and Kafka instance are in the same VPC and subnet, skip this step. The enhanced datasource connection created in step 6-2 has connected the subnet.

    +
    If the two instances are in different VPCs or subnets, perform the following steps to create an enhanced datasource connection:
    1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
    2. In the displayed dialog box, set the following parameters: For details, see the following section:
      • Connection Name: Name of the enhanced datasource connection For this example, enter dli_rds.
      • Resource Pool: Select the name of the queue created in Step 5: Creating a Queue.
      • VPC: Select the VPC of the RDS DB instance.
      • Subnet: Select the subnet of RDS DB instance.
      • Set other parameters as you need.
      +

      Click OK. Click the name of the created datasource connection to view its status. You can perform subsequent steps only after the connection status changes to Active.

      +
    3. Choose Resources > Queue Management and locate the queue created in Step 5: Creating a Queue. In the Operation column, choose More > Test Address Connectivity.
    4. In the displayed dialog box, enter floating IP address:database port of the RDS DB instance in the Address box and click Test to check whether the database is reachable.
    +
    +

+
+

Step 8: Creating a Flink OpenSource SQL Job

After the data source and data output channel are prepared, you can create a Flink OpenSource SQL job.

+
  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
  2. In the upper right corner of the Flink Jobs page, click Create Job. Set the following parameters:
    • Type: Flink OpenSource SQL
    • Name: JobSample
    • Description: Leave it blank.
    • Template Name: Do not select any template.
    +
  3. Click OK to enter the editing page.
  4. Set job running parameters. The mandatory parameters are as follows:
    • Queue: Flinktest
    • Flink Version: Select 1.12.
    • Save Job Log: Enable this function.
    • OBS Bucket: Select an OBS bucket for storing job logs and grant access permissions of the OBS bucket as prompted.
    • Enable Checkpointing: Enable this function.
    +

    You do not need to set other parameters.

    +
  5. Click Save.
  6. Edit the Flink OpenSource SQL job.
    In the SQL statement editing area, enter query and analysis statements as you need. The example statements are as follows. Note that the values of the parameters in bold must be changed according to the comments.
    CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'testkafkatopic',// Topic to be written to Kafka. Log in to the Kafka console, click the name of the created Kafka instance, and view the topic name on the Topic Management page.
+  'properties.bootstrap.servers' = "192.168.0.237:9092,192.168.0.252:9092,192.168.0.137:9092", // Replace it with the internal network address and port number of Kafka.
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE jdbcSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'jdbc',
+  'url' = "jdbc:mysql://172.16.0.116:3306/rds-dliflink", //  testrdsdb indicates the name of the created RDS database. Replace the IP address and port number with those of the RDS for MySQL instance.
+  'table-name' = 'orders',
+  'username' = "xxxxx",  //  Username of the RDS for MySQL DB instance
+  'password'="xxxxx",  //  Password of the RDS for MySQL DB instance
+  'sink.buffer-flush.max-rows' = '1'
+);
+
+insert into jdbcSink select * from kafkaSource;
    +
    +
  7. Click Check Semantics.
  8. Click Start. On the displayed Start Flink Job page, confirm the job specifications and the price, and click Start Now to start the job.

    After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, Status of the job will change from Submitting to Running.

    +

    If Status of a job is Submission failed or Running exception, the job fails to be submitted or fails to run. In this case, you can move the cursor over the status icon to view the error details. You can click to copy these details. After handling the fault based on the provided information, resubmit the job.

    +
  9. Connect to the Kafka cluster and send the following test data to the Kafka topics:
    {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"} 
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
    +
  10. Run the following SQL statement in the MySQL database to view data in the table:
    select * from order;
    +
    The following is an example of the result (note that the following data is replicated from the MySQL database but not the data style in the MySQL database):
    202103241000000001,webShop,2021-03-24 10:00:00,100.0,100.0,2021-03-24 10:02:03,0001,Alice,330106
+202103241606060001,appShop,2021-03-24 16:06:06,200.0,180.0,2021-03-24 16:10:06,0001,Alice,330106
    +
    +
+
+
+
+
+
Parent topic: Getting Started
+
+
+ diff --git a/docs/dli/umn/dli_01_0538.html b/docs/dli/umn/dli_01_0538.html new file mode 100644 index 00000000..bc09d577 --- /dev/null +++ b/docs/dli/umn/dli_01_0538.html @@ -0,0 +1,25 @@ + + + +

Service Overview

+ +

+
+ +
+ +
+ diff --git a/docs/dli/umn/dli_01_0539.html b/docs/dli/umn/dli_01_0539.html new file mode 100644 index 00000000..f0e0fb48 --- /dev/null +++ b/docs/dli/umn/dli_01_0539.html @@ -0,0 +1,29 @@ + + + +

FAQs

+ +

+
+ +
+ +
+ diff --git a/docs/dli/umn/dli_03_0001.html b/docs/dli/umn/dli_03_0001.html new file mode 100644 index 00000000..ebbb3a33 --- /dev/null +++ b/docs/dli/umn/dli_03_0001.html @@ -0,0 +1,37 @@ + + +

Product Consultation

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0002.html b/docs/dli/umn/dli_03_0002.html new file mode 100644 index 00000000..841097a4 --- /dev/null +++ b/docs/dli/umn/dli_03_0002.html @@ -0,0 +1,11 @@ + + +

What Is DLI?

+

Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark, and Flink ecosystems. It frees you from managing any server. DLI supports standard SQL and is compatible with Spark and Flink SQL. It also supports multiple access modes, and is compatible with mainstream data formats. DLI supports SQL statements and Spark applications for heterogeneous data sources, including CloudTable, RDS, GaussDB(DWS), CSS, OBS, custom databases on ECSs, and offline databases.

+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0008.html b/docs/dli/umn/dli_03_0008.html new file mode 100644 index 00000000..65fb9c29 --- /dev/null +++ b/docs/dli/umn/dli_03_0008.html @@ -0,0 +1,11 @@ + + +

What Is Column Permission Granting of a DLI Partition Table?

+

You cannot perform permission-related operations on the partition column of a partition table. However, when you grant the permission of any non-partition column in a partition table to another user, the user gets the permission of the partition column by default. When the user views the permission of the partition table, the permission of the partition column will not be displayed.

+
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0009.html b/docs/dli/umn/dli_03_0009.html new file mode 100644 index 00000000..94c995e9 --- /dev/null +++ b/docs/dli/umn/dli_03_0009.html @@ -0,0 +1,11 @@ + + +

How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?

+

DLI supports only UTF-8-encoded texts. Ensure that data is encoded using UTF-8 during table creation and import.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0010.html b/docs/dli/umn/dli_03_0010.html new file mode 100644 index 00000000..f465fe3b --- /dev/null +++ b/docs/dli/umn/dli_03_0010.html @@ -0,0 +1,11 @@ + + +

How Can I Use DLI If Data Is Not Uploaded to OBS?

+

Currently, DLI supports analysis only on the data uploaded to the cloud. In scenarios where regular (for example, on a per day basis) one-off analysis on incremental data is conducted for business, you can do as follows: Anonymize data to be analyzed and store anonymized data on OBS temporarily. After analysis is complete, export the analysis report and delete the data temporarily stored on OBS.

+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0011.html b/docs/dli/umn/dli_03_0011.html new file mode 100644 index 00000000..463da933 --- /dev/null +++ b/docs/dli/umn/dli_03_0011.html @@ -0,0 +1,12 @@ + + +

How Can I Perform Query on Data Stored on Services Rather Than DLI?

+

To perform query on data stored on services rather than DLI, perform the following steps:

+
  1. Assume that the data to be queried is stored on multiple services (for example, OBS) on cloud.
  2. On the DLI management console, create a table and set the Path parameter to the save path of the target data, for example, path of an OBS bucket. The data is actually stored on OBS and data migration is not required.
  3. On the DLI management console, edit SQL statements to query and analyze the data.
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0013.html b/docs/dli/umn/dli_03_0013.html new file mode 100644 index 00000000..13a73692 --- /dev/null +++ b/docs/dli/umn/dli_03_0013.html @@ -0,0 +1,11 @@ + + +

The Compression Ratio of OBS Tables Is Too High

+

A high compression ratio of OBS tables in the Parquet or ORC format (for example, a compression ratio of 5 or higher compared with text compression) will lead to large data volumes to be processed by a single task. In this case, you are advised to set dli.sql.files.maxPartitionBytes to 33554432 (default: 134217728) in the conf field in the submit-job request body to reduce the data to be processed per task.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0014.html b/docs/dli/umn/dli_03_0014.html new file mode 100644 index 00000000..dc613bef --- /dev/null +++ b/docs/dli/umn/dli_03_0014.html @@ -0,0 +1,11 @@ + + +

What Should I Do If Error "path obs://xxx already exists" Is Reported When Data Is Exported to OBS?

+

Create an OBS directory with a unique name. Alternatively, you can manually delete the existing OBS directory and submit the job again. However, exercise caution when deleting the existing OBS directory because the operation will delete all data in the directory.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0017.html b/docs/dli/umn/dli_03_0017.html new file mode 100644 index 00000000..a6b301aa --- /dev/null +++ b/docs/dli/umn/dli_03_0017.html @@ -0,0 +1,35 @@ + + +

How Do I Set the AK/SK for a Queue to Operate an OBS Table?

+
+

For security purposes, you are advised not to include the AK and SK information in the OBS path. In addition, if a table is created in the OBS directory, the OBS path specified by the Path field cannot contain the AK and SK information.

+
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0020.html b/docs/dli/umn/dli_03_0020.html new file mode 100644 index 00000000..7dc5e544 --- /dev/null +++ b/docs/dli/umn/dli_03_0020.html @@ -0,0 +1,91 @@ + + +

Problems Related to SQL Jobs

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0021.html b/docs/dli/umn/dli_03_0021.html new file mode 100644 index 00000000..27ff75e2 --- /dev/null +++ b/docs/dli/umn/dli_03_0021.html @@ -0,0 +1,47 @@ + + +

Problems Related to Spark Jobs

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0022.html b/docs/dli/umn/dli_03_0022.html new file mode 100644 index 00000000..1a893b13 --- /dev/null +++ b/docs/dli/umn/dli_03_0022.html @@ -0,0 +1,54 @@ + + +

Datasource Connections

+

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0023.html b/docs/dli/umn/dli_03_0023.html new file mode 100644 index 00000000..45ae1e20 --- /dev/null +++ b/docs/dli/umn/dli_03_0023.html @@ -0,0 +1,31 @@ + + +

What Can I Do When Receiving java.lang.AbstractMethodError in the Spark Job?

+

The Spark 2.3 has changed the behavior of the internal interface Logging. If the user code directly inherits the Logging and the earlier version Spark is used during compilation, the java.lang.AbstractMethodError is reported when the application runs in the Spark 2.3 environment.

+

Solutions are as follows:

+ +
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0025.html b/docs/dli/umn/dli_03_0025.html new file mode 100644 index 00000000..a73c0333 --- /dev/null +++ b/docs/dli/umn/dli_03_0025.html @@ -0,0 +1,12 @@ + + +

Which Data Formats Does DLI Support?

+

DLI supports the following data formats:

+ +
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0028.html b/docs/dli/umn/dli_03_0028.html new file mode 100644 index 00000000..5ea12f32 --- /dev/null +++ b/docs/dli/umn/dli_03_0028.html @@ -0,0 +1,11 @@ + + +

How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS Table When Creating a DLI and Associating It with the RDS Table?

+

When data is inserted into DLI, set the ID field to NULL.

+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0029.html b/docs/dli/umn/dli_03_0029.html new file mode 100644 index 00000000..c1f8fa13 --- /dev/null +++ b/docs/dli/umn/dli_03_0029.html @@ -0,0 +1,12 @@ + + +

Where Can DLI Data Be Stored?

+

DLI data can be stored in either of the following:

+ +
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0031.html b/docs/dli/umn/dli_03_0031.html new file mode 100644 index 00000000..c001beb4 --- /dev/null +++ b/docs/dli/umn/dli_03_0031.html @@ -0,0 +1,13 @@ + + +

How Do I View My Quotas?

+
  1. Log in to the management console.
  2. Click in the upper left corner and select Region and Project.
  3. Click (the My Quotas icon) in the upper right corner.

    The Service Quota page is displayed.

    +
  4. View the used and total quota of each type of resources on the displayed page.

    If a quota cannot meet service requirements, increase a quota.

    +
+
+
+
+
Parent topic: Quota
+
+
+ diff --git a/docs/dli/umn/dli_03_0032.html b/docs/dli/umn/dli_03_0032.html new file mode 100644 index 00000000..c6b3d2ae --- /dev/null +++ b/docs/dli/umn/dli_03_0032.html @@ -0,0 +1,17 @@ + + +

How Do I Increase a Quota?

+

Increasing a Quota

The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your application and inform you of the progress by phone call or email.

+

Before dialing the hotline number or sending an email, ensure that the following information has been obtained:

+ +

Learn how to obtain the service hotline and email address.

+
+
+
+
+
Parent topic: Quota
+
+
+ diff --git a/docs/dli/umn/dli_03_0037.html b/docs/dli/umn/dli_03_0037.html new file mode 100644 index 00000000..9f0112ac --- /dev/null +++ b/docs/dli/umn/dli_03_0037.html @@ -0,0 +1,91 @@ + + +

Flink Jobs

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0038.html b/docs/dli/umn/dli_03_0038.html new file mode 100644 index 00000000..5c9d1f8b --- /dev/null +++ b/docs/dli/umn/dli_03_0038.html @@ -0,0 +1,50 @@ + + +

How Do I Configure Checkpoints for Flink Jar Jobs and Save the Checkpoints to OBS?

+

The procedure is as follows:

+
  1. Add the following code to the JAR package code of the Flink Jar job:
    // Configure the pom file on which the StreamExecutionEnvironment depends.
+StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
+        env.getCheckpointConfig().setCheckpointInterval(40000);
+        env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
+        RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(new FsStateBackend("obs://${userAk}:${userSk}@${obsendpoint}:443/${bucket}/jobs/checkpoint/my_jar"), false);
+        rocksDbBackend.setOptions(new OptionsFactory() {
+            @Override
+            public DBOptions createDBOptions(DBOptions currentOptions) {
+                return currentOptions
+                        .setMaxLogFileSize(64 * 1024 * 1024)
+                        .setKeepLogFileNum(3);
+            }
+
+            @Override
+            public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions) {
+                return currentOptions;
+            }
+        });
+        env.setStateBackend(rocksDbBackend);
    +

    The preceding code saves the checkpoint to the ${bucket} bucket in jobs/checkpoint/my_jar path every 40 seconds in EXACTLY_ONCE mode.

    +

    Pay attention to the checkpoint storage path. Generally, the checkpoint is stored in the OBS bucket. The path format is as follows:

    +
    • obs://${dataUserAk}:${dataUserSk}@${obsendpoint}:443/${bucket}/xxx/xxx/xxx
    • Add the following configuration to the POM file for the packages on which the StreamExecutionEnvironment depends: 
      <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+</dependency>
      +
    +
    +
  2. Enable Restore Job from Checkpoint for a DLI Flink Jar job.
    1. On the DLI management console, choose Job Management > Flink Jobs from the navigation pane on the left.
    2. In the Operation column of the Flink Jar job, click Edit. The Flink Jar job editing page is displayed.
    3. Select Auto Restart upon Exception.
    4. Select Restore Job from Checkpoint and set the Checkpoint Path.

      The checkpoint path is the same as that you set in JAR package code. The format is as follows:

      +
      • ${bucket}/xxx/xxx/xxx
      • Example

        If the path in the JAR package is obs://xxxxxxx:xxxxxxxxxx@${obsendpoint}:443/mybucket/jobs/checkpoint/jar-3,

        +

        Set Checkpoint Path to mybucket/jobs/checkpoint/jar-3.

        +
      +
    +
    • The checkpoint path for each Flink Jar job must be unique. Otherwise, data cannot be restored.
    • DLI can access files in the checkpoint path only after DLI is authorized to access the OBS bucket.
    +
    +
  3. Check whether the job is restored from the checkpoint.
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0040.html b/docs/dli/umn/dli_03_0040.html new file mode 100644 index 00000000..76998031 --- /dev/null +++ b/docs/dli/umn/dli_03_0040.html @@ -0,0 +1,13 @@ + + +

What Should I Do If the DIS Stream Does Not Exist During the Semantic Check?

+

To rectify this fault, perform the following steps:

+
  1. Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statements to check whether the DIS stream exists.
  2. If the DIS stream was not created, create a DIS stream by referring to "Creating a DIS Stream" in the Data Ingestion Service User Guide.

    Ensure that the created DIS stream and Flink job are in the same region.

    +
  3. If a DIS stream has been created, check whether the DIS stream and the Flink job are in the same region.
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0043.html b/docs/dli/umn/dli_03_0043.html new file mode 100644 index 00000000..2b32fa22 --- /dev/null +++ b/docs/dli/umn/dli_03_0043.html @@ -0,0 +1,11 @@ + + +

How Can I Check Flink Job Results?

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0044.html b/docs/dli/umn/dli_03_0044.html new file mode 100644 index 00000000..db7d4f81 --- /dev/null +++ b/docs/dli/umn/dli_03_0044.html @@ -0,0 +1,12 @@ + + +

Does a Flink JAR Job Support Configuration File Upload? How Do I Upload a Configuration File?

+

Configuration files can be uploaded for user-defined jobs (JAR).

+
  1. Upload the configuration file to DLI through Package Management.
  2. In the Other Dependencies area of the Flink JAR job, select the created DLI package.
  3. Load the file through ClassName.class.getClassLoader().getResource("userData/fileName") in the code. In the file name, fileName indicates the name of the file to be accessed, and ClassName indicates the name of the class that needs to access the file.
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0045.html b/docs/dli/umn/dli_03_0045.html new file mode 100644 index 00000000..00c6aa8e --- /dev/null +++ b/docs/dli/umn/dli_03_0045.html @@ -0,0 +1,13 @@ + + +

What Should I Do If the OBS Bucket Selected for a Job Is Not Authorized?

+

If the OBS bucket selected for a job is not authorized, perform the following steps:

+
  1. On the DLI management console, choose Global Configuration > Service Authorization and select the Tenant Administrator (Global service) permission. Update the permission settings.
  2. In the navigation pane, choose Job Management > Flink Jobs.
  3. In the row where the target job resides, click Edit under Operation to switch to the Edit page.
  4. Configure parameters under Running Parameters on the Edit page.

    1. Select Enable Checkpointing or Save Job Log.
    2. Specify OBS Bucket.
    3. Select Authorize OBS.
    +

+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0046.html b/docs/dli/umn/dli_03_0046.html new file mode 100644 index 00000000..aab7541d --- /dev/null +++ b/docs/dli/umn/dli_03_0046.html @@ -0,0 +1,13 @@ + + +

What Should I Do If A Schema Parsing Error Is Reported When I Create a Hive Table Using CTAS

+

Currently, DLI supports the Hive syntax for creating tables of the TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET file types. If the file format specified for creating a table in the CTAS is AVRO and digits are directly used as the input of the query statement (SELECT), for example, if the query is CREATE TABLE tb_avro STORED AS AVRO AS SELECT 1, a schema parsing exception is reported.

+

If the column name is not specified, the content after SELECT is used as both the column name and inserted value. The column name of the AVRO table cannot be a digit. Otherwise, an error will be reported, indicating that the schema fails to be parsed.

+

Solution: You can use CREATE TABLE tb_avro STORED AS AVRO AS SELECT 1 AS colName to specify the column name or set the storage format to a format other than AVRO.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0047.html b/docs/dli/umn/dli_03_0047.html new file mode 100644 index 00000000..71a6e34f --- /dev/null +++ b/docs/dli/umn/dli_03_0047.html @@ -0,0 +1,18 @@ + + +

Why Is the Error Message "communication link failure" Displayed When I Use a Newly Activated Datasource Connection?

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0048.html b/docs/dli/umn/dli_03_0048.html new file mode 100644 index 00000000..6f98e404 --- /dev/null +++ b/docs/dli/umn/dli_03_0048.html @@ -0,0 +1,16 @@ + + +

How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?

+

Add the following SQL statements to the Flink job:

+
create source stream ssource(xx);
+create sink stream es1(xx) with (xx);
+create sink stream es2(xx) with (xx);
+insert into es1 select * from ssource;
+insert into es2 select * from ssource;
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0049.html b/docs/dli/umn/dli_03_0049.html new file mode 100644 index 00000000..b0eb2933 --- /dev/null +++ b/docs/dli/umn/dli_03_0049.html @@ -0,0 +1,35 @@ + + +

Queue

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0053.html b/docs/dli/umn/dli_03_0053.html new file mode 100644 index 00000000..a903a240 --- /dev/null +++ b/docs/dli/umn/dli_03_0053.html @@ -0,0 +1,18 @@ + + +

Quota

+

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0054.html b/docs/dli/umn/dli_03_0054.html new file mode 100644 index 00000000..0b43f449 --- /dev/null +++ b/docs/dli/umn/dli_03_0054.html @@ -0,0 +1,29 @@ + + +

Permission

+
+
+ + +
+
Parent topic: FAQs
+
+
+ diff --git a/docs/dli/umn/dli_03_0061.html b/docs/dli/umn/dli_03_0061.html new file mode 100644 index 00000000..a66ec3f3 --- /dev/null +++ b/docs/dli/umn/dli_03_0061.html @@ -0,0 +1,11 @@ + + +

Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the Data?

+

The temp stream in Flink SQL is similar to a subquery. It is a logical stream used to simplify the SQL logic and does not generate data storage. Therefore, there is no need to clean data.

+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0064.html b/docs/dli/umn/dli_03_0064.html new file mode 100644 index 00000000..93475576 --- /dev/null +++ b/docs/dli/umn/dli_03_0064.html @@ -0,0 +1,16 @@ + + +

What Should I Do If Logs Are Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?

+

Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:

+ +
Flink dependencies have been built in the DLI server and security hardening has been performed based on the open-source community version. To avoid dependency package compatibility issues or log output and dump issues, be careful to exclude the following files when packaging: +
+

On this basis, the taskmanager.log file rolls as the log file size and time change.

+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0065.html b/docs/dli/umn/dli_03_0065.html new file mode 100644 index 00000000..7171d5a8 --- /dev/null +++ b/docs/dli/umn/dli_03_0065.html @@ -0,0 +1,13 @@ + + +

How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?

+

Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earlier versions and are used to run Spark and Flink jobs.

+

You can perform the following steps to convert an old Spark queue to a general purpose queue.

+
  1. Purchase a general purpose queue again.
  2. Migrate the jobs in the old Spark queue to the new general queue. That is, specify a new queue when submitting Spark jobs.
  3. Release the old Spark queue, that is, delete it or unsubscribe it from the queue.
+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0066.html b/docs/dli/umn/dli_03_0066.html new file mode 100644 index 00000000..bd7a7190 --- /dev/null +++ b/docs/dli/umn/dli_03_0066.html @@ -0,0 +1,16 @@ + + +

What Should I Do If "SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;" Is Displayed When Two Tables Are Joined

+

This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.

+

For example, tables tb1 and tb2 contain the id field.

+
Incorrect command:
select id from tb1 join tb2;
+
+

Correct command:

+
select tb1.id from tb1 join tb2;
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0068.html b/docs/dli/umn/dli_03_0068.html new file mode 100644 index 00000000..974a170f --- /dev/null +++ b/docs/dli/umn/dli_03_0068.html @@ -0,0 +1,17 @@ + + +

How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?

+

Scenario

When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.

+

To solve this problem, you can configure spark.sql.shuffle.partitions to improve the concurrency of shuffle read tasks.

+
+

Configuring spark.sql.shuffle.partitions

You can use the set clause to configure the dli.sql.shuffle.partitions parameter in JDBC. The statement is as follows:

+
Statement st = conn.stamte()
+st,execute("set spark.sql.shuffle.partitions=20")
+
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0069.html b/docs/dli/umn/dli_03_0069.html new file mode 100644 index 00000000..77d25473 --- /dev/null +++ b/docs/dli/umn/dli_03_0069.html @@ -0,0 +1,33 @@ + + +

How Can I Use the count Function to Perform Aggregation?

+

The correct method for using the count function to perform aggregation is as follows:

+
SELECT
+  http_method,
+  count(http_method)
+FROM
+  apigateway
+WHERE
+  service_id = 'ecs' Group BY http_method
+

Or

+
SELECT
+  http_method
+FROM
+  apigateway
+WHERE
+  service_id = 'ecs' DISTRIBUTE BY http_method
+

If an incorrect method is used, an error will be reported.

+
SELECT
+  http_method,
+  count(http_method)
+FROM
+  apigateway
+WHERE
+  service_id = 'ecs' DISTRIBUTE BY http_method
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0071.html b/docs/dli/umn/dli_03_0071.html new file mode 100644 index 00000000..5d9ef42d --- /dev/null +++ b/docs/dli/umn/dli_03_0071.html @@ -0,0 +1,12 @@ + + +

What Should I Do If the System Notify Me that I Have No Permission to Execute the Query Statement Because the Current Account Was Restricted?

+

Check whether the account is in arrears. If yes, recharge the account.

+

If the error persists, log out and log in again.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0072.html b/docs/dli/umn/dli_03_0072.html new file mode 100644 index 00000000..e4ce9f3c --- /dev/null +++ b/docs/dli/umn/dli_03_0072.html @@ -0,0 +1,12 @@ + + +

How Do I Synchronize DLI Table Data from One Region to Another?

+

You can use the cross-region replication function of OBS. The procedure is as follows:

+
  1. Export the DLI table data in region 1 to the user-defined OBS bucket. For details, see "Exporting Data from DLI to OBS" in the Data Lake Insight User Guide.
  2. Use the OBS cross-region replication function to replicate data to the OBS bucket in region 2. For details, see "Exporting Data from DLI to OBS" in the Data Lake Insight User Guide.
  3. Import or use the corresponding data as required.
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0075.html b/docs/dli/umn/dli_03_0075.html new file mode 100644 index 00000000..09adacbd --- /dev/null +++ b/docs/dli/umn/dli_03_0075.html @@ -0,0 +1,64 @@ + + +

How Can I Map an OBS Table to a DLI Partition Table?

+

In this example, the day field is used as the partition field with the parquet encoding format (only the parquet format is supported currently) to dump car_info data to OBS. For more information, see the "File System Sink Stream" in the Data Lake Insight SQL Syntax Reference.

+
 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
create sink stream car_infos (
+  carId string,
+  carOwner string,
+  average_speed double,
+  day string
+  ) partitioned by (day)
+  with (
+    type = "filesystem",
+    file.path = "obs://obs-sink/car_infos",
+    encode = "parquet",
+    ak = "{{myAk}}",
+    sk = "{{mySk}}"
+);
+
+ +
+

Structure of the data storage directory in OBS: obs://obs-sink/car_infos/day=xx/part-x-x.

+

After the data is generated, the OBS partition table can be established for subsequent batch processing through the following SQL statements:

+
  1. Create an OBS partition table. 
    1
+2
+3
+4
+5
+6
+7
+8
    create table car_infos (
+  carId string,
+  carOwner string,
+  average_speed double
+)
+  partitioned by (day string)
+  stored as parquet
+  location 'obs://obs-sink/car-infos';
+
    + +
    +
  2. Restore partition information from the associated OBS path.
    1
    alter table car_infos recover partitions;
+
    + +
    +
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0076.html b/docs/dli/umn/dli_03_0076.html new file mode 100644 index 00000000..15014241 --- /dev/null +++ b/docs/dli/umn/dli_03_0076.html @@ -0,0 +1,15 @@ + + +

How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?

+
  1. If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management page. The procedure is as follows:
    1. Upload the egg package to the specified OBS path.
    2. Log in to the DLI management console and choose Data Management > Package Management.
    3. On the Package Management page, click Create Package in the upper right corner to create a package.
    4. In the Create Package dialog, set the following parameters:
      • Type: Select PyFile.
      • OBS Path: Select the OBS path where the egg package is stored.
      • Set Group and Group Name as you need.
      +
    5. Click OK.
    6. On the Spark job editing page where the error is reported, choose the uploaded egg package from the Python File Dependencies drop-down list and run the Spark job again.
    +
  2. To interconnect PySpark jobs with MySQL, you need to create a datasource connection to enable the network between DLI and RDS.

    For details about how to create a datasource connection on the management console, see Data Lake Insight User Guide > Enhanced Datasource Connections.

    +

    For details about how to call an API to create a datasource connection, see Data Lake Insight API Reference > Creating an Enhanced Datasource Connection.

    +
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0077.html b/docs/dli/umn/dli_03_0077.html new file mode 100644 index 00000000..0414b63d --- /dev/null +++ b/docs/dli/umn/dli_03_0077.html @@ -0,0 +1,13 @@ + + +

What Can I Do If the Specified Python Environment Cannot Be Found After a Python Package Is Added?

+

If the specified Python environment cannot be found after a Python3 package is added, the current compute cluster environment is Python2 by default.

+

You can set spark.yarn.appMasterEnv.PYSPARK_PYTHON=python3 in the conf file to specify the Python3 environment for the compute cluster.

+

New clusters use the Python3 environment by default.

+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0080.html b/docs/dli/umn/dli_03_0080.html new file mode 100644 index 00000000..75f056b8 --- /dev/null +++ b/docs/dli/umn/dli_03_0080.html @@ -0,0 +1,13 @@ + + +

Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs

+

The cluster host information is not added to the datasource connection. As a result, the KRB authentication fails, the connection times out, and no error is recorded in logs. Configure the host information and try again.

+

On the Enhanced page, select the connection and click Modify Host. In the dialog box that is displayed, enter the host information. The format is Host IP address Host name/Domain name. Multiple records are separated by line breaks.

+

For details, see section "Modifying the Host Information" in the Data Lake Insight User Guide.

+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0082.html b/docs/dli/umn/dli_03_0082.html new file mode 100644 index 00000000..06adec9f --- /dev/null +++ b/docs/dli/umn/dli_03_0082.html @@ -0,0 +1,16 @@ + + +

How Do I Run a Complex PySpark Program in DLI?

+

DLI natively supports PySpark.

+

For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are packed into JAR packages and depend on third-party JAR packages. Similarly, Python programs also depend on third-party libraries, especially big data analysis programs related to PySpark-based converged machine learning. Traditionally, the Python library is installed on the execution machine based on pip. For serverless services such as DLI, you do not need to and are unaware of the underlying compute resources. In this case, how does DLI ensure that you run their programs perfectly?

+

DLI has built-in algorithm libraries for machine learning in its compute resources. These common algorithm libraries meet the requirements of most users. What if a user's PySpark program depends on a program library that is not provided by the built-in algorithm library? Actually, the dependency of PySpark is specified based on PyFiles. On the DLI Spark job page, you can directly select the Python third-party program library (such as ZIP and EGG) stored on OBS.

+

The compressed package of the dependent third-party Python library has structure requirements. For example, if the PySpark program depends on moduleA (import moduleA), the compressed package must meet the following structure requirement:

+
Figure 1 Compressed package structure requirement
+

That is, the compressed package contains a folder named after a module name, and then the Python file of the corresponding class. Generally, the downloaded Python library may not meet this requirement. Therefore, you need to compress the Python library again. In addition, there is no requirement on the name of the compressed package. Therefore, it is recommended that you compress the packages of multiple modules into a compressed package. Now, a large and complex PySpark program is configured and runs normally.

+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0083.html b/docs/dli/umn/dli_03_0083.html new file mode 100644 index 00000000..61b2dbaa --- /dev/null +++ b/docs/dli/umn/dli_03_0083.html @@ -0,0 +1,14 @@ + + +

What Data Formats and Data Sources Are Supported by DLI Flink Jobs?

+

DLI Flink jobs support the following data formats:

+

Avro, Avro_merge, BLOB, CSV, EMAIL, JSON, ORC, Parquet, and XML.

+

DLI Flink jobs support data from the following data sources:

+

CloudTable HBase, CloudTable OpenTSDB, CSS Elasticsearch, DCS, DDS, DIS, DMS, GaussDB(DWS), EdgeHub, MRS HBase, MRS Kafka, open-source Kafka, file systems, OBS, RDS, and SMN

+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0085.html b/docs/dli/umn/dli_03_0085.html new file mode 100644 index 00000000..6ab75c37 --- /dev/null +++ b/docs/dli/umn/dli_03_0085.html @@ -0,0 +1,11 @@ + + +

How Can I Access Data Across Regions?

+
  1. Connect VPCs in different regions.
  2. Create an enhanced datasource connection on DLI and bind it to a queue.
  3. Add a DLI route.
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0086.html b/docs/dli/umn/dli_03_0086.html new file mode 100644 index 00000000..2a126246 --- /dev/null +++ b/docs/dli/umn/dli_03_0086.html @@ -0,0 +1,15 @@ + + +

How Do I Merge Small Files?

+

If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.

+
  1. Set the configuration item as follows:

    spark.sql.shuffle.partitions = Number of partitions (number of the generated small files in this case)

    +
  2. Execute the following SQL statements:
    INSERT OVERWRITE TABLE tablename
+select  * FROM  tablename distribute by rand()
    +
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0087.html b/docs/dli/umn/dli_03_0087.html new file mode 100644 index 00000000..aca9ae27 --- /dev/null +++ b/docs/dli/umn/dli_03_0087.html @@ -0,0 +1,12 @@ + + +

How Do I Set Local Variables in SQL Statements?

+

You can use the where condition statement in the select statement to filter data. For example:

+
select * from table where part = '202012'
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0088.html b/docs/dli/umn/dli_03_0088.html new file mode 100644 index 00000000..3e6babb9 --- /dev/null +++ b/docs/dli/umn/dli_03_0088.html @@ -0,0 +1,12 @@ + + +

How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?

+

In DLI, 64 CU = 64 cores and 256 GB memory.

+

In a Spark job, if the driver occupies 4 cores and 16 GB memory, the executor can occupy 60 cores and 240 GB memory.

+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0089.html b/docs/dli/umn/dli_03_0089.html new file mode 100644 index 00000000..1c1f224a --- /dev/null +++ b/docs/dli/umn/dli_03_0089.html @@ -0,0 +1,67 @@ + + +

How Do I Map an OBS Table to a Partitioned DLI Table?

+

Scenario

When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.

+
+

Procedure

In the following example, the day field is used as the partition field with the parquet encoding format (only the parquet format is supported currently) to dump car_info data to OBS.

+
 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
create sink stream car_infos (
+  carId string,
+  carOwner string,
+  average_speed double,
+  day string
+  ) partitioned by (day)
+  with (
+    type = "filesystem",
+    file.path = "obs://obs-sink/car_infos",
+    encode = "parquet",
+    ak = "{{myAk}}",
+    sk = "{{mySk}}"
+);
+
+ +
+

Structure of the data storage directory in OBS: obs://obs-sink/car_infos/day=xx/part-x-x.

+

After the data is generated, the OBS partition table can be established for subsequent batch processing through the following SQL statements:

+
  1. Create an OBS partitioned table. 
    1
+2
+3
+4
+5
+6
+7
+8
    create table car_infos (
+  carId string,
+  carOwner string,
+  average_speed double
+)
+  partitioned by (day string)
+  stored as parquet
+  location 'obs://obs-sink/car-infos';
+
    + +
    +
  2. Restore partition information from the associated OBS path.
    1
    alter table car_infos recover partitions;
+
    + +
    +
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0090.html b/docs/dli/umn/dli_03_0090.html new file mode 100644 index 00000000..cb4f81c1 --- /dev/null +++ b/docs/dli/umn/dli_03_0090.html @@ -0,0 +1,14 @@ + + +

How Do I Set Auto Restart upon Exception for a Flink Job?

+

Scenario

DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are rectified.

+
+

Procedure

  1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
  2. Click a job name and select Auto-restart upon exception on the job editing page.
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0091.html b/docs/dli/umn/dli_03_0091.html new file mode 100644 index 00000000..b7e0b770 --- /dev/null +++ b/docs/dli/umn/dli_03_0091.html @@ -0,0 +1,15 @@ + + +

How Do I View DLI SQL Logs?

+

Scenario

You can view SQL job logs for routine O&M.

+
+

Procedure

  1. Obtain the ID of the DLI job executed on the DataArts Studio console.
    Figure 1 Job ID
    +
  2. On the DLI console, choose Job Management > SQL Jobs.
  3. On the SQL Jobs page, enter the job ID.
  4. In the Operation column of the target job, choose More > View Log and download the logs to your PC.
  5. Search for job ID in the downloaded logs to view the execution logs.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0092.html b/docs/dli/umn/dli_03_0092.html new file mode 100644 index 00000000..99dcc82c --- /dev/null +++ b/docs/dli/umn/dli_03_0092.html @@ -0,0 +1,20 @@ + + +

How Do I Specify an OBS Path When Creating an OBS Table?

+

Scenario

When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.

+
+

Correct Example

CREATE TABLE `di_seller_task_activity_30d` (`user_id` STRING COMMENT' user ID...) SORTED as parquet
+LOCATION 'obs://akc-bigdata/akdc.db/di_seller_task_activity_30d'
+
+

Incorrect Example

CREATE TABLE `di_seller_task_activity_30d` (`user_id` STRING COMMENT' user ID...) SORTED as parquet
+LOCATION 'obs://akc-bigdata/akdc.db'
+

If the specified path is akdc.db, data in this path will be cleared when the insert overwrite statement is executed.

+
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0093.html b/docs/dli/umn/dli_03_0093.html new file mode 100644 index 00000000..afd6ec18 --- /dev/null +++ b/docs/dli/umn/dli_03_0093.html @@ -0,0 +1,26 @@ + + +

How Do I Eliminate Data Skew by Configuring AE Parameters?

+

Scenario

If the execution of an SQL statement takes a long time, you need to access the Spark UI to check the execution status.

+

If data skew occurs, the running time of a stage exceeds 20 minutes and only one task is running.

+
Figure 1 Data skew example
+
+

Procedure

  1. Log in to the DLI management console. Choose Job Management > SQL Jobs in the navigation pane. On the displayed page, locate the job you want to modify and click Edit in the Operation column to switch to the SQL Editor page.
  2. On the SQL editor page, click Set Property and add the following Spark parameters through the Settings pane:

    +

    The string followed by the colons (:) are the configuration parameters, and the strings following the colons are the values.

    +
    spark.sql.enableToString:false
+spark.sql.adaptive.join.enabled:true
+spark.sql.adaptive.enabled:true
+spark.sql.adaptive.skewedJoin.enabled:true
+spark.sql.adaptive.enableToString:false
+spark.sql.adaptive.skewedPartitionMaxSplits:10
    +

    spark.sql.adaptive.skewedPartitionMaxSplits indicates the maximum number of tasks for processing a skewed partition. The default value is 5, and the maximum value is 10. This parameter is optional.

    +
    +
  3. Click Execute to run the job again.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0095.html b/docs/dli/umn/dli_03_0095.html new file mode 100644 index 00000000..22a0b58f --- /dev/null +++ b/docs/dli/umn/dli_03_0095.html @@ -0,0 +1,14 @@ + + +

How Do I View DLI Queue Load?

+

Scenario

To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.

+
+

Procedure

  1. Search for Cloud Eye on the console.
  2. In the navigation pane on the left, choose Cloud Service Monitoring > Data Lake Insight.
  3. Select the queue you want to view.
+
+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0096.html b/docs/dli/umn/dli_03_0096.html new file mode 100644 index 00000000..17717467 --- /dev/null +++ b/docs/dli/umn/dli_03_0096.html @@ -0,0 +1,16 @@ + + +

How Do I Prevent Data Loss After Flink Job Restart?

+

The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due to an exception.

+ +
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0098.html b/docs/dli/umn/dli_03_0098.html new file mode 100644 index 00000000..443e9011 --- /dev/null +++ b/docs/dli/umn/dli_03_0098.html @@ -0,0 +1,12 @@ + + +

How Do I Monitor Queue Exceptions?

+

DLI allows you to subscribe to an SMN topic for failed jobs.

+
  1. Log in to the DLI console.
  2. In the navigation pane on the left, choose Queue Management.
  3. On the Queue Management page, click Create SMN Topic in the upper left corner. .
+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0099.html b/docs/dli/umn/dli_03_0099.html new file mode 100644 index 00000000..261ba5c7 --- /dev/null +++ b/docs/dli/umn/dli_03_0099.html @@ -0,0 +1,14 @@ + + +

How Do I Save Flink Job Logs?

+

When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.

+

To set the OBS bucket for storing the job logs, specify a bucket for OBS Bucket. If the selected OBS bucket is not authorized, click Authorize.

+

The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID. You can customize the bucket name in the path. /jobs/logs/Directory starting with the job ID is a fixed format.

+

In the job list, click the job name. In the Run Log tab, click the provided OBS link to go to the path.

+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0100.html b/docs/dli/umn/dli_03_0100.html new file mode 100644 index 00000000..19039a0f --- /dev/null +++ b/docs/dli/umn/dli_03_0100.html @@ -0,0 +1,63 @@ + + +

How Do I Manage Fine-Grained DLI Permissions?

+

DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM to manage DLI permissions.

+

With IAM, you can use your account to create IAM users for your employees, and assign permissions to the users to control their access to specific resource types. For example, some software developers in your enterprise need to use DLI resources but must not delete them or perform any high-risk operations. To achieve this result, you can create IAM users for the software developers and grant them only the permissions required for using DLI resources.

+

For a new user, you need to log in for the system to record the metadata before using DLI.

+
+

IAM can be used free of charge. You pay only for the resources in your account.

+

If the account has met your requirements, you do not need to create an independent IAM user for permission management. Then you can skip this section. This will not affect other functions of DLI.

+

DLI System Permissions

Table 1 lists all the system-defined roles and policies supported by DLI.

+
You can grant users permissions by using roles and policies.
  • Roles: A type of coarse-grained authorization that defines permissions related to user responsibilities. This mechanism provides only a limited number of service-level roles for authorization. When using roles to grant permissions, you need to also assign other roles on which the permissions depend to take effect. Roles are not an ideal choice for fine-grained authorization and secure access control.
  • Policies: A type of fine-grained authorization that defines permissions required to perform operations on specific cloud resources under certain conditions. This type of authorization is more flexible and ideal for secure access control. For example, you can grant DLI users only the permissions for managing a certain type of cloud servers.
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 DLI system permissions

Role/Policy Name

+

Description

+

Category

+

DLI FullAccess

+

Full permissions for DLI.

+

System-defined policy

+

DLI ReadOnlyAccess

+

Read-only permissions for DLI.

+

With read-only permissions, you can use DLI resources and perform operations that do not require fine-grained permissions. For example, create global variables, create packages and package groups, submit jobs to the default queue, create tables in the default database, create datasource connections, and delete datasource connections.

+

System-defined policy

+

Tenant Administrator

+

Tenant administrator

+
  • Administer permissions for managing and accessing all cloud services. After a database or a queue is created, the user can use the ACL to assign rights to other users.
  • Scope: project-level service
+

System-defined role

+

DLI Service Admin

+

DLI administrator

+
  • Administer permissions for managing and accessing the queues and data of DLI. After a database or a queue is created, the user can use the ACL to assign rights to other users.
  • Scope: project-level service
+

System-defined role

+
+
+
+
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0101.html b/docs/dli/umn/dli_03_0101.html new file mode 100644 index 00000000..a64a5625 --- /dev/null +++ b/docs/dli/umn/dli_03_0101.html @@ -0,0 +1,89 @@ + + +

How Do I Create a Custom Policy?

+

You can set actions and resources of different levels for various scenarios.

+
  1. Define an action.

    The format is Service name:Resource type:Action. You can use wildcard *. The following is an example.

    + +
    + + + + + + + + + + + + + +
    Table 1 Action

    Action

    +

    Description

    +

    dli:queue:submit_job

    +

    Queue submission

    +

    dli:queue:*

    +

    All operations on a queue

    +

    dli:*:*

    +

    All operations on all DLI resource types

    +
    +
    +
  2. Define a resource.

    The format is Service name:Region:Domain ID:Resource type:Resource path. The wildcard (*) indicates all resources. The five fields in the format can be as you need. The resource path can be set for different levels of permission control based on scenario requirements. If you need to set all resources of the service, you do not need to specify the resource path field. The following is an example.

    + +
    + + + + + + + + + + + + + + + + +
    Table 2 Resource

    Resource

    +

    Description

    +

    DLI:*:*:table:databases.dbname.tables.*

    +

    DLI, any region, any account ID, all table resources of database dbname

    +

    DLI:*:*:database:databases.dbname

    +

    DLI, any region, any account ID, queue of database dbname

    +

    DLI:*:*:queue:queues.*

    +

    DLI, any region, any account ID, any queue resource

    +

    DLI:*:*:jobs:jobs.flink.1

    +

    DLI, any region, any account ID, Flink job whose ID is 1

    +
    +
    +
  3. A complete policy is composed of the preceding fields in JSON strings. You can set multiple actions and resources. You can also create a policy on a console page of the IAM service. For example:

    Create a policy to authorize users to create and delete any database, submit jobs for any queue, and delete any table under any account ID in any region of DLI.

    +
    {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": " Allow",
+            "Action": [
+                  "dli:database:create_database",
+                "dli:database:drop_database",
+                  "dli:queue:submit_job",
+                  "dli:table:drop_table"
+            ],
+            "Resource": [
+                  "dli:*:*:database:*",
+                "dli:*:*:queue:*",
+                "dli:*:*:table:*"
+            ]
+        }
+    ]
+}
    +
+
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0102.html b/docs/dli/umn/dli_03_0102.html new file mode 100644 index 00000000..095d9d61 --- /dev/null +++ b/docs/dli/umn/dli_03_0102.html @@ -0,0 +1,21 @@ + + +

How Do I View the Resource Usage of DLI Spark Jobs?

+

Viewing the Configuration of a Spark Job

Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parameters of the job.

+

These parameters are displayed only when Advanced Settings are configured when you create the job .

+
+
+

Viewing Real-Time Resource Usage of a Spark Job

Perform the following operations to view the number of running CUs occupied by a Spark job in real time:

+
+
  1. Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click SparkUI in the Operation column.
  2. On the Spark UI page, view the real-time running resources of the Spark job.
    Figure 1 SparkUI
    +
  3. On the Spark UI page, view the original configuration of the Spark job (available only to new clusters).

    On the Spark UI page, click Environment to view Driver and Executor information.

    +
    Figure 2 Driver information
    +
    Figure 3 Executor information
    +
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0103.html b/docs/dli/umn/dli_03_0103.html new file mode 100644 index 00000000..7dce0067 --- /dev/null +++ b/docs/dli/umn/dli_03_0103.html @@ -0,0 +1,13 @@ + + +

How Do I Locate a Flink Job Submission Error?

+
  1. On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.

    The possible causes are as follows:

    +
    • Insufficient CUs: Increase the number of CUs of the queue.
    • Failed to generate the JAR package: Check the SQL syntax and UDFs.
    +
  2. If you cannot locate the fault or the call stack is incorrect, click the job name to go to the job details page and click the Commit Logs tab to view the job submission logs.
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0105.html b/docs/dli/umn/dli_03_0105.html new file mode 100644 index 00000000..4db45d8a --- /dev/null +++ b/docs/dli/umn/dli_03_0105.html @@ -0,0 +1,12 @@ + + +

How Do I Locate a Flink Job Running Error?

+
  1. On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enabled.
    • If the function is enabled, go to 3.
    • If the function is disabled, running logs will not be dumped to an OBS bucket. In this case, perform 2 to save job logs.
    +
  2. On the job running page, select Save Job Log and specify an OBS bucket for storing the logs. Click Start to run the job again. After the executed is complete, perform 3 and subsequent steps.
  3. In the Flink job list, click the job name. On the displayed job details page, click the Run Log tab.
  4. Click view OBS Bucket to obtain the complete run logs of the job.
  5. Download the latest jobmanager.log file, search for the keyword RUNNING to FAILED, and determine the failure cause based on the errors in the context.
  6. If the information in the jobmanager.log file is insufficient for locating the fault, find the corresponding taskmanager.log file in the run logs and search for the keyword RUNNING to FAILED to confirm the failure cause.
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0106.html b/docs/dli/umn/dli_03_0106.html new file mode 100644 index 00000000..04fd6e26 --- /dev/null +++ b/docs/dli/umn/dli_03_0106.html @@ -0,0 +1,45 @@ + + +

How Do I Optimize Performance of a Flink Job?

+

Basic Concepts and Job Monitoring

+
+

Performance Analysis

Due to Flink back pressure, the data source consumption rate can be lower than the production rate when performance of a Flink job is low. As a result, data is stacked in a Kafka consumer group. In this case, you can use back pressure and delay of the operator to find its performance bottleneck.

+ +
+

Performance Tuning

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0107.html b/docs/dli/umn/dli_03_0107.html new file mode 100644 index 00000000..eeac9832 --- /dev/null +++ b/docs/dli/umn/dli_03_0107.html @@ -0,0 +1,36 @@ + + +

How Do I Use Spark to Write Data into a DLI Table?

+

To use Spark to write data into a DLI table, configure the following parameters:

+ +

The following is an example:

+
import logging
+from operator import add
+from pyspark import SparkContext
+
+logging.basicConfig(format='%(message)s', level=logging.INFO)
+
+#import local file
+test_file_name = "D://test-data_1.txt"
+out_file_name = "D://test-data_result_1"
+
+sc = SparkContext("local","wordcount app")
+sc._jsc.hadoopConfiguration().set("fs.obs.access.key", "myak")
+sc._jsc.hadoopConfiguration().set("fs.obs.secret.key", "mysk")
+sc._jsc.hadoopConfiguration().set("fs.obs.impl", "org.apache.hadoop.fs.obs.OBSFileSystem")
+sc._jsc.hadoopConfiguration().set("fs.obs.endpoint", "myendpoint")
+
+# red: text_file rdd object
+text_file = sc.textFile(test_file_name)
+
+# counts
+counts = text_file.flatMap(lambda line: line.split(" ")).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b)
+# write
+counts.saveAsTextFile(out_file_name)
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0108.html b/docs/dli/umn/dli_03_0108.html new file mode 100644 index 00000000..fbd8f2a1 --- /dev/null +++ b/docs/dli/umn/dli_03_0108.html @@ -0,0 +1,13 @@ + + +

How Do I Create a Table Using JSON Data in an OBS Bucket?

+

DLI allows you to associate JSON data in an OBS bucket to create tables in asynchronous mode.

+

The statement for creating the table is as follows:

+
create table tb1 using json options(path 'obs://....')
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0109.html b/docs/dli/umn/dli_03_0109.html new file mode 100644 index 00000000..bcaec919 --- /dev/null +++ b/docs/dli/umn/dli_03_0109.html @@ -0,0 +1,11 @@ + + +

Does the Description of a DLI Queue Can Be Modified?

+

Currently, you are not allowed to modify the description of a created queue. You can add the description when purchasing the queue.

+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0111.html b/docs/dli/umn/dli_03_0111.html new file mode 100644 index 00000000..9c7d92c1 --- /dev/null +++ b/docs/dli/umn/dli_03_0111.html @@ -0,0 +1,13 @@ + + +

What Should I Do If the Subnet Cannot Be Found When I Create a DLI Datasource Connection?

+

When you create a VPC peering connection for the datasource connection, the following error information is displayed:

+
Failed to get subnet 2c2bd2ed-7296-4c64-9b60-ca25b5eee8fe. Response code : 404, message : {"code":"VPC.0202","message":"Query resource by id 2c2bd2ed-7296-4c64-9b60-ca25b5eee8fe fail.the subnet could not be found."}
+

Before you create a datasource connection, check whether VPC Administrator is selected. If only the global Tenant Administrator is selected, the system cannot find the subnet.

+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0115.html b/docs/dli/umn/dli_03_0115.html new file mode 100644 index 00000000..2fa39828 --- /dev/null +++ b/docs/dli/umn/dli_03_0115.html @@ -0,0 +1,12 @@ + + +

What Are the Differences Between MRS Spark and DLI Spark?

+

The Spark component of DLI is a fully managed service. You can only use the DLI Spark through its APIs. .

+

The Spark component of MRS is built on the VM in an MRS cluster. You can develop the Spark component to optimize it as needed and make API calls to use it.

+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0116.html b/docs/dli/umn/dli_03_0116.html new file mode 100644 index 00000000..88ea396f --- /dev/null +++ b/docs/dli/umn/dli_03_0116.html @@ -0,0 +1,14 @@ + + +

How Do I View SQL Execution Records?

+

Scenario

You can view the job execution records when a job is running.

+
+

Procedure

  1. Log in to the DLI management console.
  2. In the navigation pane on the left, choose Job Management > SQL Jobs.
  3. Enter a job ID or statement to search for a job.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0117.html b/docs/dli/umn/dli_03_0117.html new file mode 100644 index 00000000..8217dff6 --- /dev/null +++ b/docs/dli/umn/dli_03_0117.html @@ -0,0 +1,11 @@ + + +

What Are the Differences Between DLI Tables and OBS Tables?

+
+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0118.html b/docs/dli/umn/dli_03_0118.html new file mode 100644 index 00000000..6da9f2ed --- /dev/null +++ b/docs/dli/umn/dli_03_0118.html @@ -0,0 +1,36 @@ + + +

How Do I Read Uploaded Files for a Spark Jar Job?

+

You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get("Name of the uploaded file").

+
  • The file path in the Driver is different from that obtained by the Executor. The path obtained by the Driver cannot be passed to the Executor.
  • You still need to call SparkFiles.get("filename") in Executor to obtain the file path.
  • The SparkFiles.get() method can be called only after Spark is initialized.
+
+

The java code is as follows:

+
package main.java
+ 
+import org.apache.spark.SparkFiles
+import org.apache.spark.sql.SparkSession
+ 
+import scala.io.Source
+ 
+object DliTest {
+  def main(args:Array[String]): Unit = {
+    val spark = SparkSession.builder
+      .appName("SparkTest")
+      .getOrCreate()
+ 
+    // Driver: obtains the uploaded file.
+    println(SparkFiles.get("test"))
+ 
+    spark.sparkContext.parallelize(Array(1,2,3,4))
+         // Executor: obtains the uploaded file.
+      .map(_ => println(SparkFiles.get("test")))
+      .map(_ => println(Source.fromFile(SparkFiles.get("test")).mkString)).collect()
+  }
+}
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0119.html b/docs/dli/umn/dli_03_0119.html new file mode 100644 index 00000000..ffb27410 --- /dev/null +++ b/docs/dli/umn/dli_03_0119.html @@ -0,0 +1,14 @@ + + +

What Should I Do If Job Submission Fails Due to Flink JAR Package Conflict?

+

Symptom

The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.

+
+

Solution

Delete your JAR package that is the same as an existing JAR package of the DLI Flink platform.

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0120.html b/docs/dli/umn/dli_03_0120.html new file mode 100644 index 00000000..89ca6167 --- /dev/null +++ b/docs/dli/umn/dli_03_0120.html @@ -0,0 +1,15 @@ + + +

How Do I Change the Number of Kafka Partitions of a Flink SQL Job Without Stopping It?

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0121.html b/docs/dli/umn/dli_03_0121.html new file mode 100644 index 00000000..7137e647 --- /dev/null +++ b/docs/dli/umn/dli_03_0121.html @@ -0,0 +1,28 @@ + + +

How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0126.html b/docs/dli/umn/dli_03_0126.html new file mode 100644 index 00000000..b57d1317 --- /dev/null +++ b/docs/dli/umn/dli_03_0126.html @@ -0,0 +1,12 @@ + + +

How Do I Manage Tens of Thousands of Jobs Running on DLI?

+

If you are suggested to perform following operations to run a large number of DLI jobs:

+ +
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0127.html b/docs/dli/umn/dli_03_0127.html new file mode 100644 index 00000000..4c66b2df --- /dev/null +++ b/docs/dli/umn/dli_03_0127.html @@ -0,0 +1,12 @@ + + +

How Does a Spark Job Access a MySQL Database?

+

You can use DLI Spark jobs to access data in the MySQL database using either of the following methods:

+ +
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0128.html b/docs/dli/umn/dli_03_0128.html new file mode 100644 index 00000000..fcb0d421 --- /dev/null +++ b/docs/dli/umn/dli_03_0128.html @@ -0,0 +1,11 @@ + + +

Why Do I Need to Create a VPC Peering Connection for an Enhanced Datasource Connection?

+

You need to create a VPC peering connection to enable network connectivity. Take MRS as an example. If DLI and MRS clusters are in the same VPC, and the security group is enabled, you do not need a VPC peering connection for communications between MRS and DLI.

+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0129.html b/docs/dli/umn/dli_03_0129.html new file mode 100644 index 00000000..0a4c3e09 --- /dev/null +++ b/docs/dli/umn/dli_03_0129.html @@ -0,0 +1,11 @@ + + +

Can I Import OBS Bucket Data Shared by Other Tenants into DLI?

+

Data in the OBS bucket shared by IAM users under the same account can be imported. You cannot import data in the OBS bucket shared with other IAM account.

+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0130.html b/docs/dli/umn/dli_03_0130.html new file mode 100644 index 00000000..025ee34b --- /dev/null +++ b/docs/dli/umn/dli_03_0130.html @@ -0,0 +1,11 @@ + + +

How Much Data Can Be Processed in a Day by a Flink SQL Job?

+

The consumption capability of a Flink SQL job depends on the data source transmission, queue size, and job parameter settings. The peak consumption is 10 Mbit/s.

+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0136.html b/docs/dli/umn/dli_03_0136.html new file mode 100644 index 00000000..7914d258 --- /dev/null +++ b/docs/dli/umn/dli_03_0136.html @@ -0,0 +1,12 @@ + + +

How Do I Know Whether a Flink Job Can Be Restored from a Checkpoint After Being Restarted?

+

Check the following operations:

+ +
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0138.html b/docs/dli/umn/dli_03_0138.html new file mode 100644 index 00000000..f6b91ba6 --- /dev/null +++ b/docs/dli/umn/dli_03_0138.html @@ -0,0 +1,15 @@ + + +

What Should I Do if a Message Is Displayed Indicating that the OBS Bucket Is Not Authorized When I Select an OBS Bucket for a Flink SQL Job?

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0139.html b/docs/dli/umn/dli_03_0139.html new file mode 100644 index 00000000..c60a9381 --- /dev/null +++ b/docs/dli/umn/dli_03_0139.html @@ -0,0 +1,18 @@ + + +

How Do I Authorize a Subuser to View Flink Jobs?

+

A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.

+ +
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0140.html b/docs/dli/umn/dli_03_0140.html new file mode 100644 index 00000000..1a6cdf3d --- /dev/null +++ b/docs/dli/umn/dli_03_0140.html @@ -0,0 +1,13 @@ + + +

What Should I Do When My Account Does Not Have Permission Due To Arrears?

+

When you submit a job, a message is displayed indicating that the job fails to be submitted due to insufficient permission caused by arrears. In this case, you need to check the roles in your token:

+ +

If the two roles described about are in your token, user operations are restricted.

+
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0145.html b/docs/dli/umn/dli_03_0145.html new file mode 100644 index 00000000..b6bcde87 --- /dev/null +++ b/docs/dli/umn/dli_03_0145.html @@ -0,0 +1,17 @@ + + +

What Should I Do If Error Message "There should be at least one partition pruning predicate on partitioned table XX.YYY" Is Displayed When a Query Statement Is Executed?

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0156.html b/docs/dli/umn/dli_03_0156.html new file mode 100644 index 00000000..a1f5008b --- /dev/null +++ b/docs/dli/umn/dli_03_0156.html @@ -0,0 +1,15 @@ + + +

What Should I Do If Errors "ResponseCode: 403" and "ResponseStatus: Forbidden" Are Reported When a Spark Job Accesses OBS Data

+

Symptom

The following error is reported when a Spark job accesses OBS data:

+
Caused by: com.obs.services.exception.ObsException: Error message:Request Error.OBS servcie Error Message. -- ResponseCode: 403, ResponseStatus: Forbidden
+
+

Solution

Set the AK/SK to enable Spark jobs to access OBS data. For details, see How Do I Set the AK/SK for a Queue to Operate an OBS Table?.

+
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0157.html b/docs/dli/umn/dli_03_0157.html new file mode 100644 index 00000000..41f75040 --- /dev/null +++ b/docs/dli/umn/dli_03_0157.html @@ -0,0 +1,11 @@ + + +

What Should I Do if a Running Timeout Error Is Reported When a Spark Job Runs a Large Amount of Data?

+

When a Spark job accesses a large amount of data, for example, accessing data in a GaussDB(DWS) database, you are advised to set the number of concurrent tasks and enable multi-task processing.

+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0159.html b/docs/dli/umn/dli_03_0159.html new file mode 100644 index 00000000..e18d5af2 --- /dev/null +++ b/docs/dli/umn/dli_03_0159.html @@ -0,0 +1,14 @@ + + +

What Should I Do If "Queue plans create failed. The plan xxx target cu is out of quota" Is Displayed When I Schedule CU Changes?

+

Scenario

Queue plans create failed. The plan xxx target cu is out of quota is displayed when you create a scheduled scaling task.

+
+

Solution

The CU quota of the current account is insufficient. You need to apply for more quotas.

+
+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0160.html b/docs/dli/umn/dli_03_0160.html new file mode 100644 index 00000000..44d2dbe4 --- /dev/null +++ b/docs/dli/umn/dli_03_0160.html @@ -0,0 +1,14 @@ + + +

What Should I Do if "No such user. userName:xxxx." Is Displayed on the Flink Job Management Page When I Grant Permission to a User?

+

Symptom

Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is displayed.

+
+

Solution

Check whether the username exists and whether the user has logged in to the system before authorization.

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0161.html b/docs/dli/umn/dli_03_0161.html new file mode 100644 index 00000000..72af9e53 --- /dev/null +++ b/docs/dli/umn/dli_03_0161.html @@ -0,0 +1,22 @@ + + +

What Should I Do If GaussDB(DWS) Fails to Be Started When a Flink Jar Job Accesses GaussDB(DWS), and a Message Indicating Too Many Client Connections is Displayed?

+

Symptom

When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following error information:

+
FATAL:  Already too many clients, active/non-active/reserved: 5/508/3
+
+

Possible Causes

The number of GaussDB(DWS) database connections exceeds the upper limit. In the error information, the value of non-active indicates the number of idle connections. For example, if the value of non-active is 508, there are 508 idle connections.

+
+

Solution

Perform the following steps to solve the problem:

+
  1. Log in to the GaussDB(DWS) command window and run the following SQL statement to release all idle (non-active) connections temporarily:
    SELECT PG_TERMINATE_BACKEND(pid) from pg_stat_activity WHERE state='idle';
    +
  2. Check whether the application actively releases the connections. If the application does not, optimize the code to release the connections.
  3. On the GaussDB (DWS) management console, configure parameter session_timeout, which controls the timeout period of idle sessions. After an idle session's timeout period exceeds the specified value, the server automatically closes the connection.

    The default value of this parameter is 600 seconds. The value 0 indicates that the timeout limit is disabled. Do not set session_timeout to 0.

    +

    The procedure for setting parameter session_timeout is as follows:

    +
    1. Log in to the GaussDB(DWS) management console.
    2. In the navigation pane on the left, click Clusters.
    3. In the cluster list, find the target cluster and click its name. The Basic Information page is displayed.
    4. Click the Parameter Modifications tab and modify the value of parameter session_timeout. Then click Save.
    5. In the Modification Preview dialog box, confirm the modification and click Save.
    +
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0162.html b/docs/dli/umn/dli_03_0162.html new file mode 100644 index 00000000..2b7e9456 --- /dev/null +++ b/docs/dli/umn/dli_03_0162.html @@ -0,0 +1,13 @@ + + +

How Do I Change the Name of a Field in a Created Table?

+

The field names of tables that have been created cannot be changed.

+

You can create a table, define new table fields, and migrate data from the old table to the new one.

+

+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0164.html b/docs/dli/umn/dli_03_0164.html new file mode 100644 index 00000000..af95f972 --- /dev/null +++ b/docs/dli/umn/dli_03_0164.html @@ -0,0 +1,11 @@ + + +

Why is Error "verifyBucketExists on XXXX: status [403]" Is Reported When I Run a Spark Job with a User that has the Permission to Access the OBS Bucket?

+

Check whether the OBS bucket is used to store DLI logs on the Global Configuration > Job Configurations page. The job log bucket cannot be used for other purpose.

+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0165.html b/docs/dli/umn/dli_03_0165.html new file mode 100644 index 00000000..34a35f26 --- /dev/null +++ b/docs/dli/umn/dli_03_0165.html @@ -0,0 +1,17 @@ + + +

What Should I Do If Error Message "Authentication failed" Is Displayed When a Flink Jar Job Is Running?

+

Symptom

An exception occurred when a Flink Jar job is running. The following error information is displayed in the job log:
org.apache.flink.shaded.curator.org.apache.curator.ConnectionState - Authentication failed
+
+
+

Possible Causes

Service authorization is not configured for the account on the Global Configuration page. When the account is used to create a datasource connection to access external data, the access fails.

+
+

Solution

  1. Log in to the DLI management console. Choose Global Configuration > Service Authorization in the navigation pane.
  2. On the Service Authorization page, select all agency permissions.
  3. Click Update. If the message "Agency permissions updated successfully" is displayed, the modification is successful.
  4. After the authorization is complete, create a datasource connection and run the job again.
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0166.html b/docs/dli/umn/dli_03_0166.html new file mode 100644 index 00000000..23a4a792 --- /dev/null +++ b/docs/dli/umn/dli_03_0166.html @@ -0,0 +1,11 @@ + + +

Will Table Data in My Database Be Lost If I Delete a Queue?

+

Deleting a queue does not cause table data loss in your database.

+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0167.html b/docs/dli/umn/dli_03_0167.html new file mode 100644 index 00000000..7c892483 --- /dev/null +++ b/docs/dli/umn/dli_03_0167.html @@ -0,0 +1,15 @@ + + +

What Should I Do If Error Message DLI.0005 Is Displayed When I Use an EL Expression to Create a Table in a Flink SQL Job?

+

Symptom

When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:
DLI.0005: AnalysisException: t_user_message_input_#{date_format(date_sub(current_date(), 1), 'yyyymmddhhmmss')} is not a valid name for tables/databases. Valid names only contain alphabet characters, numbers and _.
+
+
+

Solution

Replace the number sign (#) in the table name to the dollar sign ($). The format of the EL expression used in DLI should be ${expr}.

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0168.html b/docs/dli/umn/dli_03_0168.html new file mode 100644 index 00000000..9476d1df --- /dev/null +++ b/docs/dli/umn/dli_03_0168.html @@ -0,0 +1,40 @@ + + +

What Should I Do If No Data Is Found in the DLI Table Created Using the OBS File Path That Stores the Output of a Flink job?

+

Symptom

After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.

+

For example, use the following Flink result table to write data to the obs://obs-sink/car_infos path in OBS.

+
create sink stream car_infos_sink (
+  carId string,
+  carOwner string,
+  average_speed double,
+  buyday string
+  ) partitioned by (buyday)
+  with (
+    type = "filesystem",
+    file.path = "obs://obs-sink/car_infos",
+    encode = "parquet",
+    ak = "{{myAk}}",
+    sk = "{{mySk}}"
+);
+
Use the following statements to create a DLI partition table with data retrieved from the OBS file path. No data is found when you query the car_infos table on DLI.
create table car_infos (
+  carId string,
+  carOwner string,
+  average_speed double
+)
+  partitioned by (buyday string)
+  stored as parquet
+  location 'obs://obs-sink/car_infos';
+
+
+

Solution

  1. Check whether checkpointing is enabled for the Flink result table (car_infos_sink in the preceding example) when you create the job on DLI. If checkpointing is disabled, enable it and run the job again to generate OBS data files.

    To enable checkpointing, perform the following steps:

    +
    1. Log in to the DLI management console. Choose Job Management > Flink Jobs in the left navigation pane. Locate the row that contains the target Flink job and click Edit in the Operation column.
    2. In the Running Parameters area, check whether Enable Checkpointing is enabled.
    +
  2. Check whether the structure of the Flink result table is the same as that of the DLI partitioned table. For the preceding example, check whether the fields of the car_infos_sink table are consistent with those of the car_infos table.
  3. Check whether the partitioning information of the the partitioned table is restored after it is created using the OBS file. The following statement restore partitions of the car_infos table:
    alter table car_infos recover partitions;
    +
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0169.html b/docs/dli/umn/dli_03_0169.html new file mode 100644 index 00000000..f789e8d2 --- /dev/null +++ b/docs/dli/umn/dli_03_0169.html @@ -0,0 +1,20 @@ + + +

What Should I Do If Error Message IllegalArgumentException: Buffer size too small. size is Displayed When Data Is Loaded to an OBS Foreign Table

+

Symptom

The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:

+
error.DLI.0001: IllegalArgumentException: Buffer size too small. size = 262144 needed = 2272881
+
In some cases ,the following error message is displayed:
error.DLI.0999: InvalidProtocolBufferException: EOF in compressed stream footer position: 3 length: 479 range: 0 offset: 3 limit: 479 range 0 = 0 to 479 while trying to read 143805 bytes
+
+
+

Possible Cause

The data volume of the file to be imported is large and the value of spark.sql.shuffle.partitions is too large. As a result, the cache size is insufficient.

+
+

Solution

Decrease the spark.sql.shuffle.partitions value. To set this parameter, perform the following steps:

+
  1. Log in to the DLI management console and choose Job Management > SQL Jobs. In the Operation column of the target SQL job, click Edit to go to the SQL Editor page.
  2. On the displayed page, click Set Property and set the parameter.
  3. Execute the job again.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0170.html b/docs/dli/umn/dli_03_0170.html new file mode 100644 index 00000000..a3a6f625 --- /dev/null +++ b/docs/dli/umn/dli_03_0170.html @@ -0,0 +1,11 @@ + + +

How Does DLI Ensure the Reliability of Spark Jobs When a Queue Is Abnormal?

+

You need to develop a mechanism to retry failed jobs. When a faulty queue is recovered, your application tries to submit the failed jobs to the queue again.

+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0171.html b/docs/dli/umn/dli_03_0171.html new file mode 100644 index 00000000..31f47d3f --- /dev/null +++ b/docs/dli/umn/dli_03_0171.html @@ -0,0 +1,19 @@ + + +

What Should I Do If SQL Statements Fails to be Executed on the Default Queue, and a Timeout Exception is Reported?

+

Symptom

After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:
[ERROR] Execute DLI SQL failed. Please contact DLI service.
+[ERROR] Error message:Execution Timeout
+
+
+

Possible Cause

The default queue is a public preset queue in the system for function trials. When multiple users submit jobs to this queue, traffic control might be triggered. As a result, the jobs fail to be submitted.

+
+

Solution

Buy a custom queue for your jobs. The procedure is as follows:

+
  1. In the navigation pane of the DLI management console, choose Queue Management.
  2. In the upper right corner of the Queue Management page, click Buy Queue to create a queue.
  3. On the Buy Queue page, set the required parameters as you need. Especially, set Type to For SQL.
  4. Submit your SQL jobs to the newly created queue.
+
+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0172.html b/docs/dli/umn/dli_03_0172.html new file mode 100644 index 00000000..0b6d1ab1 --- /dev/null +++ b/docs/dli/umn/dli_03_0172.html @@ -0,0 +1,25 @@ + + +

What Should I Do If "UQUERY_CONNECTOR_0001:Invoke DLI service api failed" Is Reported in the Job Log When I Use CDM to Migrate Data to DLI?

+

Symptom

After the migration job is submitted, the following error information is displayed in the log:
org.apache.sqoop.common.SqoopException: UQUERY_CONNECTOR_0001:Invoke DLI service api failed, failed reason is %s.
+at org.apache.sqoop.connector.uquery.intf.impl.UQueryWriter.close(UQueryWriter.java:42)
+at org.apache.sqoop.connector.uquery.processor.Dataconsumer.run(Dataconsumer.java:217)
+at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
+at java.util.concurrent.FutureTask.run(FutureTask.java:266)
+at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+at java.lang.Thread.run(Thread.java:748)
+
+
+

Possible Cause

When you create a migration job to DLI on the CDM console, you set Resource Queue to a DLI queue for general purpose. It should be a queue for SQL.

+
+

Solution

  1. On the DLI management console and click Queue Management in the navigation pane on the left. On the Queue Management page, check whether there are SQL queues.
    • If there are, go to 3.
    • If there are no SQL queues, go to 2 to buy an SQL queue.
    +
  2. Click Buy Queue to create a queue. Set Type to For SQL, set other parameters required, and click Buy.
  3. Go back to the CDM console and create a data migration job. Set Resource Queue to the created DLI SQL queue.
  4. Submit the migration job and view the job execution logs.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0173.html b/docs/dli/umn/dli_03_0173.html new file mode 100644 index 00000000..7704dad6 --- /dev/null +++ b/docs/dli/umn/dli_03_0173.html @@ -0,0 +1,22 @@ + + +

What Should I Do If "org.apache.hadoop.fs.obs.OBSIOException" Is Reported If DLI SQL Scripts Are Executed on DataArts Studio?

+

Symptom

When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:
DLI.0999: RuntimeException: org.apache.hadoop.fs.obs.OBSIOException: initializing on obs://xxx.csv: status [-1] - request id
+[null] - error code [null] - error message [null] - trace :com.obs.services.exception.ObsException: OBS servcie Error Message. Request Error:
+...
+Cause by: ObsException: com.obs.services.exception.ObsException: OBSs servcie Error Message. Request Error: java.net.UnknownHostException: xxx: Name or service not known
+
+
+

Possible Cause

When you execute a DLI SQL script for the first time, you did not agree to the privacy agreement on the DLI console. As a result, the error is reported when the SQL script is executed on DataArts Studio.

+
+

Solution

  1. Log in to the DLI console, click SQL Editor from the navigation pane. On the displayed page, enter an SQL statement in the editing window, for example, select 1.
  2. In the displayed Privacy Agreement dialog box, agree to the terms.

    You only need to agree to the privacy agreement when it is your first time to execute the statements.

    +
    +
  3. Run the DLI SQL script on DataArts Studio again. The script will run properly.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0174.html b/docs/dli/umn/dli_03_0174.html new file mode 100644 index 00000000..c0ad5146 --- /dev/null +++ b/docs/dli/umn/dli_03_0174.html @@ -0,0 +1,17 @@ + + +

What Should I Do If a Flink SQL Job Fails to Be Executed, and "connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null" Is Recorded in the Log?

+

Symptom

After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:
connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null
+
+
+

Possible Cause

When configuring job running parameters for the Flink SQL job, Save Job Log or Checkpointing is enabled, and an OBS bucket for saving job logs and Checkpoints is configured. However, the IAM user who runs the Flink SQL job does not have the OBS write permission.

+
+

Solution

  1. Log in to the IAM console, search for the IAM user who runs the job in the upper left corner of the Users page.
  2. Click the desired username to view the user group where the user belongs.
  3. In the navigation pane on the left, choose User Groups, and search for the user group of the target user. Click the user group name, and view the permissions of the current user in Permissions.
  4. Check whether the user group has the permission to write data to OBS, for example, OBS OperateAccess. If the user group does not have the OBS write permission, grant the permission to the user group.
  5. Wait for 5 to 10 minutes for the permission to take effect. Run the Flink SQL job again and check the job running status.
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0175.html b/docs/dli/umn/dli_03_0175.html new file mode 100644 index 00000000..3e787c51 --- /dev/null +++ b/docs/dli/umn/dli_03_0175.html @@ -0,0 +1,17 @@ + + +

Do I Need to Grant Table Permissions to a User and Project After I Delete a Table and Create One with the Same Name?

+

Scenario

User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table and created a new testTable table. If user A wants user B to retain the insert and delete permission, user A needs to grant the permissions to user B again.

+
+

Possible Cause

After a table is deleted, the table permissions are not retained. You need to grant permissions to a user or project.

+
+

Solution

Operations to grant permissions to a user or project are as follows:

+
  1. On the left of the management console, choose Data Management > Databases and Tables.
  2. Click the database name whose table permission is to be granted. The table management page of the database is displayed.
  3. Locate the row of the target table and click Permissions in the Operation column.
  4. On the displayed page, click Grant Permission in the upper right corner.
  5. In the displayed Grant Permission dialog box, select the required permissions.
  6. Click OK.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0176.html b/docs/dli/umn/dli_03_0176.html new file mode 100644 index 00000000..391ff3c0 --- /dev/null +++ b/docs/dli/umn/dli_03_0176.html @@ -0,0 +1,17 @@ + + +

What Should I Do If "Not authorized" Is Reported When a Flink SQL Job Reads DIS Data?

+

Symptom

Semantic verification for a Flink SQL job (reading DIS data) fails. The following information is displayed when the job fails:
Get dis channel xxx info failed. error info: Not authorized, please click the overview page to do the authorize action
+
+
+

Possible Cause

Before running a Flink job, the permission to obtain DIS data is not granted to the user.

+
+

Solution

  1. Log in to the DLI management console. Choose Global Configuration > Service Authorization in the navigation pane on the left.
  2. On the Service Authorization page, select DIS Administrator and click Update.
  3. Choose Job Management > Flink Jobs. On the displayed page, locate the desired Flink SQL job and restart the job.
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0177.html b/docs/dli/umn/dli_03_0177.html new file mode 100644 index 00000000..0c775c82 --- /dev/null +++ b/docs/dli/umn/dli_03_0177.html @@ -0,0 +1,16 @@ + + +

What Should I Do If Table Data Fails to Be Queried After Data Is Imported to a DLI Partitioned Table Because the File to Be Imported Does Not Contain Data in the Partitioning Column?

+

Symptom

A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be specified for a partitioned table query. As a result, table data cannot be queried.

+
+

Possible Cause

When data is imported to a DLI partitionedtable, if the file data does not contain the partitioning column, the system specifies __HIVE_DEFAULT_PARTITION__ as the column by default. If a Spark job finds that the partition is empty, null is returned.

+
+

Solution

  1. Log in to the DLI management console. In the SQL editor, click Settings.
  2. Add spark.sql.forcePartitionPredicatesOnPartitionedTable.enabled and set it to false.
  3. Query the entire table or the partitioned table.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0179.html b/docs/dli/umn/dli_03_0179.html new file mode 100644 index 00000000..d59168ea --- /dev/null +++ b/docs/dli/umn/dli_03_0179.html @@ -0,0 +1,45 @@ + + +

What Can I Do If the Datasource Connection is Created But the Network Connectivity Test Fails?

+

Description

A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:
failed to connect to specified address
+
+
+

Fault Locating

The issues here are described in order of how likely they are to occur.

+

Troubleshoot the issue by ruling out the causes described here, one by one.

+ +
+

Check Whether a Port Number Is Added to the End of the Domain Name or IP Address

The port number is required for the connectivity test.

+

The following example tests the connectivity between a queue and a specified RDS DB instance. The RDS DB instance uses port 3306.

+

The following figure shows how you should specify the IP address.

+
+

Check Whether the Information of the Peer VPC and Subnet Are Correct.

When you create an enhanced datasource connection, you need to specify the peer VPC and subnet.

+

For example, to test the connectivity between a queue and a specified RDS DB instance, you need to specify the RDS VPC and subnet information.

+
+

Check Whether the CIDR Block of the Queue Overlaps That of the Data Source

The CIDR block of the DLI queue bound with a datasource connection cannot overlap the CIDR block of the data source.

+

You can check whether they overlap by viewing the connection logs.

+

CIDR block conflicts of queue A and queue B. In this example, queue B is bound to an enhanced datasource connection to data source C. Therefore, a message is displayed, indicating that the network segment of queue A conflicts with that of data source C. As a result, a new enhanced datasource connection cannot be established.

+

Solution: Modify the CIDR block of the queue or create another queue.

+

Planing the CIDR blocks for your queues helps you to avoid this problem.

+
+

Check Whether the VPC Administrator Permission Is Granted to DLI

View the connection logs to check whether there is the required permission.

+

Figure 1 and Figure 2 show the logs when subnet ID and route ID of the destination cannot be obtained because there is no permission .

+

Solution: Grant DLI the VPC Administrator permission and cancel the IAM ReadOnlyAccess authorization.

+
Figure 1 Viewing connection logs
+
Figure 2 Viewing connection logs
+
+

Check Whether the Destination Security Group Allows Access from the Queue.

To connect to Kafka, GaussDB(DWS), and RDS instances, add security group rules for the DLI CIDR block to the security group where the instances belong. For example, to connect a queue to RDS, perform the following operations:
  1. Log in to the DLI console, choose Resources > Queue Management in the navigation pane on the left. On the displayed page, select the target queue, and click to expand the row containing the target queue to view its CIDR block.
  2. On the Instance Management page of the RDS console, click the instance name. In the Connection Information area, locate Database Port to obtain the port number of the RDS DB instance.
  3. In the Connection Information area locate the Security Group and click the group name to switch to the security group management page. Select the Inbound Rules tab and click Add Rule. Set the priority to 1, protocol to TCP, port to the database port number, and source to the CIDR block of the DLI queue. Click OK.
+
+
+

Check the Route Information of the VPC Peering Connection Corresponding to an Enhanced Datasource Connection

Check the routing table of the VPC peering connection corresponding to the enhanced datasource connection. Check whether the CIDR block of the queue overlaps other CIDR blocks in the routing table. If it does, the forwarding may be incorrect.

+
  1. Obtain the ID of the VPC peering connection created for the enhanced datasource connection.
  2. View the information about the VPC peering connection on the VPC console.
  3. View the route table information of the VPC corresponding to the queue.
+
+

Check Whether VPC Network ACL Rules Are Configured to Restrict Network Access

Check whether an ACL is configured for the subnet corresponding to the datasource connection and whether the ACL rules restrict network access.

+

For example, if you set a CIDR block whose security group rule allows access from a queue and set a network ACL rule to deny access from that CIDR block, the security group rule does not take effect.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0180.html b/docs/dli/umn/dli_03_0180.html new file mode 100644 index 00000000..1e862dfb --- /dev/null +++ b/docs/dli/umn/dli_03_0180.html @@ -0,0 +1,15 @@ + + +

How Do I Know Which Checkpoint the Flink Job I Stopped Will Be Restored to When I Start the Job Again?

+

Symptom

Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. After a Flink job is manually stopped, no message is displayed specifying the checkpoint where the Flink job will be restored if the Flink job is started again.

+
+

Solution

The generation mechanism and format of Flink checkpoints are the same as those of savepoints. You can import a savepoint of the job to restore it from the latest checkpoint saved in OBS.

+
  1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
  2. Locate the row that contains the target Flink job, and click Import Savepoint in the Operation column.
  3. In the displayed dialog box, select the OBS bucket path storing the checkpoint. The checkpoint save path is Bucket name/jobs/checkpoint/directory starting with the job ID. Click OK.
  4. Start the Flink job again. The job will be restored fom the imported savepoint.
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0181.html b/docs/dli/umn/dli_03_0181.html new file mode 100644 index 00000000..b3c9f3a1 --- /dev/null +++ b/docs/dli/umn/dli_03_0181.html @@ -0,0 +1,24 @@ + + +

How Do I Fix the Data Error Caused by CRLF Characters in a Field of the OBS File Used to Create an External OBS Table?

+

Symptom

When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.

+

The statement for creating an OBS foreign table is similar as follows:

+
CREATE TABLE test06 (name string, id int, no string) USING csv OPTIONS (path "obs://dli-test-001/test.csv");
+
The file contains the following information (example):
Jordon,88,"aa
+bb"
+
+
A carriage return exists between aa and bb in the last field. As a result, he data in the test06 table is displayed as follows:
name	id	classno
+Jordon	88	aa
+bb"	null	null
+
+
+

Solution

When creating an OBS foreign table, set multiLine to true to specify that the column data contains CRLF characters. The following is an example to solve the problem:
CREATE TABLE test06 (name string, id int, no string) USING csv OPTIONS (path "obs://dli-test-001/test.csv",multiLine=true);
+
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0182.html b/docs/dli/umn/dli_03_0182.html new file mode 100644 index 00000000..a16576a6 --- /dev/null +++ b/docs/dli/umn/dli_03_0182.html @@ -0,0 +1,18 @@ + + +

What Should I Do If a SQL Job That Has Join Operations Stays in the Running State?

+

Symptom

A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.

+
+

Possible Cause

When a Spark SQL job has join operations on small tables, all executors are automatically broadcast to quickly complete the operations. However, this increases the memory consumption of the executors. If the executor memory usage is too high, the job fails to be executed.

+
+

Solution

  1. Check whether the /*+ BROADCAST(u) */ falg is used to forcibly perform broadcast join in the executed SQL statement. If the flag is used, remove it.
  2. Set spark.sql.autoBroadcastJoinThreshold to -1.
    1. Log in to the DLI management console and choose Job Management > SQL Jobs. In the Operation column of the failed job, click Edit to switch to the SQL editor page.
    2. Click Settings in the upper right corner. In the Parameter Settings area, add spark.sql.autoBroadcastJoinThreshold and set it to -1.
    3. Click Execute again to and view the job running result.
    +
+

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0183.html b/docs/dli/umn/dli_03_0183.html new file mode 100644 index 00000000..7aa4717c --- /dev/null +++ b/docs/dli/umn/dli_03_0183.html @@ -0,0 +1,16 @@ + + +

How Do I Determine Whether There Are Too Many Jobs in the Current Queue?

+

Description

You need to check the large number of jobs in the Submitting and Running states on the queue.

+
+

Solution

Use Cloud Eye to view jobs in different states on the queue. The procedure is as follows:

+
  1. Log in the management console and search for Cloud Eye.
  2. In the navigation pane on the left, choose Cloud Service Monitoring > Data Lake Insight.
  3. On the Cloud Service Monitoring page, click the queue name.
  4. On the monitoring page, view the following metrics to check the job status:
    1. Number of jobs being submitted: Statistics of jobs in the Submitting state on the current queue
    2. Number of running jobs: Statistics of jobs in the Running state on the current queue
    3. Number of finished jobs: Statistics of jobs in the Finished state on the current queue
    +
+
+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0184.html b/docs/dli/umn/dli_03_0184.html new file mode 100644 index 00000000..b4fe5a76 --- /dev/null +++ b/docs/dli/umn/dli_03_0184.html @@ -0,0 +1,17 @@ + + +

What Can I Do If a Table Cannot Be Queried on the DLI Console?

+

Symptom

A DLI table exists but cannot be queried on the DLI console.

+
+

Possible Cause

If a table exists but cannot be queried, there is a high probability that the current user does not have the permission to query or operate the table.

+
+

Solution

Contact the user who creates the table and obtain the required permissions. To assign permissions, perform the following steps:

+
  1. Log in to the DLI management console as the user who creates the table. Choose Data Management > Databases and Tables form the navigation pane on the left.
  2. Click the database name. The table management page is displayed. In the Operation column of the target table, click Permissions. The table permission management page is displayed.
  3. Click Set Permission. In the displayed dialog box, set Authorization Object to User, set Username to the name of the user that requires the permission, and select the required permissions. For example, Select Table and Insert permissions.
  4. Click OK.
  5. Log in to the DLI console as the user that has been granted permission and check whether the table can be queried.
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0186.html b/docs/dli/umn/dli_03_0186.html new file mode 100644 index 00000000..1dec468c --- /dev/null +++ b/docs/dli/umn/dli_03_0186.html @@ -0,0 +1,16 @@ + + +

How Do I Configure the Network Between a DLI Queue and a Data Source?

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0187.html b/docs/dli/umn/dli_03_0187.html new file mode 100644 index 00000000..bf904404 --- /dev/null +++ b/docs/dli/umn/dli_03_0187.html @@ -0,0 +1,35 @@ + + +

The on Clause Is Not Added When Tables Are Joined. Cartesian Product Query Causes High Resource Usage of the Queue, and the Job Fails to Be Executed

+

Symptom

The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resources were used up. Job execution fails on the queue.

+
For example, the following SQL statement left-joins three tables without the on clause.
select 
+     case 
+        when to_char(from_unixtime(fs.special_start_time), 'yyyy-mm-dd') < '2018-10-12' and row_number() over(partition by fg.goods_no order by fs.special_start_time asc) = 1 then 1
+        when to_char(from_unixtime(fs.special_start_time), 'yyyy-mm-dd') >= '2018-10-12' and fge.is_new = 1 then 1 
+        else 0 end as is_new
+from testdb.table1 fg 
+left join testdb.table2 fs
+left join testdb.table3  fge
+where to_char(from_unixtime(fs.special_start_time), 'yyyymmdd') = substr('20220601',1,8)
+
+
+

Solution

When you use join to perform multi-table query, you must use the on clause to reduce the data volume.

+
The following example uses the on clause for the table join, which greatly reduces the result set of associated query and improves the query efficiency.
select 
+     case 
+        when to_char(from_unixtime(fs.special_start_time), 'yyyy-mm-dd') < '2018-10-12' and row_number() over(partition by fg.goods_no order by fs.special_start_time asc) = 1 then 1
+        when to_char(from_unixtime(fs.special_start_time), 'yyyy-mm-dd') >= '2018-10-12' and fge.is_new = 1 then 1 
+        else 0 end as is_new
+from testdb.table1 fg 
+left join testdb.table2 fs on fg.col1 = fs.col2
+left join testdb.table3  fge on fg.col3 = fge.col4
+where to_char(from_unixtime(fs.special_start_time), 'yyyymmdd') = substr('20220601',1,8)
+
+
+

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0188.html b/docs/dli/umn/dli_03_0188.html new file mode 100644 index 00000000..d4a53723 --- /dev/null +++ b/docs/dli/umn/dli_03_0188.html @@ -0,0 +1,11 @@ + + +

What Should I Do If a Spark job Used to Access Files in SFTP Fails and the Log Shows that the File Directory Is Abnormal?

+

Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.

+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0189.html b/docs/dli/umn/dli_03_0189.html new file mode 100644 index 00000000..3b798398 --- /dev/null +++ b/docs/dli/umn/dli_03_0189.html @@ -0,0 +1,16 @@ + + +

What Should I Do If "DLI.0002 FileNotFoundException" Is Reported During SQL Job Running

+

Symptom

An error is reported during SQL job execution:
Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]
+
+
+

Solution

Check whether there is another job that has deleted table information.

+

DLI does not allow multiple jobs to read and write the same table at the same time. Otherwise, job conflicts may occur and the jobs fail.

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0190.html b/docs/dli/umn/dli_03_0190.html new file mode 100644 index 00000000..840ab76d --- /dev/null +++ b/docs/dli/umn/dli_03_0190.html @@ -0,0 +1,16 @@ + + +

What Should I Do If Manually Added Data in a Partition of an OBS Table Cannot Be Queried?

+

Symptom

Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.

+
+

Solution

After manually adding partition data, you need to update the metadata information of the OBS table. Run the following statement on desired table:
MSCK REPAIR TABLE table_name;
+
+
+

Query the data in the OBS partitioned table.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0191.html b/docs/dli/umn/dli_03_0191.html new file mode 100644 index 00000000..8cea3ea0 --- /dev/null +++ b/docs/dli/umn/dli_03_0191.html @@ -0,0 +1,11 @@ + + +

How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?

+

Currently, DLI does not allow you to insert table data into specific fields. To insert table data, you must insert data of all table fields at a time.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0192.html b/docs/dli/umn/dli_03_0192.html new file mode 100644 index 00000000..44ca0f7e --- /dev/null +++ b/docs/dli/umn/dli_03_0192.html @@ -0,0 +1,16 @@ + + +

What Should I Do If a Job Fails Because the Job User Does Not Have the Database and Table Permission?

+

Symptom

When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:Permission denied for resource: databases.xxx,action:SPARK_APP_ACCESS_META)
+
+
+

Solution

You need to assign the database permission to the user who executes the job. The procedure is as follows:

+
  1. In the navigation pane on the left of the management console, choose Data Management > Databases and Tables.
  2. Locate the row where the target database resides and click Permissions in the Operation column.
  3. On the displayed page, click Grant Permission in the upper right corner.
  4. In the displayed dialog box, select User or Project, enter the username or select the project that needs the permission, and select the desired permissions.
  5. Click OK.
+
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0193.html b/docs/dli/umn/dli_03_0193.html new file mode 100644 index 00000000..2e750693 --- /dev/null +++ b/docs/dli/umn/dli_03_0193.html @@ -0,0 +1,11 @@ + + +

Why Cannot I View the Resource Running Status of DLI Queues on Cloud Eye?

+

DLI queues do not use resources or bandwidth when no job is running. In this case, the running status of DLI queues is not displayed on CES.

+
+
+
+
Parent topic: Queue
+
+
+ diff --git a/docs/dli/umn/dli_03_0195.html b/docs/dli/umn/dli_03_0195.html new file mode 100644 index 00000000..80734764 --- /dev/null +++ b/docs/dli/umn/dli_03_0195.html @@ -0,0 +1,16 @@ + + +

What Should I Do If the Permission Is Required to Update a Program Package?

+

Symptom

When the user update an existing program package, the following error information is displayed:
"error_code"*CLI.0003","error_msg":"Permission denied for resource 'resources. xxx', User = 'xxx', Action = "UPDATZ_RISCURCL'."
+
+
+

Solution

You need to assign the package permission to the user who executes the job. The procedure is as follows:

+
  1. In the navigation pane on the left of the DLI console, choose Data Management > Package Management.
  2. On the Package Management page, click Manage Permission in the Operation column of the package. The User Permissions page is displayed.
  3. Click Grant Permission in the upper right corner of the page to authorize a user to access a package group or package. Select the Update Group permission.
  4. Click OK.
+
+
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0196.html b/docs/dli/umn/dli_03_0196.html new file mode 100644 index 00000000..3b65808d --- /dev/null +++ b/docs/dli/umn/dli_03_0196.html @@ -0,0 +1,40 @@ + + +

How Do I Troubleshoot Slow SQL Jobs?

+

If the job runs slowly, perform the following steps to find the causes and rectify the fault:

+

Possible Cause 1: Full GC

Check whether the problem is caused by FullGC.

+
  1. Log in to the DLI console. In the navigation pane, choose Job Management > SQL Jobs.
  2. On the SQL Jobs page, locate the row that contains the target job and click More > View Log in the Operation column.
  3. Obtain the folder of the archived logs in the OBS directory. The details are as follows:
    • Spark SQL jobs:

      Locate the log folder whose name contains driver or container_ xxx _000001.

      +

      +
    • Spark Jar jobs:

      The archive log folder of a Spark Jar job starts with batch.

      +
    +
  4. Go to the archive log file directory and download the gc.log.* log file.
  5. Open the downloaded gc.log.* log file, search for keyword Full GC, and check whether time records in the file are continuous and Full GC information is recorded repeatedly.

    +
+

Cause locating and solution

+
Cause 1: There are too many small files in a table.
  1. Log in to the DLI console and go to the SQL editor page. On the SQL Editor page, select the queue and database of the faulty job.
  2. Run the following statement to check the number of files in the table and specify the table name.
    select count(distinct fn)  FROM
+(select input_file_name() as fn from table name) a
    +
  3. If there are too many small files, rectify the fault by referring to How Do I Merge Small Files?.
+
+

Cause 2: There is a broadcast table.

+
  1. Log in to the DLI console. In the navigation pane, choose Job Management > SQL Jobs.
  2. On the SQL Jobs page, locate the row that contains the target job and click to view the job details and obtain the job ID.
  3. In the Operation column of the job, click Spark UI.
  4. On the displayed page, choose SQL from the menu bar. Click the hyperlink in the Description column of the row that contains the job ID.
  5. View the DAG of the job to check whether the BroadcastNestedLoopJoin node exists.
    Figure 1 DAG
    +
  6. If the BroadcastNestedLoopJoin node exists, refer to What Should I Do If a SQL Job That Has Join Operations Stays in the Running State? to rectify the fault.
+
+

Possible Cause 2: Data Skew

Check whether the problem is caused by data skew.

+
  1. Log in to the DLI console. In the navigation pane, choose Job Management > SQL Jobs.
  2. On the SQL Jobs page, locate the row that contains the target job and click to view the job details and obtain the job ID.
  3. In the Operation column of the job, click Spark UI.
  4. On the displayed page, choose SQL from the menu bar. Click the hyperlink in the Description column of the row that contains the job ID.

    +
  5. View the running status of the current stage in the Active Stage table on the displayed page. Click the hyperlink in the Description column.

    +
  6. View the Launch Time and Duration of each task.
  7. Click Duration to sort tasks. Check whether the overall job duration is prolonged because a task has taken a long time.
    According to Figure 2, when data skew occurs, the data volume of shuffle reads of a task is much greater than that of other tasks.
    Figure 2 Data skew
    +
    +
+

Cause locating and solution

+
Shuffle data skew is caused by unbalanced number of key values in join.
  1. Perform group by and count on a join to collect statistics on the number of key values of each join. The following is an example:

    Join table lefttbl and table righttbl. num in the lefttbl table is the key value of the join. You can perform group by and count on lefttbl.num.

    +
    SELECT * FROM lefttbl a LEFT join righttbl b on a.num = b.int2;
+SELECT count(1) as count,num from lefttbl  group by lefttbl.num ORDER BY count desc;
    +
  2. Use concat(cast(round(rand() * 999999999) as string) to generate a random number for each key value.
  3. If the skew is serious and random numbers cannot be generated, see How Do I Eliminate Data Skew by Configuring AE Parameters?.
+
+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0200.html b/docs/dli/umn/dli_03_0200.html new file mode 100644 index 00000000..95fb131e --- /dev/null +++ b/docs/dli/umn/dli_03_0200.html @@ -0,0 +1,22 @@ + + +

SQL Jobs

+

Can I Create Temporary Tables on DLI?

A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in MySQL, you can use create temporary table... to create a temporary table. After a transaction or session ends, the table data is automatically deleted. Does DLI Support This Function?

+

Currently, you cannot create temporary tables on DLI. You can create a table by using SQL statements.

+
+

Can I Connect to DLI Locally? Is a Remote Connection Tool Supported?

Currently DLI can only be accessed through a browser. You must submit jobs on the console.

+
+

Will a DLI SQL Job Be Killed If It Has Been Running for More Than 12 Hours?

By default, SQL jobs that have been running for more than 12 hours will be canceled to ensure stability of queues.

+

You can use the dli.sql.job.timeout parameter (unit: second) to configure the timeout interval.

+
+

Does DLI Support Local Testing of Spark Jobs?

Currently, DLI does not support local testing of Spark jobs. You can install the DLI Livy tool and use its interactive sessions to debug Spark jobs.

+
+

Can I Delete a Row of Data from an OBS Table or DLI Table?

Deleting a row of data from an OBS table or DLI table is not allowed.

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0201.html b/docs/dli/umn/dli_03_0201.html new file mode 100644 index 00000000..31d5441a --- /dev/null +++ b/docs/dli/umn/dli_03_0201.html @@ -0,0 +1,22 @@ + + +

Spark Jobs

+

Does DLI Spark Support Scheduled Periodic Jobs?

DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.

+
+

Can I Define the Primary Key When I Create a Table with a Spark SQL Statement?

The Spark SQL syntax does not support primary key definition.

+
+

Can DLI Spark Jar Jobs Access GaussDB(DWS) Datasource Tables?

Yes.

+
+

How Do I Check the Version of the Spark Built-in Dependency Package?

DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar jobs.

+
+

Can I Download Packages on the Package Management Page?

No, the packages cannot be downloaded.

+
+

Which Path Should the Third-Party Dependency Jar File Be Stored for the Customized Spark 3.1.1 Image?

Store the third-party dependency in the /opt/spark/jars directory.

+
+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0207.html b/docs/dli/umn/dli_03_0207.html new file mode 100644 index 00000000..02642bc4 --- /dev/null +++ b/docs/dli/umn/dli_03_0207.html @@ -0,0 +1,14 @@ + + +

What Should I Do If "File not Found" Is Displayed When I Access a SQL Job?

+

Symptom

Error message "File not Found" is displayed when a SQL job is accessed.

+
+

Solution

Generally, the file cannot be found due to a read/write conflict. Check whether a job is overwriting the data when the error occurs.

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0208.html b/docs/dli/umn/dli_03_0208.html new file mode 100644 index 00000000..e79c51dc --- /dev/null +++ b/docs/dli/umn/dli_03_0208.html @@ -0,0 +1,14 @@ + + +

What Should I Do If "DLI.0003: AccessControlException XXX" Is Reported When I Access a SQL Job?

+

Symptom

Error message "DLI.0003: AccessControlException XXX" is reported when a SQL job is accessed.

+
+

Solution

View the OBS bucket in the AccessControlException and check whether you are using an account that has the permission to access the bucket.

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0209.html b/docs/dli/umn/dli_03_0209.html new file mode 100644 index 00000000..423c4fdf --- /dev/null +++ b/docs/dli/umn/dli_03_0209.html @@ -0,0 +1,14 @@ + + +

What Should I Do If "DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]" Is Reported When I Access a SQL Job?

+

Symptom

Error message "DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]" is reported when a SQL job is Accessed.

+
+

Solution

The current account does not have the permission to access the OBS bucket where the foreign table is located. Obtain the OBS permission and perform the query again.

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0210.html b/docs/dli/umn/dli_03_0210.html new file mode 100644 index 00000000..0c21aea1 --- /dev/null +++ b/docs/dli/umn/dli_03_0210.html @@ -0,0 +1,15 @@ + + +

What Should I Do If "The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget" Is Reported During SQL Statement Execution? Restricted for no budget.

+

Symptom

Error message "The current account does not have permission to perform this operation,the current account was restricted." is reported during SQL statement execution.

+
+

Solution

Check whether your account is in arrears. If it is, renew your account and try again.

+

If the error persists after renewal, log out and log in again.

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0212.html b/docs/dli/umn/dli_03_0212.html new file mode 100644 index 00000000..04d2fc0f --- /dev/null +++ b/docs/dli/umn/dli_03_0212.html @@ -0,0 +1,11 @@ + + +

Why Is All Data Overwritten When insert overwrite Is Used to Overwrite Partitioned Table?

+

To dynamically overwrite the specified partitioned data in the DataSource table, set dli.sql.dynamicPartitionOverwrite.enabled to true and then run the insert overwrite statement. (The default value of dli.sql.dynamicPartitionOverwrite.enabled is false.)

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0213.html b/docs/dli/umn/dli_03_0213.html new file mode 100644 index 00000000..962a399a --- /dev/null +++ b/docs/dli/umn/dli_03_0213.html @@ -0,0 +1,12 @@ + + +

Why Is a SQL Job Stuck in the Submitting State?

+

The possible causes and solutions are as follows:

+ +
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0214.html b/docs/dli/umn/dli_03_0214.html new file mode 100644 index 00000000..e77830be --- /dev/null +++ b/docs/dli/umn/dli_03_0214.html @@ -0,0 +1,14 @@ + + +

Why Is the create_date Field in the RDS Table Is a Timestamp in the DLI query result?

+

Spark does not have the datetime type and uses the TIMESTAMP type instead.

+

You can use a function to convert data types.

+

The following is an example.

+

select cast(create_date as string), * from table where create_date>'2221-12-01 00:00:00';

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0215.html b/docs/dli/umn/dli_03_0215.html new file mode 100644 index 00000000..c9c80f89 --- /dev/null +++ b/docs/dli/umn/dli_03_0215.html @@ -0,0 +1,12 @@ + + +

What Can I Do If datasize Cannot Be Changed After the Table Name Is Changed in a Finished SQL Job?

+

If the table name is changed immediately after SQL statements are executed, the data size of the table may be incorrect.

+

If you need to change the table name, change it 5 minutes after the SQL job is complete.

+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0220.html b/docs/dli/umn/dli_03_0220.html new file mode 100644 index 00000000..e64c7094 --- /dev/null +++ b/docs/dli/umn/dli_03_0220.html @@ -0,0 +1,16 @@ + + +

Why Is a Spark Jar Job Stuck in the Submitting State?

+

The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.

+

To view the remaining CUs of a queue, perform the following steps:

+
  1. Check the CU usage of the queue.

    Log in to the Cloud Eye console. In the navigation pane on the left, choose Cloud Service Monitoring > Data Lake Insight. On the displayed page, locate the desired queue and click View Metric in the Operation column, and check CU Usage (queue) on the displayed page.

    +
  1. Calculate the number of remaining CUs.

    Remaining CUs of a queue = Total CUs of the queue – CU usage.

    +
+

If the number of remaining CUs is less than the number of CUs required by the job, the job submission fails. The submission can be successful only after resources are available.

+
+
+
+
Parent topic: Problems Related to Spark Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0225.html b/docs/dli/umn/dli_03_0225.html new file mode 100644 index 00000000..8659f0fa --- /dev/null +++ b/docs/dli/umn/dli_03_0225.html @@ -0,0 +1,11 @@ + + +

How Do I Manage Queue Permissions?

+

You can isolate queues allocated to different users by setting permissions to ensure data query performance.

+
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0227.html b/docs/dli/umn/dli_03_0227.html new file mode 100644 index 00000000..9583e3a9 --- /dev/null +++ b/docs/dli/umn/dli_03_0227.html @@ -0,0 +1,16 @@ + + +

What Should I Do If "DLI.0003: Permission denied for resource..." Is Reported When I Run a SQL Statement?

+

Symptom

When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.

+

Error information: DLI.0003: Permission denied for resource 'databases.dli_test.tables.test.columns.col1', User = '{UserName}', Action = 'SELECT'

+
+

Solution

The user does not have the permission to query the table.

+

In the navigation pane on the left of the DLI console page, choose Data Management > Databases and Tables, search for the desired database table, view the permission configuration, and grant the table query permission to the user who requires it.

+
+
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0228.html b/docs/dli/umn/dli_03_0228.html new file mode 100644 index 00000000..60b338ee --- /dev/null +++ b/docs/dli/umn/dli_03_0228.html @@ -0,0 +1,13 @@ + + +

What Should I Do If the Table Permission Has Been Granted But the Table Still Cannot Be Queried?

+

The table permission has been granted and verified. However, after a period of time, an error is reported indicating that the table query fails.

+

There are two possible reasons:

+ +
+
+
+
Parent topic: Permission
+
+
+ diff --git a/docs/dli/umn/dli_03_0231.html b/docs/dli/umn/dli_03_0231.html new file mode 100644 index 00000000..00d4db3e --- /dev/null +++ b/docs/dli/umn/dli_03_0231.html @@ -0,0 +1,24 @@ + + +

Why Is the Data Volume Changes When Data Is Imported from DLI to OBS?

+

Symptom

When DLI is used to insert data into an OBS temporary table, only part of data is imported.

+
+

Possible Causes

Possible causes are as follows:

+ +

Run a query statement to check whether the amount of imported data is correct.

+

If OBS limits the number of files to be stored, add DISTRIBUTE BY number to the end of the insert statement. For example, if DISTRIBUTE BY 1 is added to the end of the insert statement, multiple files generated by multiple tasks can be inserted into one file.

+
+

Procedure

  1. On the DLI management console, check whether the number of results in the SQL job details is correct. The check result shows that the amount of data is correct.

    +

  2. Check whether the method to verify the data volume is correct. Perform the following steps to verify the data amount:

    1. Download the data file from OBS.
    2. Use the text editor to open the data file. The data volume is less than the expected volume.
    +

    If you used this method, you can verify that the text editor cannot read all the data.

    +

    Run the query statement to view the amount of data import into the OBS bucket. The query result indicates that all the data is imported.

    +

    This issue is caused by incorrect verification of the data volume.

    +

+
+
+
+
+
Parent topic: Problems Related to SQL Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0232.html b/docs/dli/umn/dli_03_0232.html new file mode 100644 index 00000000..526669d3 --- /dev/null +++ b/docs/dli/umn/dli_03_0232.html @@ -0,0 +1,18 @@ + + +

Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster

+

Symptom

After a Flink SQL job consumed Kafka and sank data to the Elasticsearch cluster, the job was successfully executed, but no data is available.

+
+

Cause Analysis

Possible causes are as follows:

+ +
+

Procedure

  1. Check the task log on the Flink UI. The JSON body is contained in the error message, indicating that the data format is incorrect.
  2. Check the data format. The Kafka data contains nested JSON bodies, which cannot be parsed.
  3. Use either of the following methods to solve the problem:

    • Create a JAR package with the UDF.
    • Modify the configuration data.
    +

  4. Change the data format and execute the job again.
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0233.html b/docs/dli/umn/dli_03_0233.html new file mode 100644 index 00000000..d3cb4ccd --- /dev/null +++ b/docs/dli/umn/dli_03_0233.html @@ -0,0 +1,17 @@ + + +

What Should I Do If Invalid OBS Bucket Name Is Reported After a Flink Job Submission Failed?

+

Symptom

The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was displayed.

+

+
+

Cause Analysis

  1. Check that the OBS bucket name is correct.
  2. Check that the AK/SK has the required permission.
  3. Set the dependency to provided to prevent JAR package conflicts.
  4. Check that the esdk-obs-java-3.1.3.jar version is used.
  5. Confirm that the cluster configuration is faulty.
+
+

Procedure

  1. Set the dependency to provided.
  2. Restart the clusteragent cluster after an upgrade to make the configuration take effect.
  3. Remove the OBS dependency. Otherwise, the checkpoints cannot be written to OBS.
+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0234.html b/docs/dli/umn/dli_03_0234.html new file mode 100644 index 00000000..14c6535d --- /dev/null +++ b/docs/dli/umn/dli_03_0234.html @@ -0,0 +1,34 @@ + + +

Flink Job Submission Fails Due to Hadoop JAR Package Conflicts

+

Symptom

Flink Job submission failed. The exception information is as follows:

+
Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.obs.metrics.OBSAMetricsProvider not found
+ at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2664)
+ at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2688)
+ ... 31 common frames omitted
+ Caused by: java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.obs.metrics.OBSAMetricsProvider not found
+ at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2568)
+ at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2662)
+ ... 32 common frames omitted
+
+

Cause Analysis

Flink JAR packages conflicted. The submitted Flink JAR package conflicted with the HDFS JAR package of the DLI cluster.

+
+

Procedure

  1. Configure hadoop-hdfs in the POM file as follows:

     <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope> provided </scope>
+ </dependency>
    +

    Alternatively, use the exclusions tag to exclude the association.

    +

  2. To use HDFS configuration files, change core-site.xml, hdfs-site.xml, and yarn-site.xml to mrs-core-site.xml, mrs-hdfs-site.xml and mrs-hbase-site.xml, respectively.

     conf.addResource(HBaseUtil.class.getClassLoader().getResourceAsStream("mrs-core-site.xml"), false);
+ conf.addResource(HBaseUtil.class.getClassLoader().getResourceAsStream("mrs-hdfs-site.xml"), false);
+ conf.addResource(HBaseUtil.class.getClassLoader().getResourceAsStream("mrs-hbase-site.xml"), false);
    +

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0235.html b/docs/dli/umn/dli_03_0235.html new file mode 100644 index 00000000..d6311664 --- /dev/null +++ b/docs/dli/umn/dli_03_0235.html @@ -0,0 +1,16 @@ + + +

What Should I Do If the Flink/Spark UI Page Cannot Be Displayed Properly?

+

Symptom

The Flink/Spark UI was displayed with incomplete information.

+
+

Possible Causes

When a pay-per-use queue is used to run a job, the system releases the cluster and takes about 10 minutes to create a new one. Accessing the Flink UI before completion of the creation will empty the project ID in the cache. As a result, the UI cannot be displayed. The possible cause is that the cluster was not created.

+
+

Solution

Change the queue to dedicated, so that the cluster will not be released when the queue is idle. Alternatively, submit a job, wait for a while, and then access FlinkUI.

+
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0236.html b/docs/dli/umn/dli_03_0236.html new file mode 100644 index 00000000..0dda9c50 --- /dev/null +++ b/docs/dli/umn/dli_03_0236.html @@ -0,0 +1,19 @@ + + +

Wha should I Do If JobManager and TaskManager Heartbeats Timed Out and the Flink Job Is Abnormal?

+

Symptom

JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.

+
Figure 1 Error information
+
+

Possible Causes

  1. Check whether the network is intermittently disconnected and whether the cluster load is high.
  2. If Full GC occurs frequently, check the code to determine whether memory leakage occurs.
    Figure 2 Full GC
    +
+
+

Handling Procedure

+ +
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0237.html b/docs/dli/umn/dli_03_0237.html new file mode 100644 index 00000000..7d8d5960 --- /dev/null +++ b/docs/dli/umn/dli_03_0237.html @@ -0,0 +1,18 @@ + + +

Failed to Bind a Queue to an Enhanced Datasource Connection

+

Symptom

An enhanced datasource connection failed to pass the network connectivity test. Datasource connection cannot be bound to a queue. The following error information is displayed:

+
Failed to get subnet 86ddcf50-233a-449d-9811-cfef2f603213. Response code : 404, message : {"code":"VPC.0202","message":"Query resource by id 86ddcf50-233a-449d-9811-cfef2f603213 fail.the subnet could not be found."}
+
+

Cause Analysis

VPC Administrator permissions are required to use the VPC, subnet, route, VPC peering connection, and port for DLI datasource connections.

+

The binding fails because the user does not have the required VPC permissions.

+
+

Procedure

On the DLI console, choose Global Configuration > Service Authorization, select the required VPC permission, and click Update.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0238.html b/docs/dli/umn/dli_03_0238.html new file mode 100644 index 00000000..e0b27a2a --- /dev/null +++ b/docs/dli/umn/dli_03_0238.html @@ -0,0 +1,20 @@ + + +

DLI Failed to Connect to GaussDB(DWS) Through an Enhanced Datasource Connection

+

Symptom

The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a password. The connection failed and DLI.0999: PSQLException: The connection attempt failed is reported.

+
+

Cause Analysis

Possible causes are as follows:

+ +
+

Procedure

  1. Check whether the security group is accessible.

    • Inbound rule: Check whether the inbound CIDR block and port in the security group have been enabled. If not, create the CIDR block and port you need.
    • Outbound rule: Check whether the CIDR block and port of the outbound rule are enabled. (It is recommended that all CIDR blocks be enabled.)
    +

    Both the inbound and outbound rules of the security group are configured for the subnets of the DLI queue. Set the source IP address in the inbound direction to 0.0.0.0/0 and port 8000, indicating that any IP address can access port 8000.

    +

  2. If the fault persists. Check the subnet configuration. Check the network ACL associated with the GaussDB(DWS) subnet. A network ACL is an optional layer of security for your subnets. You can associate one or more subnets with a network ACL to control traffic in and out of the subnets. After the association, the network ACL denies all traffic to and from the subnet by default until you add rules to allow traffic. The check result showed that the ACL associated with the subnet where GaussDB(DWS) resides is empty.

    A network ACL is associated and no inbound or outbound rules are configured. As a result, the IP address cannot be accessed.

    +

  3. Perform the connectivity test. After the subnet inbound and outbound rules are configured, the datasource connection passes the connectivity test.
+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0239.html b/docs/dli/umn/dli_03_0239.html new file mode 100644 index 00000000..5338e897 --- /dev/null +++ b/docs/dli/umn/dli_03_0239.html @@ -0,0 +1,22 @@ + + +

Error Message "Incorrect string value" Is Displayed When insert overwrite Is Executed on a Cross-Source RDS Table

+

Symptom

A cross-source RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incorrect string value: '\xF0\x9F\x90\xB3' for column 'robot_name' at row 1 was reported.

+
+

Cause Analysis

The data to be written contains emojis, which are encoded in the unit of four bytes. MySQL databases use the UTF-8 format, which encodes data in the unit of three bytes by default. In this case, an error occurs when the emoji data is inserted into to the MySQL database.

+

Possible causes are as follows:

+ +
+

Procedure

Change the character set to utf8mb4.

+
  1. Run the following SQL statement to change the database character set:

    ALTER DATABASE DATABASE_NAME DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
    +

  2. Run the following SQL statement to change the table character set:

    ALTER TABLE TABLE_NAME DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
    +

  3. Run the following SQL statement to change the character set for all fields in the table:

    ALTER TABLE TABLE_NAME CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
    +

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0250.html b/docs/dli/umn/dli_03_0250.html new file mode 100644 index 00000000..2c0c8e01 --- /dev/null +++ b/docs/dli/umn/dli_03_0250.html @@ -0,0 +1,24 @@ + + +

Null Pointer Error Is Displayed When the System Creates a Cross-Source RDS Table

+

Symptom

The system failed to create a cross-source RDS table, and null pointer error was reported.

+
+

Cause Analysis

The following table creation statement was used:

+
CREATE TABLE IF NOT EXISTS dli_to_rds
+ USING JDBC OPTIONS (
+ 'url'='jdbc:mysql://to-rds-1174405119-oLRHAGE7.datasource.com:3306/postgreDB',
+ 'driver'='org.postgresql.Driver',
+ 'dbtable'='pg_schema.test1',
+ 'passwdauth' = 'xxx',
+ 'encryption' = 'true');
+

The RDS database is in a PostGre cluster, and the protocol header in the URL is invalid.

+
+

Procedure

Change the URL to url'='jdbc:postgresql://to-rds-1174405119-oLRHAGE7.datasource.com:3306/postgreDB and run the creation statement. The cross-source table is successfully created.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0251.html b/docs/dli/umn/dli_03_0251.html new file mode 100644 index 00000000..165d5d1a --- /dev/null +++ b/docs/dli/umn/dli_03_0251.html @@ -0,0 +1,21 @@ + + +

Error Message "org.postgresql.util.PSQLException: ERROR: tuple concurrently updated" Is Displayed When the System Executes insert overwrite on a Cross-Source GaussDB(DWS) Table

+

Symptom

The system failed to execute insert overwrite on the cross-source GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was displayed.

+
+

Cause Analysis

Concurrent operations existed in the job. Two insert overwrite operations were executed on the table at the same time.

+

One CN was running the following statement:

+
TRUNCATE TABLE BI_MONITOR.SAA_OUTBOUND_ORDER_CUST_SUM
+

Another CN was running the following command:

+
call bi_monitor.pkg_saa_out_bound_monitor_p_saa_outbound_order_cust_sum
+

This function deletes and inserts SAA_OUTBOUND_ORDER_CUST_SUM.

+
+

Procedure

Modify job logic to prevent concurrent insert overwrite operations on the same table.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0252.html b/docs/dli/umn/dli_03_0252.html new file mode 100644 index 00000000..90918292 --- /dev/null +++ b/docs/dli/umn/dli_03_0252.html @@ -0,0 +1,19 @@ + + +

RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Cross-Source Table

+

Symptom

A cross-source table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. The data volume is 9.76 GB. The job failed after 10 million data records were imported.

+
+

Cause Analysis

  1. View driver error logs.
  2. View executor error logs.
  3. View task error logs.
+

The rowkey was poorly designed causing a large amount of traffic redirected to single or very few numbers of nodes.

+
+

Procedure

  1. Pre-partition the HBase.
  2. Hash the rowkey.
+
+

Summary and Suggestions

Distribute data to different RegionServer. Add distribute by rand() to the end of the insert statement.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0253.html b/docs/dli/umn/dli_03_0253.html new file mode 100644 index 00000000..9e28195e --- /dev/null +++ b/docs/dli/umn/dli_03_0253.html @@ -0,0 +1,21 @@ + + +

A Null Value Is Written Into a Non-Null Field When a DLI Datasource Connection Is Used to Connect to a GaussDB(DWS) Table

+

Symptom

A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indicating that DLI was writing a null value to a non-null field of the table, and the job failed.

+
The error message is as follows:
DLI.0999: PSQLException: ERROR: dn_6009_6010: null value in column "ctr" violates not-null constraint
+Detail: Failing row contains (400070309, 9.00, 25, null, 2020-09-22, 2020-09-23 04:30:01.741).
+
+
+

Cause Analysis

  1. The CIR field in the source table is of the DOUBLE type.
    Figure 1 Creation statement of the source table
    +
  2. The field type in the target table is DECIMAL(9,6).
    Figure 2 Creation statement of the target table
    +
  3. View the source table data. The CTR value that causes the problem is 1675, which exceed the precision (9 – 6 = 3 digits) of the DECIMAL(9,6) type. A null value was generated when the double value was converted to the decimal value, and the insert operation failed.
+
+

Procedure

Change the precision of the decimal data defined in the target table.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0254.html b/docs/dli/umn/dli_03_0254.html new file mode 100644 index 00000000..7218bd73 --- /dev/null +++ b/docs/dli/umn/dli_03_0254.html @@ -0,0 +1,18 @@ + + +

An Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated

+

Symptom

A cross-source GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job execution, the schema of the source table failed to be updated, and the job failed.

+
+

Cause Analysis

When the insert operation is executed on the DLI cross-source table, the GaussDB(DWS) source table is deleted and recreated. If the statement for creating the cross-source table is not updated on DLI, the GaussDB(DWS) source table will fail to be updated.

+
+

Procedure

Create a cross-source table on DLI and add table creation configuration truncate = true to clear table data but not delete the table.

+
+

Summary and Suggestions

After the source table is updated, the corresponding cross-source table must be updated too on DLI.

+
+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0257.html b/docs/dli/umn/dli_03_0257.html new file mode 100644 index 00000000..0d252f32 --- /dev/null +++ b/docs/dli/umn/dli_03_0257.html @@ -0,0 +1,12 @@ + + +

What Can I Do If a Datasource Connection Is Stuck in Creating State When I Try to Bind a Queue to It?

+

The possible causes and solutions are as follows:

+ +
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0258.html b/docs/dli/umn/dli_03_0258.html new file mode 100644 index 00000000..036c182b --- /dev/null +++ b/docs/dli/umn/dli_03_0258.html @@ -0,0 +1,12 @@ + + +

How Do I Bind a Datasource Connection to a Pay-per-Use Queue?

+

Enhanced datasource connections support only yearly/monthly and pay-per-use queues.

+

You can bind a datasource connection only when you select Dedicated Resource Mode when purchasing a pay-per-use queue. The resources are dedicated and used on demand (the billing period is equal to the lifecycle of the queue).

+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0259.html b/docs/dli/umn/dli_03_0259.html new file mode 100644 index 00000000..619f1243 --- /dev/null +++ b/docs/dli/umn/dli_03_0259.html @@ -0,0 +1,11 @@ + + +

How Do I Connect DLI to Data Sources?

+

DLI enhanced datasource connection uses VPC peering to directly connect the VPC networks of the desired data sources for point-to-point data exchanges.

+
+
+
+
Parent topic: Datasource Connections
+
+
+ diff --git a/docs/dli/umn/dli_03_0260.html b/docs/dli/umn/dli_03_0260.html new file mode 100644 index 00000000..512f49c7 --- /dev/null +++ b/docs/dli/umn/dli_03_0260.html @@ -0,0 +1,11 @@ + + +

Does DLI Have the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?

+

No. The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.

+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0263.html b/docs/dli/umn/dli_03_0263.html new file mode 100644 index 00000000..0ec4b798 --- /dev/null +++ b/docs/dli/umn/dli_03_0263.html @@ -0,0 +1,13 @@ + + +

Can a Member Account Use Global Variables Created by Other Member Accounts?

+

No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables can be replaced to improve the readability of SQL statements.

+

The restrictions on using global variables are as follows:

+ +
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0264.html b/docs/dli/umn/dli_03_0264.html new file mode 100644 index 00000000..bc85b2fc --- /dev/null +++ b/docs/dli/umn/dli_03_0264.html @@ -0,0 +1,21 @@ + + +

What Should I do If the System Failed to Create a Database and {"error_code":"DLI.1028";"error_msg":"Already reached the maximum quota of databases:XXX" Is Displayed Indicating that the Quota Is Insufficient?

+

Viewing My Quotas

  1. Log in to the management console.
  2. Click in the upper left corner and select Region and Project.
  3. Click (the My Quotas icon) in the upper right corner.

    The Service Quota page is displayed.

    +
  4. View the used and total quota of each type of resources on the displayed page.

    If a quota cannot meet service requirements, increase a quota.

    +
+
+

How Do I Apply for a Higher Quota?

The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your application and inform you of the progress by phone call or email.

+

Before dialing the hotline number or sending an email, ensure that the following information has been obtained:

+ +

Learn how to obtain the service hotline and email address.

+
+
+
+
+
Parent topic: Product Consultation
+
+
+ diff --git a/docs/dli/umn/dli_03_0265.html b/docs/dli/umn/dli_03_0265.html new file mode 100644 index 00000000..655371c9 --- /dev/null +++ b/docs/dli/umn/dli_03_0265.html @@ -0,0 +1,11 @@ + + +

What Can I Do If "Timeout expired while fetching topic metadata" Is Repeatedly Reported in Flink JobManager Logs?

+
  1. Test address connectivity.
  2. If the network is unreachable, rectify the network connection first. Ensure that the network connection between the DLI queue and the external data source is normal.
+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_03_0266.html b/docs/dli/umn/dli_03_0266.html new file mode 100644 index 00000000..c1eb1748 --- /dev/null +++ b/docs/dli/umn/dli_03_0266.html @@ -0,0 +1,11 @@ + + +

How Do I Connect a Flink jar Job to SASL_SSL?

+

You can use Flink Jar to connect to Kafka with SASL SSL authentication enabled.

+
+
+
+
Parent topic: Flink Jobs
+
+
+ diff --git a/docs/dli/umn/dli_07_0002.html b/docs/dli/umn/dli_07_0002.html new file mode 100644 index 00000000..ed097000 --- /dev/null +++ b/docs/dli/umn/dli_07_0002.html @@ -0,0 +1,25 @@ + + +

Application Scenarios

+

DLI is applicable to large-scale log analysis, federated analysis of heterogeneous data sources, and big data ETL processing.

+

Large-scale Log Analysis

+ +
+

Federated Analysis of Heterogeneous Data Sources

+
+

Big Data ETL

+ +
+
+
+
+
Parent topic: Service Overview
+
+
+ diff --git a/docs/dli/umn/dli_07_0003.html b/docs/dli/umn/dli_07_0003.html new file mode 100644 index 00000000..c0679ce1 --- /dev/null +++ b/docs/dli/umn/dli_07_0003.html @@ -0,0 +1,34 @@ + + +

Basic Concepts

+

Tenant

DLI allows multiple organizations, departments, or applications to share resources. A logical entity, also called a tenant, is provided to use diverse resources and services. A mode involving different tenants is called multi-tenant mode. A tenant corresponds to a company. Multiple sub-users can be created under a tenant and are assigned different permissions.

+
+

Project

A project is a collection of resources accessible to services. In a region, an account can create multiple projects and assign different permissions to different projects. Resources used for different projects are isolated from one another. A project can either be a department or a project team.

+
+

Database

A database is a warehouse where data is organized, stored, and managed based on the data structure. DLI management permissions are granted on a per database basis.

+

In DLI, tables and databases are metadata containers that define underlying data. The metadata in the table shows the location of the data and specifies the data structure, such as the column name, data type, and table name. A database is a collection of tables.

+
+

Metadata

Metadata is used to define data types. It describes information about the data, including the source, size, format, and other data features. In database fields, metadata interprets data content in the data warehouse.

+
+

Computing Resource

Queues in DLI are computing resources, which are the basis for using DLI. SQL jobs and Spark jobs performed by users require computing resources.

+
+

Storage Resource

Storage resources in DLI are used to store data of databases and DLI tables. To import data to DLI, storage resources must be prepared. The storage resources reflect the volume of data you are allowed to store in DLI.

+
+

SQL Job

SQL job refers to the SQL statement executed in the SQL job editor. It serves as the execution entity used for performing operations, such as importing and exporting data, in the SQL job editor.

+
+

Spark Job

Spark jobs are those submitted by users through visualized interfaces and RESTful APIs. Full-stack Spark jobs are allowed, such as Spark Core, DataSet, MLlib, and GraphX jobs.

+
+

OBS Table, DLI Table, and CloudTable Table

The table type indicates the storage location of data.

+ +

You can create a table on DLI and associate the table with other services to achieve querying data from multiple data sources.

+
+

Constants and Variables

The differences between constants and variables are as follows:

+ +
+
+
+
+
Parent topic: Service Overview
+
+
+ diff --git a/docs/dli/umn/dli_07_0005.html b/docs/dli/umn/dli_07_0005.html new file mode 100644 index 00000000..f910393a --- /dev/null +++ b/docs/dli/umn/dli_07_0005.html @@ -0,0 +1,15 @@ + + +

Constraints and Limitations on Using DLI

+

When using DLI, you may need to make adjustments according to the following restrictions:

+ +
+
+
+
Parent topic: Service Overview
+
+
+ diff --git a/docs/dli/umn/dli_07_0006.html b/docs/dli/umn/dli_07_0006.html new file mode 100644 index 00000000..4407e988 --- /dev/null +++ b/docs/dli/umn/dli_07_0006.html @@ -0,0 +1,647 @@ + + +

Permissions Management

+

If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IAM provides identity authentication, permissions management, and access control, helping you securely access your resources.

+

With IAM, you can use your account to create IAM users for your employees, and assign permissions to the users to control their access to specific resource types. For example, some software developers in your enterprise need to use DLI resources but must not delete them or perform any high-risk operations. To achieve this result, you can create IAM users for the software developers and grant them only the permissions required for using DLI resources.

+

If the account has met your requirements, you do not need to create an independent IAM user for permission management. Then you can skip this section. This will not affect other functions of DLI.

+

DLI Permissions

By default, new IAM users do not have permissions assigned. You need to add the users to one or more groups, and attach permissions policies or roles to these groups. The users then inherit permissions from the groups to which they are added. After authorization, the users can perform specified operations on DLI based on the permissions.

+

DLI is a project-level service deployed and accessed in specific physical regions. To assign ServiceStage permissions to a user group, specify the scope as region-specific projects and select projects for the permissions to take effect. If All projects is selected, the permissions will take effect for the user group in all region-specific projects. When accessing DLI, the users need to switch to a region where they have been authorized to use cloud services.

+
Type: There are roles and policies.
  • Roles: A type of coarse-grained authorization mechanism that defines permissions related to user responsibilities. This mechanism provides only a limited number of service-level roles for authorization. If one role has a dependency role required for accessing SA, assign both roles to the users. However, roles are not an ideal choice for fine-grained authorization and secure access control.
  • Policies: A type of fine-grained authorization mechanism that defines permissions required to perform operations on specific cloud resources under certain conditions. This mechanism allows for more flexible policy-based authorization, meeting requirements for secure access control. For example, you can grant DLI users only the permissions for managing a certain type of cloud servers. For the actions supported by DLI APIs, see "Permissions Policies and Supported Actions" in the Data Lake Insight API Reference.
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 DLI system permissions

Role/Policy Name

+

Description

+

Policy Type

+

DLI FullAccess

+

Full permissions for DLI.

+

System-Defined Policies

+

DLI ReadOnlyAccess

+

Read-only permissions for DLI.

+

With read-only permissions, you can use DLI resources and perform operations that do not require fine-grained permissions. For example, create global variables, create packages and package groups, submit jobs to the default queue, create tables in the default database, create datasource connections, and delete datasource connections.

+

System-Defined Policies

+

Tenant Administrator

+

Tenant administrator

+
  • Job execution permissions for DLI resources. After a database or a queue is created, the user can use the ACL to assign rights to other users.
  • Function scope: Project-level service.
+

System-defined role

+

DLI Service Admin

+

DLI administrator.

+
  • Job execution permissions for DLI resources. After a database or a queue is created, the user can use the ACL to assign rights to other users.
  • Function scope: Project-level service.
+

System-defined role

+
+
+

Table 2 lists the common SQL operations supported by each system policy of DLI. Choose proper system policies according to this table.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 Common operations supported by each system policy

Resources

+

Operation

+

Description

+

DLI FullAccess

+

DLI ReadOnlyAccess

+

Tenant Administrator

+

DLI Service Admin

+

Queue

+

DROP_QUEUE

+

Deleting a queue

+

√

+

×

+

√

+

√

+

SUBMIT_JOB

+

Submitting the job

+

√

+

×

+

√

+

√

+

CANCEL_JOB

+

Terminating the job

+

√

+

×

+

√

+

√

+

RESTART

+

Restarting a queue

+

√

+

×

+

√

+

√

+

GRANT_PRIVILEGE

+

Granting permissions to the queue

+

√

+

×

+

√

+

√

+

REVOKE_PRIVILEGE

+

Revoking permissions from the queue

+

√

+

×

+

√

+

√

+

SHOW_PRIVILEGES

+

Viewing the queue permissions of other users

+

√

+

×

+

√

+

√

+

Database

+

DROP_DATABASE

+

Deleting a database

+

√

+

×

+

√

+

√

+

CREATE_TABLE

+

Creating a table

+

√

+

×

+

√

+

√

+

CREATE_VIEW

+

Creating a view

+

√

+

×

+

√

+

√

+

EXPLAIN

+

Explaining the SQL statement as an execution plan

+

√

+

×

+

√

+

√

+

CREATE_ROLE

+

Creating a role

+

√

+

×

+

√

+

√

+

DROP_ROLE

+

Deleting a role

+

√

+

×

+

√

+

√

+

SHOW_ROLES

+

Displaying a role

+

√

+

×

+

√

+

√

+

GRANT_ROLE

+

Binding a role

+

√

+

×

+

√

+

√

+

REVOKE_ROLE

+

Unbinding the role

+

√

+

×

+

√

+

√

+

SHOW_USERS

+

Displaying the binding relationships between all roles and users

+

√

+

×

+

√

+

√

+

GRANT_PRIVILEGE

+

Granting permissions to the database

+

√

+

×

+

√

+

√

+

REVOKE_PRIVILEGE

+

Revoking permissions to the database

+

√

+

×

+

√

+

√

+

SHOW_PRIVILEGES

+

Viewing database permissions of other users

+

√

+

×

+

√

+

√

+

DISPLAY_ALL_TABLES

+

Displaying tables in the database

+

√

+

√

+

√

+

√

+

DISPLAY_DATABASE

+

Displaying databases

+

√

+

√

+

√

+

√

+

CREATE_FUNCTION

+

Creating a function

+

√

+

×

+

√

+

√

+

DROP_FUNCTION

+

Deleting a function

+

√

+

×

+

√

+

√

+

SHOW_FUNCTIONS

+

Displaying all functions

+

√

+

×

+

√

+

√

+

DESCRIBE_FUNCTION

+

Displaying function details

+

√

+

×

+

√

+

√

+

Table

+

DROP_TABLE

+

Deleting a table

+

√

+

×

+

√

+

√

+

SELECT

+

Querying a table

+

√

+

×

+

√

+

√

+

INSERT_INTO_TABLE

+

Inserting

+

√

+

×

+

√

+

√

+

ALTER_TABLE_ADD_COLUMNS

+

Adding a column

+

√

+

×

+

√

+

√

+

INSERT_OVERWRITE_TABLE

+

Rewriting

+

√

+

×

+

√

+

√

+

ALTER_TABLE_RENAME

+

Renaming a table

+

√

+

×

+

√

+

√

+

ALTER_TABLE_ADD_PARTITION

+

Adding partitions to the partition table

+

√

+

×

+

√

+

√

+

ALTER_TABLE_RENAME_PARTITION

+

Renaming a table partition

+

√

+

×

+

√

+

√

+

ALTER_TABLE_DROP_PARTITION

+

Deleting partitions from a partition table

+

√

+

×

+

√

+

√

+

SHOW_PARTITIONS

+

Displaying all partitions

+

√

+

×

+

√

+

√

+

ALTER_TABLE_RECOVER_PARTITION

+

Restoring table partitions

+

√

+

×

+

√

+

√

+

ALTER_TABLE_SET_LOCATION

+

Setting the partition path

+

√

+

×

+

√

+

√

+

GRANT_PRIVILEGE

+

Granting permissions to the table

+

√

+

×

+

√

+

√

+

REVOKE_PRIVILEGE

+

Revoking permissions from the table

+

√

+

×

+

√

+

√

+

SHOW_PRIVILEGES

+

Viewing table permissions of other users

+

√

+

×

+

√

+

√

+

DISPLAY_TABLE

+

Displaying a table

+

√

+

√

+

√

+

√

+

DESCRIBE_TABLE

+

Displaying table information

+

√

+

×

+

√

+

√

+
+
+
+
+
+
+
Parent topic: Service Overview
+
+
+ diff --git a/docs/dli/umn/dli_07_0007.html b/docs/dli/umn/dli_07_0007.html new file mode 100644 index 00000000..250da120 --- /dev/null +++ b/docs/dli/umn/dli_07_0007.html @@ -0,0 +1,18 @@ + + +

Advantages

+

Full SQL compatibility

You do not need a background in big data to use DLI for data analysis. You only need to know SQL, and you are good to go. The SQL syntax is fully compatible with the standard ANSI SQL 2003. The SQL syntax is fully compatible with the standard ANSI SQL 2003.

+
+

Storage-Compute Decoupling

DLI compute and storage loads are decoupled. This architecture allows you to flexibly configure storage and compute resources on demand, improving resource utilization and reducing costs.

+
+

Serverless DLI

DLI is fully compatible with Apache Spark and Apache Flink ecosystems and APIs. It is a serverless big data computing and analysis service that integrates real-time, offline, and interactive analysis. Offline applications can be seamlessly migrated to the cloud, reducing the migration workload. DLI provides a highly-scalable framework integrating batch and stream processing, allowing you to handle data analysis requests with ease. With a deeply optimized kernel and architecture, DLI delivers 100-fold performance improvement compared with the MapReduce model. Your analysis is backed by an industry-vetted 99.95% SLA.

+
+

Cross-Source Analysis

Analyze your data across databases. No migration required. A unified view of your data gives you a comprehensive understanding of your data and helps you innovate faster. There are no restrictions on data formats, cloud data sources, or whether the database is created online or off.

+
+
+
+
+
Parent topic: Service Overview
+
+
+ diff --git a/docs/dli/umn/en-us_image_0000001078931615.png b/docs/dli/umn/en-us_image_0000001078931615.png new file mode 100644 index 00000000..cc45c153 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001078931615.png differ diff --git a/docs/dli/umn/en-us_image_0000001103772632.png b/docs/dli/umn/en-us_image_0000001103772632.png new file mode 100644 index 00000000..0a1d49fc Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001103772632.png differ diff --git a/docs/dli/umn/en-us_image_0000001103931700.png b/docs/dli/umn/en-us_image_0000001103931700.png new file mode 100644 index 00000000..7e783de5 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001103931700.png differ diff --git a/docs/dli/umn/en-us_image_0000001104091730.png b/docs/dli/umn/en-us_image_0000001104091730.png new file mode 100644 index 00000000..e7f7f6e5 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001104091730.png differ diff --git a/docs/dli/umn/en-us_image_0000001105517798.png b/docs/dli/umn/en-us_image_0000001105517798.png new file mode 100644 index 00000000..50ccdc02 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001105517798.png differ diff --git a/docs/dli/umn/en-us_image_0000001115396324.png b/docs/dli/umn/en-us_image_0000001115396324.png new file mode 100644 index 00000000..a479b8d0 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001115396324.png differ diff --git a/docs/dli/umn/en-us_image_0000001115399366.png b/docs/dli/umn/en-us_image_0000001115399366.png new file mode 100644 index 00000000..2f337b2a Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001115399366.png differ diff --git a/docs/dli/umn/en-us_image_0000001142261092.png b/docs/dli/umn/en-us_image_0000001142261092.png new file mode 100644 index 00000000..4ac27c0b Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001142261092.png differ diff --git a/docs/dli/umn/en-us_image_0000001150931533.png b/docs/dli/umn/en-us_image_0000001150931533.png new file mode 100644 index 00000000..d5932622 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001150931533.png differ diff --git a/docs/dli/umn/en-us_image_0000001151957753.png b/docs/dli/umn/en-us_image_0000001151957753.png new file mode 100644 index 00000000..50ccdc02 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001151957753.png differ diff --git a/docs/dli/umn/en-us_image_0000001161715923.png b/docs/dli/umn/en-us_image_0000001161715923.png new file mode 100644 index 00000000..2c978d32 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001161715923.png differ diff --git a/docs/dli/umn/en-us_image_0000001161719815.png b/docs/dli/umn/en-us_image_0000001161719815.png new file mode 100644 index 00000000..26c0d6ac Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001161719815.png differ diff --git a/docs/dli/umn/en-us_image_0000001161799601.png b/docs/dli/umn/en-us_image_0000001161799601.png new file mode 100644 index 00000000..ace680e2 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001161799601.png differ diff --git a/docs/dli/umn/en-us_image_0000001174159614.png b/docs/dli/umn/en-us_image_0000001174159614.png new file mode 100644 index 00000000..74aa05c9 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001174159614.png differ diff --git a/docs/dli/umn/en-us_image_0000001200929158.png b/docs/dli/umn/en-us_image_0000001200929158.png new file mode 100644 index 00000000..e01c6ad1 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001200929158.png differ diff --git a/docs/dli/umn/en-us_image_0000001201409084.png b/docs/dli/umn/en-us_image_0000001201409084.png new file mode 100644 index 00000000..dfb9821a Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001201409084.png differ diff --git a/docs/dli/umn/en-us_image_0000001209329768.png b/docs/dli/umn/en-us_image_0000001209329768.png new file mode 100644 index 00000000..aa202e23 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001209329768.png differ diff --git a/docs/dli/umn/en-us_image_0000001209489750.png b/docs/dli/umn/en-us_image_0000001209489750.png new file mode 100644 index 00000000..33ed35c3 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001209489750.png differ diff --git a/docs/dli/umn/en-us_image_0000001209516352.png b/docs/dli/umn/en-us_image_0000001209516352.png new file mode 100644 index 00000000..33ed35c3 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001209516352.png differ diff --git a/docs/dli/umn/en-us_image_0000001209640030.png b/docs/dli/umn/en-us_image_0000001209640030.png new file mode 100644 index 00000000..ae39a88d Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001209640030.png differ diff --git a/docs/dli/umn/en-us_image_0000001209671182.png b/docs/dli/umn/en-us_image_0000001209671182.png new file mode 100644 index 00000000..345672e4 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001209671182.png differ diff --git a/docs/dli/umn/en-us_image_0000001220037927.png b/docs/dli/umn/en-us_image_0000001220037927.png new file mode 100644 index 00000000..57e5e0b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001220037927.png differ diff --git a/docs/dli/umn/en-us_image_0000001254369651.png b/docs/dli/umn/en-us_image_0000001254369651.png new file mode 100644 index 00000000..aa202e23 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001254369651.png differ diff --git a/docs/dli/umn/en-us_image_0000001254569529.png b/docs/dli/umn/en-us_image_0000001254569529.png new file mode 100644 index 00000000..8d349c36 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001254569529.png differ diff --git a/docs/dli/umn/en-us_image_0000001254569657.png b/docs/dli/umn/en-us_image_0000001254569657.png new file mode 100644 index 00000000..33ed35c3 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001254569657.png differ diff --git a/docs/dli/umn/en-us_image_0000001259009999.png b/docs/dli/umn/en-us_image_0000001259009999.png new file mode 100644 index 00000000..d0b26db3 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001259009999.png differ diff --git a/docs/dli/umn/en-us_image_0000001265889586.png b/docs/dli/umn/en-us_image_0000001265889586.png new file mode 100644 index 00000000..aa202e23 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001265889586.png differ diff --git a/docs/dli/umn/en-us_image_0000001299472334.png b/docs/dli/umn/en-us_image_0000001299472334.png new file mode 100644 index 00000000..5a0e5a9f Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001299472334.png differ diff --git a/docs/dli/umn/en-us_image_0000001299475238.png b/docs/dli/umn/en-us_image_0000001299475238.png new file mode 100644 index 00000000..4025d020 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001299475238.png differ diff --git a/docs/dli/umn/en-us_image_0000001299478654.png b/docs/dli/umn/en-us_image_0000001299478654.png new file mode 100644 index 00000000..5a0e5a9f Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001299478654.png differ diff --git a/docs/dli/umn/en-us_image_0000001299635390.png b/docs/dli/umn/en-us_image_0000001299635390.png new file mode 100644 index 00000000..025feec8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001299635390.png differ diff --git a/docs/dli/umn/en-us_image_0000001299958066.png b/docs/dli/umn/en-us_image_0000001299958066.png new file mode 100644 index 00000000..8f27557f Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001299958066.png differ diff --git a/docs/dli/umn/en-us_image_0000001310151968.png b/docs/dli/umn/en-us_image_0000001310151968.png new file mode 100644 index 00000000..cc45c153 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001310151968.png differ diff --git a/docs/dli/umn/en-us_image_0000001352514025.png b/docs/dli/umn/en-us_image_0000001352514025.png new file mode 100644 index 00000000..1d95e121 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001352514025.png differ diff --git a/docs/dli/umn/en-us_image_0000001372601066.png b/docs/dli/umn/en-us_image_0000001372601066.png new file mode 100644 index 00000000..7e7358c2 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001372601066.png differ diff --git a/docs/dli/umn/en-us_image_0000001372847466.png b/docs/dli/umn/en-us_image_0000001372847466.png new file mode 100644 index 00000000..35e28315 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001372847466.png differ diff --git a/docs/dli/umn/en-us_image_0000001373007370.png b/docs/dli/umn/en-us_image_0000001373007370.png new file mode 100644 index 00000000..1909444d Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001373007370.png differ diff --git a/docs/dli/umn/en-us_image_0000001377545298.png b/docs/dli/umn/en-us_image_0000001377545298.png new file mode 100644 index 00000000..9582364f Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001377545298.png differ diff --git a/docs/dli/umn/en-us_image_0000001391378486.png b/docs/dli/umn/en-us_image_0000001391378486.png new file mode 100644 index 00000000..2cf4fe0a Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001391378486.png differ diff --git a/docs/dli/umn/en-us_image_0000001391536818.png b/docs/dli/umn/en-us_image_0000001391536818.png new file mode 100644 index 00000000..533eaa80 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001391536818.png differ diff --git a/docs/dli/umn/en-us_image_0000001422601037.png b/docs/dli/umn/en-us_image_0000001422601037.png new file mode 100644 index 00000000..3f125bab Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001422601037.png differ diff --git a/docs/dli/umn/en-us_image_0000001427744557.png b/docs/dli/umn/en-us_image_0000001427744557.png new file mode 100644 index 00000000..0432c15d Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001427744557.png differ diff --git a/docs/dli/umn/en-us_image_0000001428187933.png b/docs/dli/umn/en-us_image_0000001428187933.png new file mode 100644 index 00000000..d0b26db3 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001428187933.png differ diff --git a/docs/dli/umn/en-us_image_0000001437320657.png b/docs/dli/umn/en-us_image_0000001437320657.png new file mode 100644 index 00000000..35e28315 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001437320657.png differ diff --git a/docs/dli/umn/en-us_image_0000001437640221.png b/docs/dli/umn/en-us_image_0000001437640221.png new file mode 100644 index 00000000..1909444d Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001437640221.png differ diff --git a/docs/dli/umn/en-us_image_0000001441378549.png b/docs/dli/umn/en-us_image_0000001441378549.png new file mode 100644 index 00000000..5ee8086d Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001441378549.png differ diff --git a/docs/dli/umn/en-us_image_0000001441656525.png b/docs/dli/umn/en-us_image_0000001441656525.png new file mode 100644 index 00000000..73a792a7 Binary files /dev/null and b/docs/dli/umn/en-us_image_0000001441656525.png differ diff --git a/docs/dli/umn/en-us_image_0206789726.jpg b/docs/dli/umn/en-us_image_0206789726.jpg new file mode 100644 index 00000000..2a1b04c7 Binary files /dev/null and b/docs/dli/umn/en-us_image_0206789726.jpg differ diff --git a/docs/dli/umn/en-us_image_0206789784.png b/docs/dli/umn/en-us_image_0206789784.png new file mode 100644 index 00000000..9aa38b1a Binary files /dev/null and b/docs/dli/umn/en-us_image_0206789784.png differ diff --git a/docs/dli/umn/en-us_image_0206789824.png b/docs/dli/umn/en-us_image_0206789824.png new file mode 100644 index 00000000..50ccdc02 Binary files /dev/null and b/docs/dli/umn/en-us_image_0206789824.png differ diff --git a/docs/dli/umn/en-us_image_0206789903.png b/docs/dli/umn/en-us_image_0206789903.png new file mode 100644 index 00000000..98418dca Binary files /dev/null and b/docs/dli/umn/en-us_image_0206789903.png differ diff --git a/docs/dli/umn/en-us_image_0207655173.png b/docs/dli/umn/en-us_image_0207655173.png new file mode 100644 index 00000000..cc45c153 Binary files /dev/null and b/docs/dli/umn/en-us_image_0207655173.png differ diff --git a/docs/dli/umn/en-us_image_0207655184.png b/docs/dli/umn/en-us_image_0207655184.png new file mode 100644 index 00000000..cc45c153 Binary files /dev/null and b/docs/dli/umn/en-us_image_0207655184.png differ diff --git a/docs/dli/umn/en-us_image_0237406526.png b/docs/dli/umn/en-us_image_0237406526.png new file mode 100644 index 00000000..97bb371e Binary files /dev/null and b/docs/dli/umn/en-us_image_0237406526.png differ diff --git a/docs/dli/umn/en-us_image_0237532018.png b/docs/dli/umn/en-us_image_0237532018.png new file mode 100644 index 00000000..fe3f53b4 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237532018.png differ diff --git a/docs/dli/umn/en-us_image_0237539075.png b/docs/dli/umn/en-us_image_0237539075.png new file mode 100644 index 00000000..97bb371e Binary files /dev/null and b/docs/dli/umn/en-us_image_0237539075.png differ diff --git a/docs/dli/umn/en-us_image_0237539077.png b/docs/dli/umn/en-us_image_0237539077.png new file mode 100644 index 00000000..97bb371e Binary files /dev/null and b/docs/dli/umn/en-us_image_0237539077.png differ diff --git a/docs/dli/umn/en-us_image_0237983632.png b/docs/dli/umn/en-us_image_0237983632.png new file mode 100644 index 00000000..f7c394b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237983632.png differ diff --git a/docs/dli/umn/en-us_image_0237984360.png b/docs/dli/umn/en-us_image_0237984360.png new file mode 100644 index 00000000..f7c394b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237984360.png differ diff --git a/docs/dli/umn/en-us_image_0237990324.png b/docs/dli/umn/en-us_image_0237990324.png new file mode 100644 index 00000000..f7c394b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237990324.png differ diff --git a/docs/dli/umn/en-us_image_0237994908.png b/docs/dli/umn/en-us_image_0237994908.png new file mode 100644 index 00000000..f7c394b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237994908.png differ diff --git a/docs/dli/umn/en-us_image_0237994909.png b/docs/dli/umn/en-us_image_0237994909.png new file mode 100644 index 00000000..f7c394b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237994909.png differ diff --git a/docs/dli/umn/en-us_image_0237994910.png b/docs/dli/umn/en-us_image_0237994910.png new file mode 100644 index 00000000..f7c394b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237994910.png differ diff --git a/docs/dli/umn/en-us_image_0237994911.png b/docs/dli/umn/en-us_image_0237994911.png new file mode 100644 index 00000000..f7c394b8 Binary files /dev/null and b/docs/dli/umn/en-us_image_0237994911.png differ diff --git a/docs/dli/umn/en-us_image_0264894338.png b/docs/dli/umn/en-us_image_0264894338.png new file mode 100644 index 00000000..4c0c7244 Binary files /dev/null and b/docs/dli/umn/en-us_image_0264894338.png differ diff --git a/docs/dli/umn/en-us_image_0264934285.png b/docs/dli/umn/en-us_image_0264934285.png new file mode 100644 index 00000000..6af3695d Binary files /dev/null and b/docs/dli/umn/en-us_image_0264934285.png differ diff --git a/docs/dli/umn/en-us_image_0264975521.png b/docs/dli/umn/en-us_image_0264975521.png new file mode 100644 index 00000000..b8ae06bc Binary files /dev/null and b/docs/dli/umn/en-us_image_0264975521.png differ diff --git a/docs/dli/umn/en-us_image_0269598720.png b/docs/dli/umn/en-us_image_0269598720.png new file mode 100644 index 00000000..e91e6418 Binary files /dev/null and b/docs/dli/umn/en-us_image_0269598720.png differ diff --git a/docs/dli/umn/en-us_image_0269598721.png b/docs/dli/umn/en-us_image_0269598721.png new file mode 100644 index 00000000..752ca78a Binary files /dev/null and b/docs/dli/umn/en-us_image_0269598721.png differ diff --git a/docs/dli/umn/en-us_image_0269598722.png b/docs/dli/umn/en-us_image_0269598722.png new file mode 100644 index 00000000..f385f5b0 Binary files /dev/null and b/docs/dli/umn/en-us_image_0269598722.png differ diff --git a/docs/dli/umn/en-us_image_0269598723.png b/docs/dli/umn/en-us_image_0269598723.png new file mode 100644 index 00000000..9b916e74 Binary files /dev/null and b/docs/dli/umn/en-us_image_0269598723.png differ diff --git a/docs/dli/umn/en-us_image_0269598724.png b/docs/dli/umn/en-us_image_0269598724.png new file mode 100644 index 00000000..78978728 Binary files /dev/null and b/docs/dli/umn/en-us_image_0269598724.png differ diff --git a/docs/dli/umn/en-us_image_0269598745.png b/docs/dli/umn/en-us_image_0269598745.png new file mode 100644 index 00000000..6e19cfc7 Binary files /dev/null and b/docs/dli/umn/en-us_image_0269598745.png differ diff --git a/docs/dli/umn/en-us_image_0276441461.png b/docs/dli/umn/en-us_image_0276441461.png new file mode 100644 index 00000000..97bb371e Binary files /dev/null and b/docs/dli/umn/en-us_image_0276441461.png differ diff --git a/docs/dli/umn/en-us_image_0296823520.png b/docs/dli/umn/en-us_image_0296823520.png new file mode 100644 index 00000000..25a80eda Binary files /dev/null and b/docs/dli/umn/en-us_image_0296823520.png differ diff --git a/docs/dli/umn/public_sys-resources/caution_3.0-en-us.png b/docs/dli/umn/public_sys-resources/caution_3.0-en-us.png new file mode 100644 index 00000000..60f60762 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/caution_3.0-en-us.png differ diff --git a/docs/dli/umn/public_sys-resources/danger_3.0-en-us.png b/docs/dli/umn/public_sys-resources/danger_3.0-en-us.png new file mode 100644 index 00000000..47a9c723 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/danger_3.0-en-us.png differ diff --git a/docs/dli/umn/public_sys-resources/delta.gif b/docs/dli/umn/public_sys-resources/delta.gif new file mode 100644 index 00000000..0d1b1f67 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/delta.gif differ diff --git a/docs/dli/umn/public_sys-resources/deltaend.gif b/docs/dli/umn/public_sys-resources/deltaend.gif new file mode 100644 index 00000000..cc7da0fc Binary files /dev/null and b/docs/dli/umn/public_sys-resources/deltaend.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-arrowdn.gif b/docs/dli/umn/public_sys-resources/icon-arrowdn.gif new file mode 100644 index 00000000..37942803 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-arrowdn.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-arrowrt.gif b/docs/dli/umn/public_sys-resources/icon-arrowrt.gif new file mode 100644 index 00000000..6aaaa11c Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-arrowrt.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-caution.gif b/docs/dli/umn/public_sys-resources/icon-caution.gif new file mode 100644 index 00000000..079c79b2 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-caution.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-danger.gif b/docs/dli/umn/public_sys-resources/icon-danger.gif new file mode 100644 index 00000000..079c79b2 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-danger.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-huawei.gif b/docs/dli/umn/public_sys-resources/icon-huawei.gif new file mode 100644 index 00000000..a31d60f8 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-huawei.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-note.gif b/docs/dli/umn/public_sys-resources/icon-note.gif new file mode 100644 index 00000000..31be2b03 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-note.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-notice.gif b/docs/dli/umn/public_sys-resources/icon-notice.gif new file mode 100644 index 00000000..40907065 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-notice.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-tip.gif b/docs/dli/umn/public_sys-resources/icon-tip.gif new file mode 100644 index 00000000..c47bae05 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-tip.gif differ diff --git a/docs/dli/umn/public_sys-resources/icon-warning.gif b/docs/dli/umn/public_sys-resources/icon-warning.gif new file mode 100644 index 00000000..079c79b2 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/icon-warning.gif differ diff --git a/docs/dli/umn/public_sys-resources/note_3.0-en-us.png b/docs/dli/umn/public_sys-resources/note_3.0-en-us.png new file mode 100644 index 00000000..57a0e1f5 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/note_3.0-en-us.png differ diff --git a/docs/dli/umn/public_sys-resources/notice_3.0-en-us.png b/docs/dli/umn/public_sys-resources/notice_3.0-en-us.png new file mode 100644 index 00000000..fa4b6499 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/notice_3.0-en-us.png differ diff --git a/docs/dli/umn/public_sys-resources/warning_3.0-en-us.png b/docs/dli/umn/public_sys-resources/warning_3.0-en-us.png new file mode 100644 index 00000000..def5c356 Binary files /dev/null and b/docs/dli/umn/public_sys-resources/warning_3.0-en-us.png differ