Upgrade version to 1.6.0 (#5142)

* add 1.6.0 update ddl sql

* modify default python3 path

* Fix build error

* Upgrade version to 1.6.0

Author: peacewong

.github/workflows/integration-test.yml

@@ -45,7 +45,7 @@ jobs:
       TAG: ${{ github.sha }}
       SKIP_TEST: true
       HUB: ghcr.io/apache/linkis
-      LINKIS_VERSION: 1.5.0
+      LINKIS_VERSION: 1.6.0
     steps:
       - name: Free up disk space
         run: |

.github/workflows/publish-docker.yaml

@@ -34,7 +34,7 @@ jobs:
       TAG: ${{ github.sha }}
       SKIP_TEST: true
       HUB: ghcr.io/apache/linkis
-      LINKIS_VERSION: 1.5.0
+      LINKIS_VERSION: 1.6.0
     steps:
     - name: Checkout
       uses: actions/checkout@v2

.github/workflows/publish-snapshot.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
         branch:
-          - dev-1.5.0
+          - dev-1.6.0
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2

linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala

@@ -70,7 +70,7 @@ object Configuration extends Logging {
   // Only the specified token has permission to call some api
   val GOVERNANCE_STATION_ADMIN_TOKEN_STARTWITH = "ADMIN-"
 
-  val VARIABLE_OPERATION: Boolean = CommonVars("wds.linkis.variable.operation", false).getValue
+  val VARIABLE_OPERATION: Boolean = CommonVars("wds.linkis.variable.operation", true).getValue
 
   val ERROR_MSG_TIP =
     CommonVars(

linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java

@@ -132,7 +132,7 @@ public Message execute(HttpServletRequest req, @RequestBody Map<String, Object>
                 + jobReqId
                 + " in "
                 + Sender.getThisServiceInstance().toString()
-                + ". \n Please wait it to be scheduled(您的任务已经提交，进入排队中，如果一直没有更新日志，是任务并发达到了限制，可以在ITSM提Linkis参数修改单)"),
+                + ". \n Please wait it to be scheduled(您的任务已经提交，进入排队中，如果一直没有更新日志，是任务并发达到了限制，可以进行参数修改)"),
         job);
     message = Message.ok();
     message.setMethod("/api/entrance/execute");
@@ -186,7 +186,7 @@ public Message submit(HttpServletRequest req, @RequestBody Map<String, Object> j
                 + jobReqId
                 + " in "
                 + Sender.getThisServiceInstance().toString()
-                + ". \n Please wait it to be scheduled(您的任务已经提交，进入排队中，如果一直没有更新日志，是任务并发达到了限制，可以在ITSM提Linkis参数修改单)"),
+                + ". \n Please wait it to be scheduled(您的任务已经提交，进入排队中，如果一直没有更新日志，是任务并发达到了限制，可以进行参数修改)"),
         job);
     String execID =
         ZuulEntranceUtils.generateExecID(

linkis-dist/package/db/linkis_ddl.sql

@@ -36,8 +36,8 @@ CREATE TABLE `linkis_ps_configuration_config_key`(
     `name`             varchar(50)  DEFAULT NULL,
     `default_value`    varchar(200) DEFAULT NULL COMMENT 'Adopted when user does not set key',
     `validate_type`    varchar(50)  DEFAULT NULL COMMENT 'Validate type, one of the following: None, NumInterval, FloatInterval, Include, Regex, OPF, Custom Rules',
-    `validate_range`   varchar(50)  DEFAULT NULL COMMENT 'Validate range',
-    `engine_conn_type` varchar(50)  DEFAULT NULL COMMENT 'engine type,such as spark,hive etc',
+    `validate_range`   varchar(150)  DEFAULT NULL COMMENT 'Validate range',
+    `engine_conn_type` varchar(50)  DEFAULT '' COMMENT 'engine type,such as spark,hive etc',
     `is_hidden`        tinyint(1)   DEFAULT NULL COMMENT 'Whether it is hidden from user. If set to 1(true), then user cannot modify, however, it could still be used in back-end',
     `is_advanced`      tinyint(1)   DEFAULT NULL COMMENT 'Whether it is an advanced parameter. If set to 1(true), parameters would be displayed only when user choose to do so',
     `level`            tinyint(1)   DEFAULT NULL COMMENT 'Basis for displaying sorting in the front-end. Higher the level is, higher the rank the parameter gets',
@@ -47,6 +47,7 @@ CREATE TABLE `linkis_ps_configuration_config_key`(
     `en_name` varchar(100) DEFAULT NULL COMMENT 'english name',
     `en_treeName` varchar(100) DEFAULT NULL COMMENT 'english treeName',
     `template_required` tinyint(1) DEFAULT 0 COMMENT 'template required 0 none / 1 must',
+    UNIQUE KEY `uniq_key_ectype` (`key`,`engine_conn_type`),
     PRIMARY KEY (`id`)
 )ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
 
@@ -629,6 +630,7 @@ CREATE TABLE if not exists `linkis_ps_bml_resources_task` (
   `start_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Starting time',
   `end_time` datetime DEFAULT NULL COMMENT 'End Time',
   `last_update_time` datetime NOT NULL COMMENT 'Last update time',
+   unique key `uniq_rid_version` (resource_id, version),
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
@@ -856,7 +858,8 @@ CREATE TABLE `linkis_cg_ec_resource_info_record` (
     `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
     PRIMARY KEY (`id`),
     KEY `idx_ticket_id` (`ticket_id`),
-    UNIQUE KEY `uniq_tid_lv` (`ticket_id`,`label_value`)
+    UNIQUE KEY `uniq_tid_lv` (`ticket_id`,`label_value`),
+    UNIQUE KEY uniq_sinstance_status_cuser_ctime (service_instance, status, create_user, create_time)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
 
 DROP TABLE IF EXISTS `linkis_cg_manager_label_service_instance`;
@@ -906,6 +909,7 @@ CREATE TABLE `linkis_cg_manager_service_instance_metrics` (
   `healthy_status` varchar(255) COLLATE utf8_bin DEFAULT NULL,
   `update_time` datetime DEFAULT CURRENT_TIMESTAMP,
   `create_time` datetime DEFAULT CURRENT_TIMESTAMP,
+   description varchar(256) CHARSET utf8mb4 COLLATE utf8mb4_bin DEFAULT '',
   PRIMARY KEY (`instance`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
 
@@ -1064,6 +1068,7 @@ CREATE TABLE `linkis_cg_tenant_label_config` (
   `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `desc` varchar(100) COLLATE utf8_bin NOT NULL,
   `bussiness_user` varchar(50) COLLATE utf8_bin NOT NULL,
+  `is_valid` varchar(1) CHARSET utf8mb4 COLLATE utf8mb4_bin DEFAULT 'Y' COMMENT 'is valid',
   PRIMARY KEY (`id`),
   UNIQUE KEY `uniq_user_creator` (`user`,`creator`)
 ) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
@@ -1084,3 +1089,22 @@ CREATE TABLE `linkis_cg_user_ip_config` (
   PRIMARY KEY (`id`),
   UNIQUE KEY `uniq_user_creator` (`user`,`creator`)
 ) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_org_user
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_org_user`;
+CREATE TABLE `linkis_org_user` (
+    `cluster_code` varchar(16) COMMENT 'cluster code',
+    `user_type` varchar(64) COMMENT 'user type',
+    `user_name` varchar(128) COMMENT 'username',
+    `org_id` varchar(16) COMMENT 'org id',
+    `org_name` varchar(64) COMMENT 'org name',
+    `queue_name` varchar(64) COMMENT 'yarn queue name',
+    `db_name` varchar(64) COMMENT 'default db name',
+    `interface_user` varchar(64) COMMENT 'interface user',
+    `is_union_analyse` varchar(64) COMMENT 'is union analyse',
+    `create_time` varchar(64) COMMENT 'create time',
+    `user_itsm_no` varchar(64) COMMENT 'user itsm no',
+    PRIMARY KEY (`user_name`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE=utf8mb4_bin COMMENT ='user org info';

linkis-dist/package/db/linkis_ddl_pg.sql

@@ -78,7 +78,7 @@ DROP TABLE IF EXISTS "linkis_ps_configuration_config_value";
 CREATE TABLE linkis_ps_configuration_config_value (
 	id bigserial NOT NULL,
 	config_key_id int4 NULL,
-	config_value varchar(200) NULL,
+	config_value varchar(500) NULL,
 	config_label_id int8 NULL,
 	update_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
 	create_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -996,6 +996,7 @@ CREATE TABLE linkis_cg_manager_service_instance_metrics (
 	healthy_status varchar(255) NULL,
 	update_time timestamp(6) NULL DEFAULT CURRENT_TIMESTAMP,
 	create_time timestamp(6) NULL DEFAULT CURRENT_TIMESTAMP,
+    description varchar(256) NULL,
 	CONSTRAINT linkis_manager_service_instance_metrics_pkey PRIMARY KEY (instance)
 );
 
@@ -1167,3 +1168,19 @@ CREATE TABLE "linkis_cg_user_ip_config" (
   CONSTRAINT linkis_cg_user_ip_config_pkey PRIMARY KEY (id)
 );
 CREATE UNIQUE INDEX uniq_user_creator_uic ON linkis_cg_user_ip_config USING btree ("user",creator);
+
+DROP TABLE IF EXISTS linkis_org_user;
+
+CREATE TABLE linkis_org_user (
+  cluster_code varchar(16) COMMENT 'cluster code',
+  user_type varchar(64) COMMENT 'user type',
+  user_name varchar(128) PRIMARY KEY COMMENT 'username',
+  org_id varchar(16) COMMENT 'org id',
+  org_name varchar(64) COMMENT 'org name',
+  queue_name varchar(64) COMMENT 'yarn queue name',
+  db_name varchar(64) COMMENT 'default db name',
+  interface_user varchar(64) COMMENT 'interface user',
+  is_union_analyse varchar(64) COMMENT 'is union analyse',
+  create_time varchar(64) COMMENT 'create time',
+  user_itsm_no varchar(64) COMMENT 'user itsm no'
+) COMMENT ON TABLE linkis_org_user IS 'user org info';

linkis-dist/package/db/linkis_dml.sql

@@ -169,7 +169,7 @@ INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`,
 INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('linkis.trino.ssl.truststore.password', 'Trino服务器SSL truststore密码', 'truststore密码', 'null', 'None', '', 'trino', 0, 0, 1, '数据源配置');
 
 -- Configuration first level directory
-insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-全局设置,*-*', 'OPTIONAL', 2, now(), now());
+insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-GlobalSettings,*-*', 'OPTIONAL', 2, now(), now());
 insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-IDE,*-*', 'OPTIONAL', 2, now(), now());
 insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-Visualis,*-*', 'OPTIONAL', 2, now(), now());
 insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-nodeexecution,*-*', 'OPTIONAL', 2, now(), now());
@@ -269,7 +269,7 @@ insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_featur
 
 
 -- Associate first-level and second-level directories
-select @label_id := id from linkis_cg_manager_label where `label_value` = '*-全局设置,*-*';
+select @label_id := id from linkis_cg_manager_label where `label_value` = '*-GlobalSettings,*-*';
 insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 1);
 
 select @label_id := id from linkis_cg_manager_label where `label_value` = '*-IDE,*-*';

linkis-dist/package/db/upgrade/1.6.0_schema/mysql/linkis_ddl.sql

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE linkis_ps_bml_resources_task ADD CONSTRAINT  uniq_rid_version UNIQUE (resource_id, version);
+ALTER TABLE linkis_cg_ec_resource_info_record ADD UNIQUE INDEX uniq_sinstance_status_cuser_ctime (service_instance, status, create_user, create_time);
+ALTER TABLE linkis_cg_manager_service_instance_metrics ADD COLUMN description varchar(256) CHARSET utf8mb4 COLLATE utf8mb4_bin DEFAULT '';
+ALTER TABLE  linkis_ps_configuration_config_value modify COLUMN  config_value varchar(500) CHARSET utf8mb4 COLLATE utf8mb4_bin;
+ALTER TABLE `linkis_ps_configuration_config_key` ADD UNIQUE `uniq_key_ectype` (`key`,`engine_conn_type`);
+ALTER TABLE `linkis_ps_configuration_config_key` modify column `engine_conn_type` varchar(50) DEFAULT '' COMMENT 'engine type,such as spark,hive etc';
+ALTER TABLE `linkis_ps_udf_version`  ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time';
+ALTER TABLE `linkis_ps_udf_tree` ADD CONSTRAINT  `uniq_parent_name_uname_category` UNIQUE (`parent`,`name`, `user_name`, `category`);
+ALTER TABLE `linkis_cg_ec_resource_info_record` MODIFY COLUMN `metrics` text CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'ec metrics';
+ALTER TABLE `linkis_ps_configuration_config_key`
+    CHANGE COLUMN `validate_range` `validate_range` VARCHAR(150) NULL DEFAULT NULL COMMENT 'Validate range' COLLATE 'utf8_bin' AFTER `validate_type`;
+ALTER TABLE linkis_cg_tenant_label_config ADD COLUMN is_valid varchar(1) CHARSET utf8mb4 COLLATE utf8mb4_bin DEFAULT 'Y' COMMENT '是否有效';
+
+
+-- ----------------------------
+-- Table structure for linkis_org_user
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_org_user`;
+CREATE TABLE `linkis_org_user` (
+    `cluster_code` varchar(16) COMMENT 'cluster code',
+    `user_type` varchar(64) COMMENT 'user type',
+    `user_name` varchar(128) COMMENT 'username',
+    `org_id` varchar(16) COMMENT 'org id',
+    `org_name` varchar(64) COMMENT 'org name',
+    `queue_name` varchar(64) COMMENT 'yarn queue name',
+    `db_name` varchar(64) COMMENT 'default db name',
+    `interface_user` varchar(64) COMMENT 'interface user',
+    `is_union_analyse` varchar(64) COMMENT 'is union analyse',
+    `create_time` varchar(64) COMMENT 'create time',
+    `user_itsm_no` varchar(64) COMMENT 'user itsm no',
+    PRIMARY KEY (`user_name`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE=utf8mb4_bin COMMENT ='user org info';
+
+
+

linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala

@@ -153,7 +153,7 @@ object SparkConfiguration extends Logging {
     CommonVars("wds.linkis.spark.engineconn.fatal.log", "error writing class;OutOfMemoryError")
 
   val PYSPARK_PYTHON3_PATH =
-    CommonVars[String]("pyspark.python3.path", "/appcom/Install/anaconda3/bin/python")
+    CommonVars[String]("pyspark.python3.path", "python3")
 
   val ENABLE_REPLACE_PACKAGE_NAME =
     CommonVars("wds.linkis.spark.engine.scala.replace_package_header.enable", true)