提交 d18b47a1 authored 作者: 宋勇's avatar 宋勇

atasource-jdbc-demeng

datasource-jdbc-access datasource-http datasource-xml datasource-csv datasource-excel 增加
上级 373502c0
...@@ -141,7 +141,6 @@ ...@@ -141,7 +141,6 @@
<artifactId>aws-java-sdk-bundle</artifactId> <artifactId>aws-java-sdk-bundle</artifactId>
</dependency> </dependency>
</dependencies>
</dependencies>
</project> </project>
package org.apache.seatunnel.datasource.plugin.csv; package org.apache.seatunnel.datasource.plugin.csv;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
import org.apache.hadoop.conf.Configuration;
import lombok.extern.slf4j.Slf4j;
import java.util.Map; import java.util.Map;
@Slf4j @Slf4j
...@@ -29,7 +31,7 @@ public class CSVAConfiguration { ...@@ -29,7 +31,7 @@ public class CSVAConfiguration {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"S3 datasource endpoint is null, please check your config"); "S3 datasource endpoint is null, please check your config");
} }
String bucket = s3Options.get( CSVOptionRule.BUCKET.key()); String bucket = s3Options.get(CSVOptionRule.BUCKET.key());
String protocol = DEFAULT_PROTOCOL; String protocol = DEFAULT_PROTOCOL;
if (bucket.startsWith(S3A_PROTOCOL)) { if (bucket.startsWith(S3A_PROTOCOL)) {
......
...@@ -17,13 +17,14 @@ ...@@ -17,13 +17,14 @@
package org.apache.seatunnel.datasource.plugin.csv; package org.apache.seatunnel.datasource.plugin.csv;
import com.google.auto.service.AutoService;
import com.google.common.collect.Sets;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
import com.google.auto.service.AutoService;
import com.google.common.collect.Sets;
import java.util.Set; import java.util.Set;
@AutoService(DataSourceFactory.class) @AutoService(DataSourceFactory.class)
......
...@@ -17,18 +17,20 @@ ...@@ -17,18 +17,20 @@
package org.apache.seatunnel.datasource.plugin.csv; package org.apache.seatunnel.datasource.plugin.csv;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSON;
import io.minio.*; import io.minio.*;
import io.minio.errors.*; import io.minio.errors.*;
import io.minio.messages.Bucket; import io.minio.messages.Bucket;
import io.minio.messages.Item; import io.minio.messages.Item;
import lombok.NonNull; import lombok.NonNull;
import org.apache.commons.lang3.StringUtils;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
......
...@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option; ...@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option;
import org.apache.seatunnel.api.configuration.Options; import org.apache.seatunnel.api.configuration.Options;
import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.OptionRule;
import java.util.Arrays;
import java.util.Map; import java.util.Map;
public class CSVOptionRule { public class CSVOptionRule {
...@@ -129,7 +128,6 @@ public class CSVOptionRule { ...@@ -129,7 +128,6 @@ public class CSVOptionRule {
.build(); .build();
} }
public enum S3aAwsCredentialsProvider { public enum S3aAwsCredentialsProvider {
SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"), SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"),
......
...@@ -141,7 +141,6 @@ ...@@ -141,7 +141,6 @@
<artifactId>aws-java-sdk-bundle</artifactId> <artifactId>aws-java-sdk-bundle</artifactId>
</dependency> </dependency>
</dependencies>
</dependencies>
</project> </project>
package org.apache.seatunnel.datasource.plugin.excel; package org.apache.seatunnel.datasource.plugin.excel;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
import org.apache.hadoop.conf.Configuration;
import lombok.extern.slf4j.Slf4j;
import java.util.Map; import java.util.Map;
@Slf4j @Slf4j
...@@ -29,7 +31,7 @@ public class ExcelAConfiguration { ...@@ -29,7 +31,7 @@ public class ExcelAConfiguration {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"S3 datasource endpoint is null, please check your config"); "S3 datasource endpoint is null, please check your config");
} }
String bucket = s3Options.get( ExcelOptionRule.BUCKET.key()); String bucket = s3Options.get(ExcelOptionRule.BUCKET.key());
String protocol = DEFAULT_PROTOCOL; String protocol = DEFAULT_PROTOCOL;
if (bucket.startsWith(S3A_PROTOCOL)) { if (bucket.startsWith(S3A_PROTOCOL)) {
......
...@@ -17,13 +17,14 @@ ...@@ -17,13 +17,14 @@
package org.apache.seatunnel.datasource.plugin.excel; package org.apache.seatunnel.datasource.plugin.excel;
import com.google.auto.service.AutoService;
import com.google.common.collect.Sets;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
import com.google.auto.service.AutoService;
import com.google.common.collect.Sets;
import java.util.Set; import java.util.Set;
@AutoService(DataSourceFactory.class) @AutoService(DataSourceFactory.class)
......
...@@ -17,18 +17,20 @@ ...@@ -17,18 +17,20 @@
package org.apache.seatunnel.datasource.plugin.excel; package org.apache.seatunnel.datasource.plugin.excel;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSON;
import io.minio.*; import io.minio.*;
import io.minio.errors.*; import io.minio.errors.*;
import io.minio.messages.Bucket; import io.minio.messages.Bucket;
import io.minio.messages.Item; import io.minio.messages.Item;
import lombok.NonNull; import lombok.NonNull;
import org.apache.commons.lang3.StringUtils;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
......
...@@ -129,7 +129,6 @@ public class ExcelOptionRule { ...@@ -129,7 +129,6 @@ public class ExcelOptionRule {
.build(); .build();
} }
public enum S3aAwsCredentialsProvider { public enum S3aAwsCredentialsProvider {
SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"), SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"),
......
...@@ -27,8 +27,7 @@ public class HttpAConfiguration { ...@@ -27,8 +27,7 @@ public class HttpAConfiguration {
public static HttpConfiguration getConfiguration(Map<String, String> ftpOption) { public static HttpConfiguration getConfiguration(Map<String, String> ftpOption) {
if (!ftpOption.containsKey(HttpOptionRule.URL.key())) { if (!ftpOption.containsKey(HttpOptionRule.URL.key())) {
throw new IllegalArgumentException( throw new IllegalArgumentException("url is null, please check your config");
"url is null, please check your config");
} }
HttpConfiguration httpAConfiguration = new HttpConfiguration(); HttpConfiguration httpAConfiguration = new HttpConfiguration();
...@@ -37,7 +36,6 @@ public class HttpAConfiguration { ...@@ -37,7 +36,6 @@ public class HttpAConfiguration {
httpAConfiguration.setToken(HttpOptionRule.TOKEN.key()); httpAConfiguration.setToken(HttpOptionRule.TOKEN.key());
httpAConfiguration.setRequest_params(HttpOptionRule.REQUEST_PARAMS.key()); httpAConfiguration.setRequest_params(HttpOptionRule.REQUEST_PARAMS.key());
return httpAConfiguration; return httpAConfiguration;
} }
} }
package org.apache.seatunnel.datasource.plugin.http; package org.apache.seatunnel.datasource.plugin.http;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
...@@ -8,14 +7,9 @@ public class HttpClientService { ...@@ -8,14 +7,9 @@ public class HttpClientService {
public static HttpClient connect(HttpConfiguration conf) throws Exception { public static HttpClient connect(HttpConfiguration conf) throws Exception {
// 创建HttpClient实例 // 创建HttpClient实例
HttpClient client = HttpClients.createDefault(); HttpClient client = HttpClients.createDefault();
return client; return client;
} }
} }
...@@ -10,12 +10,11 @@ public class HttpConfiguration { ...@@ -10,12 +10,11 @@ public class HttpConfiguration {
public HttpConfiguration() {} public HttpConfiguration() {}
public HttpConfiguration( public HttpConfiguration(String url, String method, String token, String request_params) {
String url,String method, String token,String request_params ) {
this.url = url; this.url = url;
this.token = token; this.token = token;
this.method=method; this.method = method;
this.request_params=request_params; this.request_params = request_params;
} }
public String getUrl() { public String getUrl() {
...@@ -26,7 +25,6 @@ public class HttpConfiguration { ...@@ -26,7 +25,6 @@ public class HttpConfiguration {
this.url = url; this.url = url;
} }
public String getMethod() { public String getMethod() {
return method; return method;
} }
......
...@@ -17,23 +17,21 @@ ...@@ -17,23 +17,21 @@
package org.apache.seatunnel.datasource.plugin.http; package org.apache.seatunnel.datasource.plugin.http;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.*; import org.apache.http.client.methods.*;
import org.apache.http.entity.StringEntity; import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import lombok.NonNull; import lombok.NonNull;
import java.net.URI; import java.net.URI;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
...@@ -75,7 +73,8 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -75,7 +73,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
System.out.println("url:" + url); System.out.println("url:" + url);
HttpGet httpGet = new HttpGet(url); HttpGet httpGet = new HttpGet(url);
if (StringUtils.isNotBlank(token)) { if (StringUtils.isNotBlank(token)) {
httpGet.setHeader("Authorization", "Bearer " + token.replace("Bearer ", "").trim()); httpGet.setHeader(
"Authorization", "Bearer " + token.replace("Bearer ", "").trim());
} }
// 执行请求并获得响应 // 执行请求并获得响应
...@@ -83,7 +82,8 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -83,7 +82,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
} else if (StringUtils.isBlank(method) || "POST".equals(method.toUpperCase())) { } else if (StringUtils.isBlank(method) || "POST".equals(method.toUpperCase())) {
HttpPost httpPost = new HttpPost(url); HttpPost httpPost = new HttpPost(url);
if (StringUtils.isNotBlank(token)) { if (StringUtils.isNotBlank(token)) {
httpPost.setHeader("Authorization", "Bearer " + token.replace("Bearer ", "").trim()); httpPost.setHeader(
"Authorization", "Bearer " + token.replace("Bearer ", "").trim());
} }
// 设置请求体(例如:JSON数据) // 设置请求体(例如:JSON数据)
StringEntity requestEntity = new StringEntity(parmams); StringEntity requestEntity = new StringEntity(parmams);
...@@ -94,7 +94,8 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -94,7 +94,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
} else if (StringUtils.isBlank(method) || "PUT".equals(method.toUpperCase())) { } else if (StringUtils.isBlank(method) || "PUT".equals(method.toUpperCase())) {
HttpPut httpPut = new HttpPut(url); HttpPut httpPut = new HttpPut(url);
if (StringUtils.isNotBlank(token)) { if (StringUtils.isNotBlank(token)) {
httpPut.setHeader("Authorization", "Bearer " + token.replace("Bearer ", "").trim()); httpPut.setHeader(
"Authorization", "Bearer " + token.replace("Bearer ", "").trim());
} }
// 设置请求体(例如:JSON数据) // 设置请求体(例如:JSON数据)
StringEntity requestEntity = new StringEntity(parmams); StringEntity requestEntity = new StringEntity(parmams);
...@@ -110,7 +111,8 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -110,7 +111,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
HttpDelete httpDelete = new HttpDelete(url); HttpDelete httpDelete = new HttpDelete(url);
if (StringUtils.isNotBlank(token)) { if (StringUtils.isNotBlank(token)) {
httpDelete.setHeader("Authorization", "Bearer " + token.replace("Bearer ", "").trim()); httpDelete.setHeader(
"Authorization", "Bearer " + token.replace("Bearer ", "").trim());
} }
// 执行请求并获得响应 // 执行请求并获得响应
...@@ -119,7 +121,8 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -119,7 +121,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
} else if (StringUtils.isBlank(method) || "PATCH".equals(method.toUpperCase())) { } else if (StringUtils.isBlank(method) || "PATCH".equals(method.toUpperCase())) {
HttpPatch httpPatch = new HttpPatch(url); HttpPatch httpPatch = new HttpPatch(url);
if (StringUtils.isNotBlank(token)) { if (StringUtils.isNotBlank(token)) {
httpPatch.setHeader("Authorization", "Bearer " + token.replace("Bearer ", "").trim()); httpPatch.setHeader(
"Authorization", "Bearer " + token.replace("Bearer ", "").trim());
} }
// 设置请求体(例如:JSON数据) // 设置请求体(例如:JSON数据)
StringEntity requestEntity = new StringEntity(parmams); StringEntity requestEntity = new StringEntity(parmams);
...@@ -133,7 +136,8 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -133,7 +136,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
} }
HttpOptions httpOptions = new HttpOptions(url); HttpOptions httpOptions = new HttpOptions(url);
if (StringUtils.isNotBlank(token)) { if (StringUtils.isNotBlank(token)) {
httpOptions.setHeader("Authorization", "Bearer " + token.replace("Bearer ", "").trim()); httpOptions.setHeader(
"Authorization", "Bearer " + token.replace("Bearer ", "").trim());
} }
// 执行请求并获得响应 // 执行请求并获得响应
...@@ -145,7 +149,8 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -145,7 +149,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
} }
HttpHead httpHead = new HttpHead(url); HttpHead httpHead = new HttpHead(url);
if (StringUtils.isNotBlank(token)) { if (StringUtils.isNotBlank(token)) {
httpHead.setHeader("Authorization", "Bearer " + token.replace("Bearer ", "").trim()); httpHead.setHeader(
"Authorization", "Bearer " + token.replace("Bearer ", "").trim());
} }
// 执行请求并获得响应 // 执行请求并获得响应
...@@ -160,7 +165,6 @@ public class HttpDatasourceChannel implements DataSourceChannel { ...@@ -160,7 +165,6 @@ public class HttpDatasourceChannel implements DataSourceChannel {
String responseBody = EntityUtils.toString(response.getEntity()); String responseBody = EntityUtils.toString(response.getEntity());
System.out.println("Response Body: " + responseBody); System.out.println("Response Body: " + responseBody);
if (statusCode == 200) { if (statusCode == 200) {
return true; return true;
} else { } else {
......
...@@ -47,11 +47,8 @@ public class HttpOptionRule { ...@@ -47,11 +47,8 @@ public class HttpOptionRule {
.noDefaultValue() .noDefaultValue()
.withDescription("the http user token to use when connecting to the broker"); .withDescription("the http user token to use when connecting to the broker");
public static OptionRule optionRule() { public static OptionRule optionRule() {
return OptionRule.builder().required(URL,METHOD).optional(TOKEN,REQUEST_PARAMS).build(); return OptionRule.builder().required(URL, METHOD).optional(TOKEN, REQUEST_PARAMS).build();
} }
public static OptionRule metadataRule() { public static OptionRule metadataRule() {
...@@ -59,8 +56,6 @@ public class HttpOptionRule { ...@@ -59,8 +56,6 @@ public class HttpOptionRule {
} }
public enum FileFormat { public enum FileFormat {
JSON("json"), JSON("json"),
; ;
......
...@@ -56,7 +56,17 @@ ...@@ -56,7 +56,17 @@
<artifactId>ucanaccess</artifactId> <artifactId>ucanaccess</artifactId>
<version>5.0.1</version> <version>5.0.1</version>
</dependency> </dependency>
<dependency>
<groupId>com.cdzhiyong.unified</groupId>
<artifactId>minio-spring-boot-starter</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.14</version>
</dependency>
</dependencies> </dependencies>
</project> </project>
...@@ -37,15 +37,22 @@ public class AccessDataSourceConfig { ...@@ -37,15 +37,22 @@ public class AccessDataSourceConfig {
.type(DatasourcePluginTypeEnum.DATABASE.getCode()) .type(DatasourcePluginTypeEnum.DATABASE.getCode())
.build(); .build();
public static final Set<String> MYSQL_SYSTEM_DATABASES = public static final Set<String> MYSQL_SYSTEM_DATABASES = Sets.newHashSet("SYSTEM", "ROLL");
Sets.newHashSet("SYSTEM", "ROLL");
public static final OptionRule OPTION_RULE = public static final OptionRule OPTION_RULE =
OptionRule.builder() OptionRule.builder()
.required(org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule.URL, org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule.DRIVER) .required(
.optional(org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule.USER, org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule.PASSWORD) org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule.URL,
org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule
.DRIVER)
.optional(
org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule
.USER,
org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule
.PASSWORD)
.build(); .build();
// public static final OptionRule METADATA_RULE = // public static final OptionRule METADATA_RULE =
// OptionRule.builder().required(org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.DATABASE, org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.TABLE).build(); //
// OptionRule.builder().required(org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.DATABASE, org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.TABLE).build();
} }
...@@ -27,8 +27,7 @@ public class AccessOptionRule { ...@@ -27,8 +27,7 @@ public class AccessOptionRule {
.stringType() .stringType()
.noDefaultValue() .noDefaultValue()
.withDescription( .withDescription(
"jdbc url, eg:" "jdbc url, eg:" + " http://localhost:9000/bucket/filename.mdb");
+ " http://localhost:9000/bucket/filename.mdb");
public static final Option<String> USER = public static final Option<String> USER =
Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
...@@ -36,11 +35,12 @@ public class AccessOptionRule { ...@@ -36,11 +35,12 @@ public class AccessOptionRule {
public static final Option<String> PASSWORD = public static final Option<String> PASSWORD =
Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
// public static final Option<String> DATABASE = // public static final Option<String> DATABASE =
// Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); // Options.key("database").stringType().noDefaultValue().withDescription("jdbc
// database");
// public static final Option<String> TABLE = // public static final Option<String> TABLE =
// Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); // Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
public static final Option<DriverType> DRIVER = public static final Option<DriverType> DRIVER =
Options.key("driver") Options.key("driver")
......
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
<version>1.0.0-SNAPSHOT</version> <version>1.0.0-SNAPSHOT</version>
</parent> </parent>
<artifactId>datasource-jdbc-demeng</artifactId> <artifactId>datasource-jdbc-dameng</artifactId>
<properties> <properties>
<mysql-connector.version>8.0.28</mysql-connector.version> <mysql-connector.version>8.0.28</mysql-connector.version>
......
...@@ -25,9 +25,9 @@ import com.google.common.collect.Sets; ...@@ -25,9 +25,9 @@ import com.google.common.collect.Sets;
import java.util.Set; import java.util.Set;
public class DemengDataSourceConfig { public class DamengDataSourceConfig {
public static final String PLUGIN_NAME = "JDBC-Demeng"; public static final String PLUGIN_NAME = "JDBC-Dameng";
public static final DataSourcePluginInfo MYSQL_DATASOURCE_PLUGIN_INFO = public static final DataSourcePluginInfo MYSQL_DATASOURCE_PLUGIN_INFO =
DataSourcePluginInfo.builder() DataSourcePluginInfo.builder()
...@@ -37,15 +37,16 @@ public class DemengDataSourceConfig { ...@@ -37,15 +37,16 @@ public class DemengDataSourceConfig {
.type(DatasourcePluginTypeEnum.DATABASE.getCode()) .type(DatasourcePluginTypeEnum.DATABASE.getCode())
.build(); .build();
public static final Set<String> MYSQL_SYSTEM_DATABASES = public static final Set<String> MYSQL_SYSTEM_DATABASES = Sets.newHashSet("SYSTEM", "ROLL");
Sets.newHashSet("SYSTEM", "ROLL");
public static final OptionRule OPTION_RULE = public static final OptionRule OPTION_RULE =
OptionRule.builder() OptionRule.builder()
.required(DemengOptionRule.URL, DemengOptionRule.DRIVER) .required(DamengOptionRule.URL, DamengOptionRule.DRIVER)
.optional(DemengOptionRule.USER, DemengOptionRule.PASSWORD) .optional(DamengOptionRule.USER, DamengOptionRule.PASSWORD)
.build(); .build();
public static final OptionRule METADATA_RULE = public static final OptionRule METADATA_RULE =
OptionRule.builder().required(DemengOptionRule.DATABASE, DemengOptionRule.TABLE).build(); OptionRule.builder()
.required(DamengOptionRule.DATABASE, DamengOptionRule.TABLE)
.build();
} }
...@@ -29,20 +29,20 @@ import java.util.Set; ...@@ -29,20 +29,20 @@ import java.util.Set;
@Slf4j @Slf4j
@AutoService(DataSourceFactory.class) @AutoService(DataSourceFactory.class)
public class DemengJdbcDataSourceFactory implements DataSourceFactory { public class DamengJdbcDataSourceFactory implements DataSourceFactory {
@Override @Override
public String factoryIdentifier() { public String factoryIdentifier() {
return DemengDataSourceConfig.PLUGIN_NAME; return DamengDataSourceConfig.PLUGIN_NAME;
} }
@Override @Override
public Set<DataSourcePluginInfo> supportedDataSources() { public Set<DataSourcePluginInfo> supportedDataSources() {
return Sets.newHashSet(DemengDataSourceConfig.MYSQL_DATASOURCE_PLUGIN_INFO); return Sets.newHashSet(DamengDataSourceConfig.MYSQL_DATASOURCE_PLUGIN_INFO);
} }
@Override @Override
public DataSourceChannel createChannel() { public DataSourceChannel createChannel() {
return new DemengJdbcDataSourceChannel(); return new DamengJdbcDataSourceChannel();
} }
} }
...@@ -20,15 +20,13 @@ package org.apache.seatunnel.datasource.plugin.demeng.jdbc; ...@@ -20,15 +20,13 @@ package org.apache.seatunnel.datasource.plugin.demeng.jdbc;
import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.Option;
import org.apache.seatunnel.api.configuration.Options; import org.apache.seatunnel.api.configuration.Options;
public class DemengOptionRule { public class DamengOptionRule {
public static final Option<String> URL = public static final Option<String> URL =
Options.key("url") Options.key("url")
.stringType() .stringType()
.noDefaultValue() .noDefaultValue()
.withDescription( .withDescription("jdbc url, eg:" + " jdbc:dm://localhost:5236");
"jdbc url, eg:"
+ " jdbc:dm://localhost:5236");
public static final Option<String> USER = public static final Option<String> USER =
Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
......
...@@ -141,7 +141,6 @@ ...@@ -141,7 +141,6 @@
<artifactId>aws-java-sdk-bundle</artifactId> <artifactId>aws-java-sdk-bundle</artifactId>
</dependency> </dependency>
</dependencies>
</dependencies>
</project> </project>
package org.apache.seatunnel.datasource.plugin.xml; package org.apache.seatunnel.datasource.plugin.xml;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
import org.apache.hadoop.conf.Configuration;
import lombok.extern.slf4j.Slf4j;
import java.util.Map; import java.util.Map;
@Slf4j @Slf4j
......
...@@ -17,13 +17,14 @@ ...@@ -17,13 +17,14 @@
package org.apache.seatunnel.datasource.plugin.xml; package org.apache.seatunnel.datasource.plugin.xml;
import com.google.auto.service.AutoService;
import com.google.common.collect.Sets;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
import com.google.auto.service.AutoService;
import com.google.common.collect.Sets;
import java.util.Set; import java.util.Set;
@AutoService(DataSourceFactory.class) @AutoService(DataSourceFactory.class)
......
...@@ -17,18 +17,20 @@ ...@@ -17,18 +17,20 @@
package org.apache.seatunnel.datasource.plugin.xml; package org.apache.seatunnel.datasource.plugin.xml;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSON;
import io.minio.*; import io.minio.*;
import io.minio.errors.*; import io.minio.errors.*;
import io.minio.messages.Bucket; import io.minio.messages.Bucket;
import io.minio.messages.Item; import io.minio.messages.Item;
import lombok.NonNull; import lombok.NonNull;
import org.apache.commons.lang3.StringUtils;
import org.apache.seatunnel.api.configuration.util.OptionRule;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
import org.apache.seatunnel.datasource.plugin.api.model.TableField;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
......
...@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option; ...@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option;
import org.apache.seatunnel.api.configuration.Options; import org.apache.seatunnel.api.configuration.Options;
import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.OptionRule;
import java.util.Arrays;
import java.util.Map; import java.util.Map;
public class XMLOptionRule { public class XMLOptionRule {
...@@ -121,7 +120,7 @@ public class XMLOptionRule { ...@@ -121,7 +120,7 @@ public class XMLOptionRule {
return OptionRule.builder() return OptionRule.builder()
.required(PATH, TYPE) .required(PATH, TYPE)
.conditional(TYPE, FileFormat.XML.type, DELIMITER) .conditional(TYPE, FileFormat.XML.type, DELIMITER)
.conditional(TYPE,FileFormat.XML.type, SCHEMA) .conditional(TYPE, FileFormat.XML.type, SCHEMA)
.optional(PARSE_PARSE_PARTITION_FROM_PATH) .optional(PARSE_PARSE_PARTITION_FROM_PATH)
.optional(DATE_FORMAT) .optional(DATE_FORMAT)
.optional(DATETIME_FORMAT) .optional(DATETIME_FORMAT)
......
...@@ -50,7 +50,7 @@ ...@@ -50,7 +50,7 @@
<module>datasource-redis</module> <module>datasource-redis</module>
<module>datasource-rabbitmq</module> <module>datasource-rabbitmq</module>
<module>datasource-ftp</module> <module>datasource-ftp</module>
<module>datasource-jdbc-demeng</module> <module>datasource-jdbc-dameng</module>
<module>datasource-jdbc-access</module> <module>datasource-jdbc-access</module>
<module>datasource-http</module> <module>datasource-http</module>
<module>datasource-xml</module> <module>datasource-xml</module>
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论