Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
S
seatunnel-web
项目
项目
详情
活动
周期分析
仓库
仓库
文件
提交
分支
标签
贡献者
图表
比较
统计图
议题
0
议题
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
CI / CD
CI / CD
流水线
作业
日程
统计图
Wiki
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
宋勇
seatunnel-web
Commits
d18b47a1
提交
d18b47a1
authored
4月 13, 2024
作者:
宋勇
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
atasource-jdbc-demeng
datasource-jdbc-access datasource-http datasource-xml datasource-csv datasource-excel 增加
上级
373502c0
隐藏空白字符变更
内嵌
并排
正在显示
30 个修改的文件
包含
317 行增加
和
216 行删除
+317
-216
pom.xml
...ource/seatunnel-datasource-plugins/datasource-csv/pom.xml
+1
-2
CSVAConfiguration.java
...he/seatunnel/datasource/plugin/csv/CSVAConfiguration.java
+5
-3
CSVDataSourceFactory.java
...seatunnel/datasource/plugin/csv/CSVDataSourceFactory.java
+4
-3
CSVDatasourceChannel.java
...seatunnel/datasource/plugin/csv/CSVDatasourceChannel.java
+10
-8
CSVOptionRule.java
...apache/seatunnel/datasource/plugin/csv/CSVOptionRule.java
+0
-2
pom.xml
...rce/seatunnel-datasource-plugins/datasource-excel/pom.xml
+1
-2
ExcelAConfiguration.java
...eatunnel/datasource/plugin/excel/ExcelAConfiguration.java
+5
-3
ExcelDataSourceFactory.java
...unnel/datasource/plugin/excel/ExcelDataSourceFactory.java
+4
-3
ExcelDatasourceChannel.java
...unnel/datasource/plugin/excel/ExcelDatasourceChannel.java
+8
-6
ExcelOptionRule.java
...he/seatunnel/datasource/plugin/excel/ExcelOptionRule.java
+0
-1
HttpAConfiguration.java
.../seatunnel/datasource/plugin/http/HttpAConfiguration.java
+2
-4
HttpClientService.java
...e/seatunnel/datasource/plugin/http/HttpClientService.java
+0
-6
HttpConfiguration.java
...e/seatunnel/datasource/plugin/http/HttpConfiguration.java
+3
-5
HttpDatasourceChannel.java
...atunnel/datasource/plugin/http/HttpDatasourceChannel.java
+19
-15
HttpOptionRule.java
...ache/seatunnel/datasource/plugin/http/HttpOptionRule.java
+1
-6
pom.xml
...atunnel-datasource-plugins/datasource-jdbc-access/pom.xml
+10
-0
AccessDataSourceConfig.java
...datasource/plugin/access/jdbc/AccessDataSourceConfig.java
+13
-6
AccessJdbcDataSourceChannel.java
...ource/plugin/access/jdbc/AccessJdbcDataSourceChannel.java
+131
-54
AccessOptionRule.java
...unnel/datasource/plugin/access/jdbc/AccessOptionRule.java
+6
-6
pom.xml
...atunnel-datasource-plugins/datasource-jdbc-dameng/pom.xml
+1
-1
DamengDataSourceConfig.java
...datasource/plugin/demeng/jdbc/DamengDataSourceConfig.java
+8
-7
DamengJdbcDataSourceChannel.java
...ource/plugin/demeng/jdbc/DamengJdbcDataSourceChannel.java
+61
-50
DamengJdbcDataSourceFactory.java
...ource/plugin/demeng/jdbc/DamengJdbcDataSourceFactory.java
+4
-4
DamengOptionRule.java
...unnel/datasource/plugin/demeng/jdbc/DamengOptionRule.java
+2
-4
pom.xml
...ource/seatunnel-datasource-plugins/datasource-xml/pom.xml
+1
-2
XMLAConfiguration.java
...he/seatunnel/datasource/plugin/xml/XMLAConfiguration.java
+4
-2
XMLDataSourceFactory.java
...seatunnel/datasource/plugin/xml/XMLDataSourceFactory.java
+3
-2
XMLDatasourceChannel.java
...seatunnel/datasource/plugin/xml/XMLDatasourceChannel.java
+8
-6
XMLOptionRule.java
...apache/seatunnel/datasource/plugin/xml/XMLOptionRule.java
+1
-2
pom.xml
seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
+1
-1
没有找到文件。
seatunnel-datasource/seatunnel-datasource-plugins/datasource-csv/pom.xml
浏览文件 @
d18b47a1
...
@@ -141,7 +141,6 @@
...
@@ -141,7 +141,6 @@
<artifactId>
aws-java-sdk-bundle
</artifactId>
<artifactId>
aws-java-sdk-bundle
</artifactId>
</dependency>
</dependency>
</dependencies>
</dependencies>
</project>
</project>
seatunnel-datasource/seatunnel-datasource-plugins/datasource-csv/src/main/java/org/apache/seatunnel/datasource/plugin/csv/CSVAConfiguration.java
浏览文件 @
d18b47a1
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
csv
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
csv
;
import
lombok.extern.slf4j.Slf4j
;
import
org.apache.hadoop.conf.Configuration
;
import
org.apache.seatunnel.shade.com.typesafe.config.Config
;
import
org.apache.seatunnel.shade.com.typesafe.config.Config
;
import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
;
import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
;
import
org.apache.hadoop.conf.Configuration
;
import
lombok.extern.slf4j.Slf4j
;
import
java.util.Map
;
import
java.util.Map
;
@Slf4j
@Slf4j
...
@@ -29,7 +31,7 @@ public class CSVAConfiguration {
...
@@ -29,7 +31,7 @@ public class CSVAConfiguration {
throw
new
IllegalArgumentException
(
throw
new
IllegalArgumentException
(
"S3 datasource endpoint is null, please check your config"
);
"S3 datasource endpoint is null, please check your config"
);
}
}
String
bucket
=
s3Options
.
get
(
CSVOptionRule
.
BUCKET
.
key
());
String
bucket
=
s3Options
.
get
(
CSVOptionRule
.
BUCKET
.
key
());
String
protocol
=
DEFAULT_PROTOCOL
;
String
protocol
=
DEFAULT_PROTOCOL
;
if
(
bucket
.
startsWith
(
S3A_PROTOCOL
))
{
if
(
bucket
.
startsWith
(
S3A_PROTOCOL
))
{
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-csv/src/main/java/org/apache/seatunnel/datasource/plugin/csv/CSVDataSourceFactory.java
浏览文件 @
d18b47a1
...
@@ -17,13 +17,14 @@
...
@@ -17,13 +17,14 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
csv
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
csv
;
import
com.google.auto.service.AutoService
;
import
com.google.common.collect.Sets
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceFactory
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceFactory
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo
;
import
org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum
;
import
org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum
;
import
com.google.auto.service.AutoService
;
import
com.google.common.collect.Sets
;
import
java.util.Set
;
import
java.util.Set
;
@AutoService
(
DataSourceFactory
.
class
)
@AutoService
(
DataSourceFactory
.
class
)
...
@@ -52,6 +53,6 @@ public class CSVDataSourceFactory implements DataSourceFactory {
...
@@ -52,6 +53,6 @@ public class CSVDataSourceFactory implements DataSourceFactory {
@Override
@Override
public
DataSourceChannel
createChannel
()
{
public
DataSourceChannel
createChannel
()
{
return
CSVDatasourceChannel
.
getInstance
();
return
CSVDatasourceChannel
.
getInstance
();
}
}
}
}
seatunnel-datasource/seatunnel-datasource-plugins/datasource-csv/src/main/java/org/apache/seatunnel/datasource/plugin/csv/CSVDatasourceChannel.java
浏览文件 @
d18b47a1
...
@@ -17,18 +17,20 @@
...
@@ -17,18 +17,20 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
csv
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
csv
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.common.utils.SeaTunnelException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.commons.lang3.StringUtils
;
import
com.alibaba.fastjson2.JSON
;
import
com.alibaba.fastjson2.JSON
;
import
io.minio.*
;
import
io.minio.*
;
import
io.minio.errors.*
;
import
io.minio.errors.*
;
import
io.minio.messages.Bucket
;
import
io.minio.messages.Bucket
;
import
io.minio.messages.Item
;
import
io.minio.messages.Item
;
import
lombok.NonNull
;
import
lombok.NonNull
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.common.utils.SeaTunnelException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
java.io.BufferedReader
;
import
java.io.BufferedReader
;
import
java.io.IOException
;
import
java.io.IOException
;
...
@@ -293,7 +295,7 @@ public class CSVDatasourceChannel implements DataSourceChannel {
...
@@ -293,7 +295,7 @@ public class CSVDatasourceChannel implements DataSourceChannel {
return
all
;
return
all
;
}
}
public
CSVClientService
createS3Client
(
Map
<
String
,
String
>
requestParams
)
{
public
CSVClientService
createS3Client
(
Map
<
String
,
String
>
requestParams
)
{
int
i
=
requestParams
.
get
(
"fs.s3a.endpoint"
).
lastIndexOf
(
":"
);
int
i
=
requestParams
.
get
(
"fs.s3a.endpoint"
).
lastIndexOf
(
":"
);
String
endpoint
=
requestParams
.
get
(
"fs.s3a.endpoint"
)
+
""
;
String
endpoint
=
requestParams
.
get
(
"fs.s3a.endpoint"
)
+
""
;
Integer
port
=
Integer
port
=
...
@@ -304,7 +306,7 @@ public class CSVDatasourceChannel implements DataSourceChannel {
...
@@ -304,7 +306,7 @@ public class CSVDatasourceChannel implements DataSourceChannel {
String
password
=
requestParams
.
get
(
"secret_key"
)
+
""
;
String
password
=
requestParams
.
get
(
"secret_key"
)
+
""
;
// String bucket = requestParams.get("bucket") + "";
// String bucket = requestParams.get("bucket") + "";
try
{
try
{
s3ClientService
=
new
CSVClientService
(
endpoint
,
provider
,
username
,
password
,
port
);
s3ClientService
=
new
CSVClientService
(
endpoint
,
provider
,
username
,
password
,
port
);
return
s3ClientService
;
return
s3ClientService
;
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
throw
new
SeaTunnelException
(
"创建Mqtt客户端错误!"
);
throw
new
SeaTunnelException
(
"创建Mqtt客户端错误!"
);
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-csv/src/main/java/org/apache/seatunnel/datasource/plugin/csv/CSVOptionRule.java
浏览文件 @
d18b47a1
...
@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option;
...
@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option;
import
org.apache.seatunnel.api.configuration.Options
;
import
org.apache.seatunnel.api.configuration.Options
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
java.util.Arrays
;
import
java.util.Map
;
import
java.util.Map
;
public
class
CSVOptionRule
{
public
class
CSVOptionRule
{
...
@@ -129,7 +128,6 @@ public class CSVOptionRule {
...
@@ -129,7 +128,6 @@ public class CSVOptionRule {
.
build
();
.
build
();
}
}
public
enum
S3aAwsCredentialsProvider
{
public
enum
S3aAwsCredentialsProvider
{
SimpleAWSCredentialsProvider
(
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
),
SimpleAWSCredentialsProvider
(
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
),
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-excel/pom.xml
浏览文件 @
d18b47a1
...
@@ -141,7 +141,6 @@
...
@@ -141,7 +141,6 @@
<artifactId>
aws-java-sdk-bundle
</artifactId>
<artifactId>
aws-java-sdk-bundle
</artifactId>
</dependency>
</dependency>
</dependencies>
</dependencies>
</project>
</project>
seatunnel-datasource/seatunnel-datasource-plugins/datasource-excel/src/main/java/org/apache/seatunnel/datasource/plugin/excel/ExcelAConfiguration.java
浏览文件 @
d18b47a1
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
excel
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
excel
;
import
lombok.extern.slf4j.Slf4j
;
import
org.apache.hadoop.conf.Configuration
;
import
org.apache.seatunnel.shade.com.typesafe.config.Config
;
import
org.apache.seatunnel.shade.com.typesafe.config.Config
;
import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
;
import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
;
import
org.apache.hadoop.conf.Configuration
;
import
lombok.extern.slf4j.Slf4j
;
import
java.util.Map
;
import
java.util.Map
;
@Slf4j
@Slf4j
...
@@ -29,7 +31,7 @@ public class ExcelAConfiguration {
...
@@ -29,7 +31,7 @@ public class ExcelAConfiguration {
throw
new
IllegalArgumentException
(
throw
new
IllegalArgumentException
(
"S3 datasource endpoint is null, please check your config"
);
"S3 datasource endpoint is null, please check your config"
);
}
}
String
bucket
=
s3Options
.
get
(
ExcelOptionRule
.
BUCKET
.
key
());
String
bucket
=
s3Options
.
get
(
ExcelOptionRule
.
BUCKET
.
key
());
String
protocol
=
DEFAULT_PROTOCOL
;
String
protocol
=
DEFAULT_PROTOCOL
;
if
(
bucket
.
startsWith
(
S3A_PROTOCOL
))
{
if
(
bucket
.
startsWith
(
S3A_PROTOCOL
))
{
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-excel/src/main/java/org/apache/seatunnel/datasource/plugin/excel/ExcelDataSourceFactory.java
浏览文件 @
d18b47a1
...
@@ -17,13 +17,14 @@
...
@@ -17,13 +17,14 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
excel
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
excel
;
import
com.google.auto.service.AutoService
;
import
com.google.common.collect.Sets
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceFactory
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceFactory
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo
;
import
org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum
;
import
org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum
;
import
com.google.auto.service.AutoService
;
import
com.google.common.collect.Sets
;
import
java.util.Set
;
import
java.util.Set
;
@AutoService
(
DataSourceFactory
.
class
)
@AutoService
(
DataSourceFactory
.
class
)
...
@@ -52,6 +53,6 @@ public class ExcelDataSourceFactory implements DataSourceFactory {
...
@@ -52,6 +53,6 @@ public class ExcelDataSourceFactory implements DataSourceFactory {
@Override
@Override
public
DataSourceChannel
createChannel
()
{
public
DataSourceChannel
createChannel
()
{
return
ExcelDatasourceChannel
.
getInstance
();
return
ExcelDatasourceChannel
.
getInstance
();
}
}
}
}
seatunnel-datasource/seatunnel-datasource-plugins/datasource-excel/src/main/java/org/apache/seatunnel/datasource/plugin/excel/ExcelDatasourceChannel.java
浏览文件 @
d18b47a1
...
@@ -17,18 +17,20 @@
...
@@ -17,18 +17,20 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
excel
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
excel
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.common.utils.SeaTunnelException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.commons.lang3.StringUtils
;
import
com.alibaba.fastjson2.JSON
;
import
com.alibaba.fastjson2.JSON
;
import
io.minio.*
;
import
io.minio.*
;
import
io.minio.errors.*
;
import
io.minio.errors.*
;
import
io.minio.messages.Bucket
;
import
io.minio.messages.Bucket
;
import
io.minio.messages.Item
;
import
io.minio.messages.Item
;
import
lombok.NonNull
;
import
lombok.NonNull
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.common.utils.SeaTunnelException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
java.io.BufferedReader
;
import
java.io.BufferedReader
;
import
java.io.IOException
;
import
java.io.IOException
;
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-excel/src/main/java/org/apache/seatunnel/datasource/plugin/excel/ExcelOptionRule.java
浏览文件 @
d18b47a1
...
@@ -129,7 +129,6 @@ public class ExcelOptionRule {
...
@@ -129,7 +129,6 @@ public class ExcelOptionRule {
.
build
();
.
build
();
}
}
public
enum
S3aAwsCredentialsProvider
{
public
enum
S3aAwsCredentialsProvider
{
SimpleAWSCredentialsProvider
(
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
),
SimpleAWSCredentialsProvider
(
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
),
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-http/src/main/java/org/apache/seatunnel/datasource/plugin/http/HttpAConfiguration.java
浏览文件 @
d18b47a1
...
@@ -27,17 +27,15 @@ public class HttpAConfiguration {
...
@@ -27,17 +27,15 @@ public class HttpAConfiguration {
public
static
HttpConfiguration
getConfiguration
(
Map
<
String
,
String
>
ftpOption
)
{
public
static
HttpConfiguration
getConfiguration
(
Map
<
String
,
String
>
ftpOption
)
{
if
(!
ftpOption
.
containsKey
(
HttpOptionRule
.
URL
.
key
()))
{
if
(!
ftpOption
.
containsKey
(
HttpOptionRule
.
URL
.
key
()))
{
throw
new
IllegalArgumentException
(
throw
new
IllegalArgumentException
(
"url is null, please check your config"
);
"url is null, please check your config"
);
}
}
HttpConfiguration
httpAConfiguration
=
new
HttpConfiguration
();
HttpConfiguration
httpAConfiguration
=
new
HttpConfiguration
();
httpAConfiguration
.
setUrl
(
HttpOptionRule
.
URL
.
key
());
httpAConfiguration
.
setUrl
(
HttpOptionRule
.
URL
.
key
());
httpAConfiguration
.
setMethod
(
HttpOptionRule
.
METHOD
.
key
());
httpAConfiguration
.
setMethod
(
HttpOptionRule
.
METHOD
.
key
());
httpAConfiguration
.
setToken
(
HttpOptionRule
.
TOKEN
.
key
());
httpAConfiguration
.
setToken
(
HttpOptionRule
.
TOKEN
.
key
());
httpAConfiguration
.
setRequest_params
(
HttpOptionRule
.
REQUEST_PARAMS
.
key
());
httpAConfiguration
.
setRequest_params
(
HttpOptionRule
.
REQUEST_PARAMS
.
key
());
return
httpAConfiguration
;
return
httpAConfiguration
;
}
}
}
}
seatunnel-datasource/seatunnel-datasource-plugins/datasource-http/src/main/java/org/apache/seatunnel/datasource/plugin/http/HttpClientService.java
浏览文件 @
d18b47a1
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
http
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
http
;
import
org.apache.http.client.HttpClient
;
import
org.apache.http.client.HttpClient
;
import
org.apache.http.impl.client.HttpClients
;
import
org.apache.http.impl.client.HttpClients
;
...
@@ -8,14 +7,9 @@ public class HttpClientService {
...
@@ -8,14 +7,9 @@ public class HttpClientService {
public
static
HttpClient
connect
(
HttpConfiguration
conf
)
throws
Exception
{
public
static
HttpClient
connect
(
HttpConfiguration
conf
)
throws
Exception
{
// 创建HttpClient实例
// 创建HttpClient实例
HttpClient
client
=
HttpClients
.
createDefault
();
HttpClient
client
=
HttpClients
.
createDefault
();
return
client
;
return
client
;
}
}
}
}
seatunnel-datasource/seatunnel-datasource-plugins/datasource-http/src/main/java/org/apache/seatunnel/datasource/plugin/http/HttpConfiguration.java
浏览文件 @
d18b47a1
...
@@ -10,12 +10,11 @@ public class HttpConfiguration {
...
@@ -10,12 +10,11 @@ public class HttpConfiguration {
public
HttpConfiguration
()
{}
public
HttpConfiguration
()
{}
public
HttpConfiguration
(
public
HttpConfiguration
(
String
url
,
String
method
,
String
token
,
String
request_params
)
{
String
url
,
String
method
,
String
token
,
String
request_params
)
{
this
.
url
=
url
;
this
.
url
=
url
;
this
.
token
=
token
;
this
.
token
=
token
;
this
.
method
=
method
;
this
.
method
=
method
;
this
.
request_params
=
request_params
;
this
.
request_params
=
request_params
;
}
}
public
String
getUrl
()
{
public
String
getUrl
()
{
...
@@ -26,7 +25,6 @@ public class HttpConfiguration {
...
@@ -26,7 +25,6 @@ public class HttpConfiguration {
this
.
url
=
url
;
this
.
url
=
url
;
}
}
public
String
getMethod
()
{
public
String
getMethod
()
{
return
method
;
return
method
;
}
}
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-http/src/main/java/org/apache/seatunnel/datasource/plugin/http/HttpDatasourceChannel.java
浏览文件 @
d18b47a1
...
@@ -17,23 +17,21 @@
...
@@ -17,23 +17,21 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
http
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
http
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.client.HttpClient
;
import
org.apache.http.client.HttpClient
;
import
org.apache.http.client.methods.*
;
import
org.apache.http.client.methods.*
;
import
org.apache.http.entity.StringEntity
;
import
org.apache.http.entity.StringEntity
;
import
org.apache.http.util.EntityUtils
;
import
org.apache.http.util.EntityUtils
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
lombok.NonNull
;
import
lombok.NonNull
;
import
java.net.URI
;
import
java.net.URI
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
java.util.Objects
;
import
java.util.Objects
;
...
@@ -75,7 +73,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -75,7 +73,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
System
.
out
.
println
(
"url:"
+
url
);
System
.
out
.
println
(
"url:"
+
url
);
HttpGet
httpGet
=
new
HttpGet
(
url
);
HttpGet
httpGet
=
new
HttpGet
(
url
);
if
(
StringUtils
.
isNotBlank
(
token
))
{
if
(
StringUtils
.
isNotBlank
(
token
))
{
httpGet
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
httpGet
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
}
}
// 执行请求并获得响应
// 执行请求并获得响应
...
@@ -83,7 +82,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -83,7 +82,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
}
else
if
(
StringUtils
.
isBlank
(
method
)
||
"POST"
.
equals
(
method
.
toUpperCase
()))
{
}
else
if
(
StringUtils
.
isBlank
(
method
)
||
"POST"
.
equals
(
method
.
toUpperCase
()))
{
HttpPost
httpPost
=
new
HttpPost
(
url
);
HttpPost
httpPost
=
new
HttpPost
(
url
);
if
(
StringUtils
.
isNotBlank
(
token
))
{
if
(
StringUtils
.
isNotBlank
(
token
))
{
httpPost
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
httpPost
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
}
}
// 设置请求体(例如:JSON数据)
// 设置请求体(例如:JSON数据)
StringEntity
requestEntity
=
new
StringEntity
(
parmams
);
StringEntity
requestEntity
=
new
StringEntity
(
parmams
);
...
@@ -94,7 +94,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -94,7 +94,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
}
else
if
(
StringUtils
.
isBlank
(
method
)
||
"PUT"
.
equals
(
method
.
toUpperCase
()))
{
}
else
if
(
StringUtils
.
isBlank
(
method
)
||
"PUT"
.
equals
(
method
.
toUpperCase
()))
{
HttpPut
httpPut
=
new
HttpPut
(
url
);
HttpPut
httpPut
=
new
HttpPut
(
url
);
if
(
StringUtils
.
isNotBlank
(
token
))
{
if
(
StringUtils
.
isNotBlank
(
token
))
{
httpPut
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
httpPut
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
}
}
// 设置请求体(例如:JSON数据)
// 设置请求体(例如:JSON数据)
StringEntity
requestEntity
=
new
StringEntity
(
parmams
);
StringEntity
requestEntity
=
new
StringEntity
(
parmams
);
...
@@ -110,7 +111,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -110,7 +111,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
HttpDelete
httpDelete
=
new
HttpDelete
(
url
);
HttpDelete
httpDelete
=
new
HttpDelete
(
url
);
if
(
StringUtils
.
isNotBlank
(
token
))
{
if
(
StringUtils
.
isNotBlank
(
token
))
{
httpDelete
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
httpDelete
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
}
}
// 执行请求并获得响应
// 执行请求并获得响应
...
@@ -119,7 +121,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -119,7 +121,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
}
else
if
(
StringUtils
.
isBlank
(
method
)
||
"PATCH"
.
equals
(
method
.
toUpperCase
()))
{
}
else
if
(
StringUtils
.
isBlank
(
method
)
||
"PATCH"
.
equals
(
method
.
toUpperCase
()))
{
HttpPatch
httpPatch
=
new
HttpPatch
(
url
);
HttpPatch
httpPatch
=
new
HttpPatch
(
url
);
if
(
StringUtils
.
isNotBlank
(
token
))
{
if
(
StringUtils
.
isNotBlank
(
token
))
{
httpPatch
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
httpPatch
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
}
}
// 设置请求体(例如:JSON数据)
// 设置请求体(例如:JSON数据)
StringEntity
requestEntity
=
new
StringEntity
(
parmams
);
StringEntity
requestEntity
=
new
StringEntity
(
parmams
);
...
@@ -133,7 +136,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -133,7 +136,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
}
}
HttpOptions
httpOptions
=
new
HttpOptions
(
url
);
HttpOptions
httpOptions
=
new
HttpOptions
(
url
);
if
(
StringUtils
.
isNotBlank
(
token
))
{
if
(
StringUtils
.
isNotBlank
(
token
))
{
httpOptions
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
httpOptions
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
}
}
// 执行请求并获得响应
// 执行请求并获得响应
...
@@ -145,7 +149,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -145,7 +149,8 @@ public class HttpDatasourceChannel implements DataSourceChannel {
}
}
HttpHead
httpHead
=
new
HttpHead
(
url
);
HttpHead
httpHead
=
new
HttpHead
(
url
);
if
(
StringUtils
.
isNotBlank
(
token
))
{
if
(
StringUtils
.
isNotBlank
(
token
))
{
httpHead
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
httpHead
.
setHeader
(
"Authorization"
,
"Bearer "
+
token
.
replace
(
"Bearer "
,
""
).
trim
());
}
}
// 执行请求并获得响应
// 执行请求并获得响应
...
@@ -160,7 +165,6 @@ public class HttpDatasourceChannel implements DataSourceChannel {
...
@@ -160,7 +165,6 @@ public class HttpDatasourceChannel implements DataSourceChannel {
String
responseBody
=
EntityUtils
.
toString
(
response
.
getEntity
());
String
responseBody
=
EntityUtils
.
toString
(
response
.
getEntity
());
System
.
out
.
println
(
"Response Body: "
+
responseBody
);
System
.
out
.
println
(
"Response Body: "
+
responseBody
);
if
(
statusCode
==
200
)
{
if
(
statusCode
==
200
)
{
return
true
;
return
true
;
}
else
{
}
else
{
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-http/src/main/java/org/apache/seatunnel/datasource/plugin/http/HttpOptionRule.java
浏览文件 @
d18b47a1
...
@@ -47,11 +47,8 @@ public class HttpOptionRule {
...
@@ -47,11 +47,8 @@ public class HttpOptionRule {
.
noDefaultValue
()
.
noDefaultValue
()
.
withDescription
(
"the http user token to use when connecting to the broker"
);
.
withDescription
(
"the http user token to use when connecting to the broker"
);
public
static
OptionRule
optionRule
()
{
public
static
OptionRule
optionRule
()
{
return
OptionRule
.
builder
().
required
(
URL
,
METHOD
).
optional
(
TOKEN
,
REQUEST_PARAMS
).
build
();
return
OptionRule
.
builder
().
required
(
URL
,
METHOD
).
optional
(
TOKEN
,
REQUEST_PARAMS
).
build
();
}
}
public
static
OptionRule
metadataRule
()
{
public
static
OptionRule
metadataRule
()
{
...
@@ -59,8 +56,6 @@ public class HttpOptionRule {
...
@@ -59,8 +56,6 @@ public class HttpOptionRule {
}
}
public
enum
FileFormat
{
public
enum
FileFormat
{
JSON
(
"json"
),
JSON
(
"json"
),
;
;
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-access/pom.xml
浏览文件 @
d18b47a1
...
@@ -56,7 +56,17 @@
...
@@ -56,7 +56,17 @@
<artifactId>
ucanaccess
</artifactId>
<artifactId>
ucanaccess
</artifactId>
<version>
5.0.1
</version>
<version>
5.0.1
</version>
</dependency>
</dependency>
<dependency>
<groupId>
com.cdzhiyong.unified
</groupId>
<artifactId>
minio-spring-boot-starter
</artifactId>
<version>
1.0.0
</version>
</dependency>
<dependency>
<groupId>
org.apache.httpcomponents
</groupId>
<artifactId>
httpclient
</artifactId>
<version>
4.5.14
</version>
</dependency>
</dependencies>
</dependencies>
</project>
</project>
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-access/src/main/java/org/apache/seatunnel/datasource/plugin/access/jdbc/AccessDataSourceConfig.java
浏览文件 @
d18b47a1
...
@@ -37,15 +37,22 @@ public class AccessDataSourceConfig {
...
@@ -37,15 +37,22 @@ public class AccessDataSourceConfig {
.
type
(
DatasourcePluginTypeEnum
.
DATABASE
.
getCode
())
.
type
(
DatasourcePluginTypeEnum
.
DATABASE
.
getCode
())
.
build
();
.
build
();
public
static
final
Set
<
String
>
MYSQL_SYSTEM_DATABASES
=
public
static
final
Set
<
String
>
MYSQL_SYSTEM_DATABASES
=
Sets
.
newHashSet
(
"SYSTEM"
,
"ROLL"
);
Sets
.
newHashSet
(
"SYSTEM"
,
"ROLL"
);
public
static
final
OptionRule
OPTION_RULE
=
public
static
final
OptionRule
OPTION_RULE
=
OptionRule
.
builder
()
OptionRule
.
builder
()
.
required
(
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
URL
,
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
DRIVER
)
.
required
(
.
optional
(
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
USER
,
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
PASSWORD
)
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
URL
,
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
DRIVER
)
.
optional
(
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
USER
,
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
PASSWORD
)
.
build
();
.
build
();
// public static final OptionRule METADATA_RULE =
// public static final OptionRule METADATA_RULE =
// OptionRule.builder().required(org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.DATABASE, org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.TABLE).build();
//
// OptionRule.builder().required(org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.DATABASE, org.apache.seatunnel.datasource.plugin.demeng.jdbc.DemengOptionRule.TABLE).build();
}
}
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-access/src/main/java/org/apache/seatunnel/datasource/plugin/access/jdbc/AccessJdbcDataSourceChannel.java
浏览文件 @
d18b47a1
...
@@ -17,17 +17,21 @@
...
@@ -17,17 +17,21 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannelExt
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannelExt
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
lombok.NonNull
;
import
lombok.NonNull
;
import
java.io.File
;
import
java.io.FileOutputStream
;
import
java.io.InputStream
;
import
java.net.HttpURLConnection
;
import
java.net.URL
;
import
java.sql.*
;
import
java.sql.*
;
import
java.util.*
;
import
java.util.*
;
import
java.util.function.Function
;
import
java.util.function.Function
;
...
@@ -39,7 +43,8 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -39,7 +43,8 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
List
<
String
>
typeList
=
Arrays
.
asList
(
"varchar"
,
"char"
,
"json"
);
List
<
String
>
typeList
=
Arrays
.
asList
(
"varchar"
,
"char"
,
"json"
);
public
static
class
Holder
{
public
static
class
Holder
{
private
static
final
AccessJdbcDataSourceChannel
INSTANCE
=
new
AccessJdbcDataSourceChannel
();
private
static
final
AccessJdbcDataSourceChannel
INSTANCE
=
new
AccessJdbcDataSourceChannel
();
}
}
public
static
AccessJdbcDataSourceChannel
getInstance
()
{
public
static
AccessJdbcDataSourceChannel
getInstance
()
{
...
@@ -60,17 +65,15 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -60,17 +65,15 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
List
<
Map
>
reList
=
new
ArrayList
<>();
List
<
Map
>
reList
=
new
ArrayList
<>();
ResultSetMetaData
rsmd
=
rs
.
getMetaData
();
ResultSetMetaData
rsmd
=
rs
.
getMetaData
();
int
columnCount
=
rsmd
.
getColumnCount
();
//
ResultSet的总列数
int
columnCount
=
rsmd
.
getColumnCount
();
//
ResultSet的总列数
while
(
rs
.
next
())
{
while
(
rs
.
next
())
{
Map
map
=
new
HashMap
();
Map
map
=
new
HashMap
();
for
(
int
i
=
1
;
i
<=
columnCount
;
i
++)
{
for
(
int
i
=
1
;
i
<=
columnCount
;
i
++)
{
Object
value
=
rs
.
getObject
(
i
);
Object
value
=
rs
.
getObject
(
i
);
String
columnName
=
rsmd
.
getColumnName
(
i
);
String
columnName
=
rsmd
.
getColumnName
(
i
);
map
.
put
(
columnName
,
value
);
map
.
put
(
columnName
,
value
);
}
}
reList
.
add
(
map
);
reList
.
add
(
map
);
}
}
return
reList
;
return
reList
;
}
}
...
@@ -87,36 +90,41 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -87,36 +90,41 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
// 定义结果集对象
// 定义结果集对象
ResultSet
rs
=
null
;
ResultSet
rs
=
null
;
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
String
sql_selectAll
=
"SELECT CAST(null AS VARCHAR(50)) AS TABLE_CAT,\n"
+
String
sql_selectAll
=
"CAST(null AS VARCHAR(50)) AS TABLE_SCHEM,\n"
+
"SELECT CAST(null AS VARCHAR(50)) AS TABLE_CAT,\n"
"r.TABLE_NAME AS TABLE_NAME,\n"
+
+
"CAST(null AS VARCHAR(50)) AS TABLE_SCHEM,\n"
"l.TABLE_TYPE,\n"
+
+
"r.TABLE_NAME AS TABLE_NAME,\n"
"l.REMARKS,\n"
+
+
"l.TABLE_TYPE,\n"
"l.TYPE_CAT,\n"
+
+
"l.REMARKS,\n"
"l.TYPE_SCHEM,\n"
+
+
"l.TYPE_CAT,\n"
"l.TYPE_NAME,\n"
+
+
"l.TYPE_SCHEM,\n"
"l.SELF_REFERENCING_COL_NAME,\n"
+
+
"l.TYPE_NAME,\n"
"l.REF_GENERATION,\n"
+
+
"l.SELF_REFERENCING_COL_NAME,\n"
"l.HSQLDB_TYPE,\n"
+
+
"l.REF_GENERATION,\n"
"l.READ_ONLY,\n"
+
+
"l.HSQLDB_TYPE,\n"
"l.COMMIT_ACTION \n"
+
+
"l.READ_ONLY,\n"
"FROM INFORMATION_SCHEMA.SYSTEM_TABLES l \n"
+
+
"l.COMMIT_ACTION \n"
"INNER JOIN UCA_METADATA.TABLES r \n"
+
+
"FROM INFORMATION_SCHEMA.SYSTEM_TABLES l \n"
"ON( l.TABLE_NAME= r.ESCAPED_TABLE_NAME) \n"
+
+
"INNER JOIN UCA_METADATA.TABLES r \n"
"WHERE \n"
+
+
"ON( l.TABLE_NAME= r.ESCAPED_TABLE_NAME) \n"
"TABLE_CAT = 'PUBLIC' AND TABLE_SCHEM = 'PUBLIC' AND TABLE_NAME LIKE '%' ESCAPE '\\';"
;
+
"WHERE \n"
+
"TABLE_CAT = 'PUBLIC' AND TABLE_SCHEM = 'PUBLIC' AND TABLE_NAME LIKE '%' ESCAPE '\\';"
;
state
=
connection
.
createStatement
();
state
=
connection
.
createStatement
();
//执行查询的 SQL 语句
//
执行查询的 SQL 语句
rs
=
state
.
executeQuery
(
sql_selectAll
);
rs
=
state
.
executeQuery
(
sql_selectAll
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
tableNames
=
resultSet
.
stream
().
filter
(
f
->
f
.
get
(
"TABLE_NAME"
)
!=
null
).
map
(
m
->
m
.
get
(
"TABLE_NAME"
)
+
""
).
collect
(
Collectors
.
toList
());
tableNames
=
resultSet
.
stream
()
.
filter
(
f
->
f
.
get
(
"TABLE_NAME"
)
!=
null
)
.
map
(
m
->
m
.
get
(
"TABLE_NAME"
)
+
""
)
.
collect
(
Collectors
.
toList
());
return
tableNames
;
return
tableNames
;
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
}
finally
{
}
finally
{
try
{
try
{
//关闭资源
//
关闭资源
rs
.
close
();
rs
.
close
();
state
.
close
();
state
.
close
();
}
catch
(
SQLException
e
)
{
}
catch
(
SQLException
e
)
{
...
@@ -131,7 +139,6 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -131,7 +139,6 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
List
<
String
>
tableNames
=
new
ArrayList
<>();
List
<
String
>
tableNames
=
new
ArrayList
<>();
tableNames
.
add
(
"all"
);
tableNames
.
add
(
"all"
);
return
tableNames
;
return
tableNames
;
}
}
@Override
@Override
...
@@ -156,7 +163,9 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -156,7 +163,9 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
String
primaryKey
=
getPrimaryKey
(
metaData
,
database
,
table
);
String
primaryKey
=
getPrimaryKey
(
metaData
,
database
,
table
);
String
sql
=
String
sql
=
"select * from INFORMATION_SCHEMA.SYSTEM_columns where TABLE_NAME='"
+
table
.
toUpperCase
()
+
"' "
;
"select * from INFORMATION_SCHEMA.SYSTEM_columns where TABLE_NAME='"
+
table
.
toUpperCase
()
+
"' "
;
Statement
statement
=
connection
.
createStatement
();
Statement
statement
=
connection
.
createStatement
();
ResultSet
resultSet
=
statement
.
executeQuery
(
sql
);
ResultSet
resultSet
=
statement
.
executeQuery
(
sql
);
while
(
resultSet
.
next
())
{
while
(
resultSet
.
next
())
{
...
@@ -176,7 +185,7 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -176,7 +185,7 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
// tableField.setScale(resultSet.getString("SCALE"));
// tableField.setScale(resultSet.getString("SCALE"));
}
}
String
extra
=
resultSet
.
getString
(
"IS_AUTOINCREMENT"
);
String
extra
=
resultSet
.
getString
(
"IS_AUTOINCREMENT"
);
if
(
StringUtils
.
isNotBlank
(
extra
)
&&
"YES"
.
equals
(
extra
.
trim
().
toUpperCase
()))
{
if
(
StringUtils
.
isNotBlank
(
extra
)
&&
"YES"
.
equals
(
extra
.
trim
().
toUpperCase
()))
{
tableField
.
setHasAutoIncrement
(
true
);
tableField
.
setHasAutoIncrement
(
true
);
}
}
...
@@ -230,15 +239,82 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -230,15 +239,82 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
private
Connection
getConnection
(
Map
<
String
,
String
>
requestParams
,
String
databaseName
)
private
Connection
getConnection
(
Map
<
String
,
String
>
requestParams
,
String
databaseName
)
throws
SQLException
,
ClassNotFoundException
{
throws
SQLException
,
ClassNotFoundException
{
// checkNotNull(requestParams.get(org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule.DRIVER.key()));
// checkNotNull(requestParams.get(org.apache.seatunnel.datasource.plugin.access.jdbc.AccessOptionRule.DRIVER.key()));
checkNotNull
(
requestParams
.
get
(
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
URL
.
key
()),
"Jdbc url cannot be null"
);
checkNotNull
(
requestParams
.
get
(
AccessOptionRule
.
URL
.
key
()),
"Jdbc url cannot be null"
);
String
url
=
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
access
.
jdbc
.
AccessOptionRule
.
URL
.
key
();
String
url
=
AccessOptionRule
.
URL
.
key
();
String
substring
=
url
.
substring
(
url
.
lastIndexOf
(
"/"
),
url
.
length
()).
replace
(
"/"
,
""
);
if
(
StringUtils
.
isBlank
(
substring
))
{
throw
new
RuntimeException
(
"没找到文件名称!"
);
}
String
[]
split
=
substring
.
split
(
"\\."
);
String
prefix
=
split
[
0
];
String
suffix
=
""
;
if
(
split
.
length
>
1
)
{
suffix
=
"."
+
split
[
1
];
}
// 定义 SQL 语句执行对象
Statement
state
=
null
;
// 定义结果集对象
ResultSet
rs
=
null
;
Connection
conn
=
null
;
File
path
=
new
File
(
"/tmp"
);
if
(!
path
.
exists
())
{
path
.
mkdir
();
}
FileOutputStream
outputStream
=
null
;
InputStream
in
=
null
;
HttpURLConnection
con
=
null
;
File
file
=
null
;
try
{
file
=
File
.
createTempFile
(
prefix
,
suffix
,
path
);
outputStream
=
new
FileOutputStream
(
file
);
URL
urldownl
=
new
URL
(
url
);
con
=
(
HttpURLConnection
)
urldownl
.
openConnection
();
con
.
setRequestMethod
(
"GET"
);
in
=
con
.
getInputStream
();
byte
[]
buffer
=
new
byte
[
1024
];
int
bytesRead
;
while
((
bytesRead
=
in
.
read
(
buffer
))
!=
-
1
)
{
outputStream
.
write
(
buffer
,
0
,
bytesRead
);
}
outputStream
.
flush
();
}
catch
(
Exception
e
)
{
System
.
out
.
println
(
"读取url 文件失败!"
+
e
);
throw
new
RuntimeException
(
"读取url 文件失败!"
);
}
finally
{
try
{
if
(
in
!=
null
)
{
in
.
close
();
}
if
(
con
!=
null
)
{
con
.
disconnect
();
}
if
(
outputStream
!=
null
)
{
outputStream
.
close
();
}
}
catch
(
Exception
ee
)
{
System
.
out
.
println
(
"关闭流错误!"
+
ee
);
}
}
String
database
=
path
+
"/"
+
file
.
getName
();
// JDBC连接字符串
// 在application.properties或application.yml文件中配置数据库连接信息。你需要指定驱动类名为net.ucanaccess.jdbc.UcanaccessDriver,并提供Access数据库的URL。例如,URL可以是jdbc:ucanaccess://D:/Access2003/database/db_test.mdb;openExclusive=false;ignoreCase=true。[1]
String
connectionString
=
"jdbc:ucanaccess://"
+
database
+
";openExclusive=false;ignoreCase=true"
;
// 加载UCanAccess JDBC驱动
// 加载UCanAccess JDBC驱动
Class
.
forName
(
"net.ucanaccess.jdbc.UcanaccessDriver"
);
Class
.
forName
(
"net.ucanaccess.jdbc.UcanaccessDriver"
);
// 建立连接
// 建立连接
Connection
conn
=
DriverManager
.
getConnection
(
url
);
conn
=
DriverManager
.
getConnection
(
url
);
System
.
out
.
println
(
"Connected to the database successfully"
);
System
.
out
.
println
(
"Connected to the database successfully"
);
return
conn
;
return
conn
;
}
}
...
@@ -255,26 +331,27 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -255,26 +331,27 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
// 定义结果集对象
// 定义结果集对象
ResultSet
rs
=
null
;
ResultSet
rs
=
null
;
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
String
sql_selectAll
=
"SELECT CAST(null AS VARCHAR(50)) AS TABLE_CAT,\n"
+
String
sql_selectAll
=
"CAST(null AS VARCHAR(50)) AS TABLE_SCHEM,\n"
+
"SELECT CAST(null AS VARCHAR(50)) AS TABLE_CAT,\n"
"r.TABLE_NAME AS TABLE_NAME,\n"
+
+
"CAST(null AS VARCHAR(50)) AS TABLE_SCHEM,\n"
"l.TABLE_TYPE,\n"
+
+
"r.TABLE_NAME AS TABLE_NAME,\n"
"l.REMARKS,\n"
+
+
"l.TABLE_TYPE,\n"
"l.TYPE_CAT,\n"
+
+
"l.REMARKS,\n"
"l.TYPE_SCHEM,\n"
+
+
"l.TYPE_CAT,\n"
"l.TYPE_NAME,\n"
+
+
"l.TYPE_SCHEM,\n"
"l.SELF_REFERENCING_COL_NAME,\n"
+
+
"l.TYPE_NAME,\n"
"l.REF_GENERATION,\n"
+
+
"l.SELF_REFERENCING_COL_NAME,\n"
"l.HSQLDB_TYPE,\n"
+
+
"l.REF_GENERATION,\n"
"l.READ_ONLY,\n"
+
+
"l.HSQLDB_TYPE,\n"
"l.COMMIT_ACTION \n"
+
+
"l.READ_ONLY,\n"
"FROM INFORMATION_SCHEMA.SYSTEM_TABLES l \n"
+
+
"l.COMMIT_ACTION \n"
"INNER JOIN UCA_METADATA.TABLES r \n"
+
+
"FROM INFORMATION_SCHEMA.SYSTEM_TABLES l \n"
"ON( l.TABLE_NAME= r.ESCAPED_TABLE_NAME) \n"
+
+
"INNER JOIN UCA_METADATA.TABLES r \n"
"WHERE \n"
+
+
"ON( l.TABLE_NAME= r.ESCAPED_TABLE_NAME) \n"
"TABLE_CAT = 'PUBLIC' AND TABLE_SCHEM = 'PUBLIC' AND TABLE_NAME LIKE '%' ESCAPE '\\';"
;
+
"WHERE \n"
+
"TABLE_CAT = 'PUBLIC' AND TABLE_SCHEM = 'PUBLIC' AND TABLE_NAME LIKE '%' ESCAPE '\\';"
;
state
=
connection
.
createStatement
();
state
=
connection
.
createStatement
();
//执行查询的 SQL 语句
//
执行查询的 SQL 语句
rs
=
state
.
executeQuery
(
sql_selectAll
);
rs
=
state
.
executeQuery
(
sql_selectAll
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
...
@@ -283,7 +360,7 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -283,7 +360,7 @@ public class AccessJdbcDataSourceChannel implements DataSourceChannelExt {
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
}
finally
{
}
finally
{
try
{
try
{
//关闭资源
//
关闭资源
rs
.
close
();
rs
.
close
();
state
.
close
();
state
.
close
();
}
catch
(
SQLException
e
)
{
}
catch
(
SQLException
e
)
{
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-access/src/main/java/org/apache/seatunnel/datasource/plugin/access/jdbc/AccessOptionRule.java
浏览文件 @
d18b47a1
...
@@ -27,8 +27,7 @@ public class AccessOptionRule {
...
@@ -27,8 +27,7 @@ public class AccessOptionRule {
.
stringType
()
.
stringType
()
.
noDefaultValue
()
.
noDefaultValue
()
.
withDescription
(
.
withDescription
(
"jdbc url, eg:"
"jdbc url, eg:"
+
" http://localhost:9000/bucket/filename.mdb"
);
+
" http://localhost:9000/bucket/filename.mdb"
);
public
static
final
Option
<
String
>
USER
=
public
static
final
Option
<
String
>
USER
=
Options
.
key
(
"user"
).
stringType
().
noDefaultValue
().
withDescription
(
"jdbc user"
);
Options
.
key
(
"user"
).
stringType
().
noDefaultValue
().
withDescription
(
"jdbc user"
);
...
@@ -36,11 +35,12 @@ public class AccessOptionRule {
...
@@ -36,11 +35,12 @@ public class AccessOptionRule {
public
static
final
Option
<
String
>
PASSWORD
=
public
static
final
Option
<
String
>
PASSWORD
=
Options
.
key
(
"password"
).
stringType
().
noDefaultValue
().
withDescription
(
"jdbc password"
);
Options
.
key
(
"password"
).
stringType
().
noDefaultValue
().
withDescription
(
"jdbc password"
);
// public static final Option<String> DATABASE =
// public static final Option<String> DATABASE =
// Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
// Options.key("database").stringType().noDefaultValue().withDescription("jdbc
// database");
// public static final Option<String> TABLE =
// public static final Option<String> TABLE =
// Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
// Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
public
static
final
Option
<
DriverType
>
DRIVER
=
public
static
final
Option
<
DriverType
>
DRIVER
=
Options
.
key
(
"driver"
)
Options
.
key
(
"driver"
)
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
e
meng/pom.xml
→
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
a
meng/pom.xml
浏览文件 @
d18b47a1
...
@@ -22,7 +22,7 @@
...
@@ -22,7 +22,7 @@
<version>
1.0.0-SNAPSHOT
</version>
<version>
1.0.0-SNAPSHOT
</version>
</parent>
</parent>
<artifactId>
datasource-jdbc-d
e
meng
</artifactId>
<artifactId>
datasource-jdbc-d
a
meng
</artifactId>
<properties>
<properties>
<mysql-connector.version>
8.0.28
</mysql-connector.version>
<mysql-connector.version>
8.0.28
</mysql-connector.version>
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
emeng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/De
mengDataSourceConfig.java
→
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
ameng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/Da
mengDataSourceConfig.java
浏览文件 @
d18b47a1
...
@@ -25,9 +25,9 @@ import com.google.common.collect.Sets;
...
@@ -25,9 +25,9 @@ import com.google.common.collect.Sets;
import
java.util.Set
;
import
java.util.Set
;
public
class
D
e
mengDataSourceConfig
{
public
class
D
a
mengDataSourceConfig
{
public
static
final
String
PLUGIN_NAME
=
"JDBC-D
e
meng"
;
public
static
final
String
PLUGIN_NAME
=
"JDBC-D
a
meng"
;
public
static
final
DataSourcePluginInfo
MYSQL_DATASOURCE_PLUGIN_INFO
=
public
static
final
DataSourcePluginInfo
MYSQL_DATASOURCE_PLUGIN_INFO
=
DataSourcePluginInfo
.
builder
()
DataSourcePluginInfo
.
builder
()
...
@@ -37,15 +37,16 @@ public class DemengDataSourceConfig {
...
@@ -37,15 +37,16 @@ public class DemengDataSourceConfig {
.
type
(
DatasourcePluginTypeEnum
.
DATABASE
.
getCode
())
.
type
(
DatasourcePluginTypeEnum
.
DATABASE
.
getCode
())
.
build
();
.
build
();
public
static
final
Set
<
String
>
MYSQL_SYSTEM_DATABASES
=
public
static
final
Set
<
String
>
MYSQL_SYSTEM_DATABASES
=
Sets
.
newHashSet
(
"SYSTEM"
,
"ROLL"
);
Sets
.
newHashSet
(
"SYSTEM"
,
"ROLL"
);
public
static
final
OptionRule
OPTION_RULE
=
public
static
final
OptionRule
OPTION_RULE
=
OptionRule
.
builder
()
OptionRule
.
builder
()
.
required
(
D
emengOptionRule
.
URL
,
De
mengOptionRule
.
DRIVER
)
.
required
(
D
amengOptionRule
.
URL
,
Da
mengOptionRule
.
DRIVER
)
.
optional
(
D
emengOptionRule
.
USER
,
De
mengOptionRule
.
PASSWORD
)
.
optional
(
D
amengOptionRule
.
USER
,
Da
mengOptionRule
.
PASSWORD
)
.
build
();
.
build
();
public
static
final
OptionRule
METADATA_RULE
=
public
static
final
OptionRule
METADATA_RULE
=
OptionRule
.
builder
().
required
(
DemengOptionRule
.
DATABASE
,
DemengOptionRule
.
TABLE
).
build
();
OptionRule
.
builder
()
.
required
(
DamengOptionRule
.
DATABASE
,
DamengOptionRule
.
TABLE
)
.
build
();
}
}
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
emeng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/De
mengJdbcDataSourceChannel.java
→
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
ameng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/Da
mengJdbcDataSourceChannel.java
浏览文件 @
d18b47a1
...
@@ -17,13 +17,12 @@
...
@@ -17,13 +17,12 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
demeng
.
jdbc
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
demeng
.
jdbc
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannelExt
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannelExt
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
lombok.NonNull
;
import
lombok.NonNull
;
...
@@ -35,41 +34,41 @@ import java.util.stream.Collectors;
...
@@ -35,41 +34,41 @@ import java.util.stream.Collectors;
import
static
com
.
google
.
common
.
base
.
Preconditions
.
checkNotNull
;
import
static
com
.
google
.
common
.
base
.
Preconditions
.
checkNotNull
;
public
class
D
e
mengJdbcDataSourceChannel
implements
DataSourceChannelExt
{
public
class
D
a
mengJdbcDataSourceChannel
implements
DataSourceChannelExt
{
List
<
String
>
typeList
=
Arrays
.
asList
(
"varchar"
,
"char"
,
"json"
);
List
<
String
>
typeList
=
Arrays
.
asList
(
"varchar"
,
"char"
,
"json"
);
public
static
class
Holder
{
public
static
class
Holder
{
private
static
final
DemengJdbcDataSourceChannel
INSTANCE
=
new
DemengJdbcDataSourceChannel
();
private
static
final
DamengJdbcDataSourceChannel
INSTANCE
=
new
DamengJdbcDataSourceChannel
();
}
}
public
static
D
e
mengJdbcDataSourceChannel
getInstance
()
{
public
static
D
a
mengJdbcDataSourceChannel
getInstance
()
{
return
Holder
.
INSTANCE
;
return
Holder
.
INSTANCE
;
}
}
@Override
@Override
public
OptionRule
getDataSourceOptions
(
@NonNull
String
pluginName
)
{
public
OptionRule
getDataSourceOptions
(
@NonNull
String
pluginName
)
{
return
D
e
mengDataSourceConfig
.
OPTION_RULE
;
return
D
a
mengDataSourceConfig
.
OPTION_RULE
;
}
}
@Override
@Override
public
OptionRule
getDatasourceMetadataFieldsByDataSourceName
(
@NonNull
String
pluginName
)
{
public
OptionRule
getDatasourceMetadataFieldsByDataSourceName
(
@NonNull
String
pluginName
)
{
return
D
e
mengDataSourceConfig
.
METADATA_RULE
;
return
D
a
mengDataSourceConfig
.
METADATA_RULE
;
}
}
public
static
List
<
Map
>
getResultSet
(
ResultSet
rs
)
throws
SQLException
{
public
static
List
<
Map
>
getResultSet
(
ResultSet
rs
)
throws
SQLException
{
List
<
Map
>
reList
=
new
ArrayList
<>();
List
<
Map
>
reList
=
new
ArrayList
<>();
ResultSetMetaData
rsmd
=
rs
.
getMetaData
()
;
ResultSetMetaData
rsmd
=
rs
.
getMetaData
();
int
columnCount
=
rsmd
.
getColumnCount
();
//
ResultSet的总列数
int
columnCount
=
rsmd
.
getColumnCount
();
//
ResultSet的总列数
while
(
rs
.
next
())
{
while
(
rs
.
next
())
{
Map
map
=
new
HashMap
();
Map
map
=
new
HashMap
();
for
(
int
i
=
1
;
i
<=
columnCount
;
i
++)
{
for
(
int
i
=
1
;
i
<=
columnCount
;
i
++)
{
Object
value
=
rs
.
getObject
(
i
);
Object
value
=
rs
.
getObject
(
i
);
String
columnName
=
rsmd
.
getColumnName
(
i
);
String
columnName
=
rsmd
.
getColumnName
(
i
);
map
.
put
(
columnName
,
value
);
map
.
put
(
columnName
,
value
);
}
}
reList
.
add
(
map
);
reList
.
add
(
map
);
}
}
return
reList
;
return
reList
;
}
}
...
@@ -87,20 +86,25 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -87,20 +86,25 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
ResultSet
rs
=
null
;
ResultSet
rs
=
null
;
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
String
sql_selectAll
=
"SELECT * FROM user_tables"
;
String
sql_selectAll
=
"SELECT * FROM user_tables"
;
if
(
StringUtils
.
isNotBlank
(
database
))
{
if
(
StringUtils
.
isNotBlank
(
database
))
{
sql_selectAll
=
sql_selectAll
+
" where TABLESPACE_NAME='"
+
database
.
toUpperCase
()
+
"'"
;
sql_selectAll
=
sql_selectAll
+
" where TABLESPACE_NAME='"
+
database
.
toUpperCase
()
+
"'"
;
}
}
state
=
connection
.
createStatement
();
state
=
connection
.
createStatement
();
//执行查询的 SQL 语句
//
执行查询的 SQL 语句
rs
=
state
.
executeQuery
(
sql_selectAll
);
rs
=
state
.
executeQuery
(
sql_selectAll
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
tableNames
=
resultSet
.
stream
().
filter
(
f
->
f
.
get
(
"TABLE_NAME"
)
!=
null
).
map
(
m
->
m
.
get
(
"TABLE_NAME"
)
+
""
).
collect
(
Collectors
.
toList
());
tableNames
=
resultSet
.
stream
()
.
filter
(
f
->
f
.
get
(
"TABLE_NAME"
)
!=
null
)
.
map
(
m
->
m
.
get
(
"TABLE_NAME"
)
+
""
)
.
collect
(
Collectors
.
toList
());
return
tableNames
;
return
tableNames
;
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
}
finally
{
}
finally
{
try
{
try
{
//关闭资源
//
关闭资源
rs
.
close
();
rs
.
close
();
state
.
close
();
state
.
close
();
}
catch
(
SQLException
e
)
{
}
catch
(
SQLException
e
)
{
...
@@ -118,19 +122,24 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -118,19 +122,24 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
// 定义结果集对象
// 定义结果集对象
ResultSet
rs
=
null
;
ResultSet
rs
=
null
;
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
String
sql_selectAll
=
"SELECT TABLESPACE_NAME FROM user_tables group by TABLESPACE_NAME "
;
String
sql_selectAll
=
"SELECT TABLESPACE_NAME FROM user_tables group by TABLESPACE_NAME "
;
state
=
connection
.
createStatement
();
state
=
connection
.
createStatement
();
//执行查询的 SQL 语句
//
执行查询的 SQL 语句
rs
=
state
.
executeQuery
(
sql_selectAll
);
rs
=
state
.
executeQuery
(
sql_selectAll
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
tableNames
=
resultSet
.
stream
().
filter
(
f
->
f
.
get
(
"TABLESPACE_NAME"
)
!=
null
).
map
(
m
->
m
.
get
(
"TABLESPACE_NAME"
)
+
""
).
collect
(
Collectors
.
toList
());
tableNames
=
resultSet
.
stream
()
.
filter
(
f
->
f
.
get
(
"TABLESPACE_NAME"
)
!=
null
)
.
map
(
m
->
m
.
get
(
"TABLESPACE_NAME"
)
+
""
)
.
collect
(
Collectors
.
toList
());
return
tableNames
;
return
tableNames
;
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
throw
new
DataSourcePluginException
(
"get Databases names failed"
,
e
);
throw
new
DataSourcePluginException
(
"get Databases names failed"
,
e
);
}
finally
{
}
finally
{
try
{
try
{
//关闭资源
//
关闭资源
rs
.
close
();
rs
.
close
();
state
.
close
();
state
.
close
();
}
catch
(
SQLException
e
)
{
}
catch
(
SQLException
e
)
{
...
@@ -161,7 +170,9 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -161,7 +170,9 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
String
primaryKey
=
getPrimaryKey
(
metaData
,
database
,
table
);
String
primaryKey
=
getPrimaryKey
(
metaData
,
database
,
table
);
String
sql
=
String
sql
=
" select * from SYSCOLUMNS where id in (select id from SYSOBJECTS where name='"
+
table
.
toUpperCase
()+
"')"
;
" select * from SYSCOLUMNS where id in (select id from SYSOBJECTS where name='"
+
table
.
toUpperCase
()
+
"')"
;
Statement
statement
=
connection
.
createStatement
();
Statement
statement
=
connection
.
createStatement
();
ResultSet
resultSet
=
statement
.
executeQuery
(
sql
);
ResultSet
resultSet
=
statement
.
executeQuery
(
sql
);
while
(
resultSet
.
next
())
{
while
(
resultSet
.
next
())
{
...
@@ -186,11 +197,11 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -186,11 +197,11 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
}
}
tableField
.
setName
(
columnName
);
tableField
.
setName
(
columnName
);
// tableField.setComment(resultSet.getString("COLUMN_COMMENT"));
// tableField.setComment(resultSet.getString("COLUMN_COMMENT"));
// tableField.setDefaultValue(resultSet.getString("COLUMN_DEFAULT"));
// tableField.setDefaultValue(resultSet.getString("COLUMN_DEFAULT"));
String
nullable
=
resultSet
.
getString
(
"NULLABLE$"
);
String
nullable
=
resultSet
.
getString
(
"NULLABLE$"
);
if
(
"Y"
.
equals
(
nullable
))
{
if
(
"Y"
.
equals
(
nullable
))
{
tableField
.
setNullable
(
true
);
tableField
.
setNullable
(
true
);
}
else
{
}
else
{
tableField
.
setNullable
(
false
);
tableField
.
setNullable
(
false
);
...
@@ -236,21 +247,20 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -236,21 +247,20 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
private
Connection
getConnection
(
Map
<
String
,
String
>
requestParams
,
String
databaseName
)
private
Connection
getConnection
(
Map
<
String
,
String
>
requestParams
,
String
databaseName
)
throws
SQLException
,
ClassNotFoundException
{
throws
SQLException
,
ClassNotFoundException
{
checkNotNull
(
requestParams
.
get
(
DemengOptionRule
.
DRIVER
.
key
()));
checkNotNull
(
requestParams
.
get
(
DamengOptionRule
.
DRIVER
.
key
()));
checkNotNull
(
requestParams
.
get
(
DemengOptionRule
.
URL
.
key
()),
"Jdbc url cannot be null"
);
checkNotNull
(
requestParams
.
get
(
DamengOptionRule
.
URL
.
key
()),
"Jdbc url cannot be null"
);
String
url
=
String
url
=
DamengOptionRule
.
URL
.
key
();
DemengOptionRule
.
URL
.
key
();
if
(
requestParams
.
containsKey
(
DamengOptionRule
.
USER
.
key
()))
{
if
(
requestParams
.
containsKey
(
DemengOptionRule
.
USER
.
key
()))
{
String
username
=
requestParams
.
get
(
DamengOptionRule
.
USER
.
key
());
String
username
=
requestParams
.
get
(
DemengOptionRule
.
USER
.
key
());
String
password
=
requestParams
.
get
(
DamengOptionRule
.
PASSWORD
.
key
());
String
password
=
requestParams
.
get
(
DemengOptionRule
.
PASSWORD
.
key
());
Class
.
forName
(
DamengOptionRule
.
DriverType
.
DEMENG
.
getDriverClassName
());
Class
.
forName
(
DemengOptionRule
.
DriverType
.
DEMENG
.
getDriverClassName
());
Connection
conn
=
DriverManager
.
getConnection
(
url
,
username
,
password
);
Connection
conn
=
DriverManager
.
getConnection
(
url
,
username
,
password
);
conn
.
setAutoCommit
(
true
);
conn
.
setAutoCommit
(
true
);
return
conn
;
return
conn
;
}
}
Class
.
forName
(
D
e
mengOptionRule
.
DriverType
.
DEMENG
.
getDriverClassName
());
Class
.
forName
(
D
a
mengOptionRule
.
DriverType
.
DEMENG
.
getDriverClassName
());
Connection
conn
=
DriverManager
.
getConnection
(
url
,
"SYSDBA"
,
null
);
Connection
conn
=
DriverManager
.
getConnection
(
url
,
"SYSDBA"
,
null
);
conn
.
setAutoCommit
(
true
);
conn
.
setAutoCommit
(
true
);
return
conn
;
return
conn
;
}
}
...
@@ -269,20 +279,21 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -269,20 +279,21 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
ResultSet
rs
=
null
;
ResultSet
rs
=
null
;
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
try
(
Connection
connection
=
getConnection
(
requestParams
))
{
String
sql_selectAll
=
"SELECT * FROM user_tables"
;
String
sql_selectAll
=
"SELECT * FROM user_tables"
;
if
(
StringUtils
.
isNotBlank
(
database
))
{
if
(
StringUtils
.
isNotBlank
(
database
))
{
sql_selectAll
=
sql_selectAll
+
" where TABLESPACE_NAME='"
+
database
.
toUpperCase
()
+
"'"
;
sql_selectAll
=
sql_selectAll
+
" where TABLESPACE_NAME='"
+
database
.
toUpperCase
()
+
"'"
;
}
}
state
=
connection
.
createStatement
();
state
=
connection
.
createStatement
();
//执行查询的 SQL 语句
//
执行查询的 SQL 语句
rs
=
state
.
executeQuery
(
sql_selectAll
);
rs
=
state
.
executeQuery
(
sql_selectAll
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
List
<
Map
>
resultSet
=
getResultSet
(
rs
);
return
resultSet
;
return
resultSet
;
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
}
catch
(
ClassNotFoundException
|
SQLException
e
)
{
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
throw
new
DataSourcePluginException
(
"get table names failed"
,
e
);
}
finally
{
}
finally
{
try
{
try
{
//关闭资源
//
关闭资源
rs
.
close
();
rs
.
close
();
state
.
close
();
state
.
close
();
}
catch
(
SQLException
e
)
{
}
catch
(
SQLException
e
)
{
...
@@ -299,10 +310,10 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
...
@@ -299,10 +310,10 @@ public class DemengJdbcDataSourceChannel implements DataSourceChannelExt {
Map
<
String
,
String
>
options
,
Map
<
String
,
String
>
options
,
String
tableName
)
{
String
tableName
)
{
List
<
Map
>
list
=
getTableAlls
(
pluginName
,
requestParams
,
database
,
tableName
,
options
);
List
<
Map
>
list
=
getTableAlls
(
pluginName
,
requestParams
,
database
,
tableName
,
options
);
if
(
CollectionUtils
.
isNotEmpty
(
list
))
{
if
(
CollectionUtils
.
isNotEmpty
(
list
))
{
return
list
.
get
(
0
);
return
list
.
get
(
0
);
}
}
return
null
;
return
null
;
}
}
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
emeng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/De
mengJdbcDataSourceFactory.java
→
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
ameng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/Da
mengJdbcDataSourceFactory.java
浏览文件 @
d18b47a1
...
@@ -29,20 +29,20 @@ import java.util.Set;
...
@@ -29,20 +29,20 @@ import java.util.Set;
@Slf4j
@Slf4j
@AutoService
(
DataSourceFactory
.
class
)
@AutoService
(
DataSourceFactory
.
class
)
public
class
D
e
mengJdbcDataSourceFactory
implements
DataSourceFactory
{
public
class
D
a
mengJdbcDataSourceFactory
implements
DataSourceFactory
{
@Override
@Override
public
String
factoryIdentifier
()
{
public
String
factoryIdentifier
()
{
return
D
e
mengDataSourceConfig
.
PLUGIN_NAME
;
return
D
a
mengDataSourceConfig
.
PLUGIN_NAME
;
}
}
@Override
@Override
public
Set
<
DataSourcePluginInfo
>
supportedDataSources
()
{
public
Set
<
DataSourcePluginInfo
>
supportedDataSources
()
{
return
Sets
.
newHashSet
(
D
e
mengDataSourceConfig
.
MYSQL_DATASOURCE_PLUGIN_INFO
);
return
Sets
.
newHashSet
(
D
a
mengDataSourceConfig
.
MYSQL_DATASOURCE_PLUGIN_INFO
);
}
}
@Override
@Override
public
DataSourceChannel
createChannel
()
{
public
DataSourceChannel
createChannel
()
{
return
new
D
e
mengJdbcDataSourceChannel
();
return
new
D
a
mengJdbcDataSourceChannel
();
}
}
}
}
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
emeng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/De
mengOptionRule.java
→
seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-d
ameng/src/main/java/org/apache/seatunnel/datasource/plugin/demeng/jdbc/Da
mengOptionRule.java
浏览文件 @
d18b47a1
...
@@ -20,15 +20,13 @@ package org.apache.seatunnel.datasource.plugin.demeng.jdbc;
...
@@ -20,15 +20,13 @@ package org.apache.seatunnel.datasource.plugin.demeng.jdbc;
import
org.apache.seatunnel.api.configuration.Option
;
import
org.apache.seatunnel.api.configuration.Option
;
import
org.apache.seatunnel.api.configuration.Options
;
import
org.apache.seatunnel.api.configuration.Options
;
public
class
D
e
mengOptionRule
{
public
class
D
a
mengOptionRule
{
public
static
final
Option
<
String
>
URL
=
public
static
final
Option
<
String
>
URL
=
Options
.
key
(
"url"
)
Options
.
key
(
"url"
)
.
stringType
()
.
stringType
()
.
noDefaultValue
()
.
noDefaultValue
()
.
withDescription
(
.
withDescription
(
"jdbc url, eg:"
+
" jdbc:dm://localhost:5236"
);
"jdbc url, eg:"
+
" jdbc:dm://localhost:5236"
);
public
static
final
Option
<
String
>
USER
=
public
static
final
Option
<
String
>
USER
=
Options
.
key
(
"user"
).
stringType
().
noDefaultValue
().
withDescription
(
"jdbc user"
);
Options
.
key
(
"user"
).
stringType
().
noDefaultValue
().
withDescription
(
"jdbc user"
);
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-xml/pom.xml
浏览文件 @
d18b47a1
...
@@ -141,7 +141,6 @@
...
@@ -141,7 +141,6 @@
<artifactId>
aws-java-sdk-bundle
</artifactId>
<artifactId>
aws-java-sdk-bundle
</artifactId>
</dependency>
</dependency>
</dependencies>
</dependencies>
</project>
</project>
seatunnel-datasource/seatunnel-datasource-plugins/datasource-xml/src/main/java/org/apache/seatunnel/datasource/plugin/xml/XMLAConfiguration.java
浏览文件 @
d18b47a1
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
xml
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
xml
;
import
lombok.extern.slf4j.Slf4j
;
import
org.apache.hadoop.conf.Configuration
;
import
org.apache.seatunnel.shade.com.typesafe.config.Config
;
import
org.apache.seatunnel.shade.com.typesafe.config.Config
;
import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
;
import
org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
;
import
org.apache.hadoop.conf.Configuration
;
import
lombok.extern.slf4j.Slf4j
;
import
java.util.Map
;
import
java.util.Map
;
@Slf4j
@Slf4j
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-xml/src/main/java/org/apache/seatunnel/datasource/plugin/xml/XMLDataSourceFactory.java
浏览文件 @
d18b47a1
...
@@ -17,13 +17,14 @@
...
@@ -17,13 +17,14 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
xml
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
xml
;
import
com.google.auto.service.AutoService
;
import
com.google.common.collect.Sets
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceFactory
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceFactory
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo
;
import
org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum
;
import
org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum
;
import
com.google.auto.service.AutoService
;
import
com.google.common.collect.Sets
;
import
java.util.Set
;
import
java.util.Set
;
@AutoService
(
DataSourceFactory
.
class
)
@AutoService
(
DataSourceFactory
.
class
)
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-xml/src/main/java/org/apache/seatunnel/datasource/plugin/xml/XMLDatasourceChannel.java
浏览文件 @
d18b47a1
...
@@ -17,18 +17,20 @@
...
@@ -17,18 +17,20 @@
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
xml
;
package
org
.
apache
.
seatunnel
.
datasource
.
plugin
.
xml
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.common.utils.SeaTunnelException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
org.apache.commons.lang3.StringUtils
;
import
com.alibaba.fastjson2.JSON
;
import
com.alibaba.fastjson2.JSON
;
import
io.minio.*
;
import
io.minio.*
;
import
io.minio.errors.*
;
import
io.minio.errors.*
;
import
io.minio.messages.Bucket
;
import
io.minio.messages.Bucket
;
import
io.minio.messages.Item
;
import
io.minio.messages.Item
;
import
lombok.NonNull
;
import
lombok.NonNull
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.common.utils.SeaTunnelException
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourceChannel
;
import
org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException
;
import
org.apache.seatunnel.datasource.plugin.api.model.TableField
;
import
java.io.BufferedReader
;
import
java.io.BufferedReader
;
import
java.io.IOException
;
import
java.io.IOException
;
...
...
seatunnel-datasource/seatunnel-datasource-plugins/datasource-xml/src/main/java/org/apache/seatunnel/datasource/plugin/xml/XMLOptionRule.java
浏览文件 @
d18b47a1
...
@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option;
...
@@ -21,7 +21,6 @@ import org.apache.seatunnel.api.configuration.Option;
import
org.apache.seatunnel.api.configuration.Options
;
import
org.apache.seatunnel.api.configuration.Options
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
org.apache.seatunnel.api.configuration.util.OptionRule
;
import
java.util.Arrays
;
import
java.util.Map
;
import
java.util.Map
;
public
class
XMLOptionRule
{
public
class
XMLOptionRule
{
...
@@ -121,7 +120,7 @@ public class XMLOptionRule {
...
@@ -121,7 +120,7 @@ public class XMLOptionRule {
return
OptionRule
.
builder
()
return
OptionRule
.
builder
()
.
required
(
PATH
,
TYPE
)
.
required
(
PATH
,
TYPE
)
.
conditional
(
TYPE
,
FileFormat
.
XML
.
type
,
DELIMITER
)
.
conditional
(
TYPE
,
FileFormat
.
XML
.
type
,
DELIMITER
)
.
conditional
(
TYPE
,
FileFormat
.
XML
.
type
,
SCHEMA
)
.
conditional
(
TYPE
,
FileFormat
.
XML
.
type
,
SCHEMA
)
.
optional
(
PARSE_PARSE_PARTITION_FROM_PATH
)
.
optional
(
PARSE_PARSE_PARTITION_FROM_PATH
)
.
optional
(
DATE_FORMAT
)
.
optional
(
DATE_FORMAT
)
.
optional
(
DATETIME_FORMAT
)
.
optional
(
DATETIME_FORMAT
)
...
...
seatunnel-datasource/seatunnel-datasource-plugins/pom.xml
浏览文件 @
d18b47a1
...
@@ -50,7 +50,7 @@
...
@@ -50,7 +50,7 @@
<module>
datasource-redis
</module>
<module>
datasource-redis
</module>
<module>
datasource-rabbitmq
</module>
<module>
datasource-rabbitmq
</module>
<module>
datasource-ftp
</module>
<module>
datasource-ftp
</module>
<module>
datasource-jdbc-d
e
meng
</module>
<module>
datasource-jdbc-d
a
meng
</module>
<module>
datasource-jdbc-access
</module>
<module>
datasource-jdbc-access
</module>
<module>
datasource-http
</module>
<module>
datasource-http
</module>
<module>
datasource-xml
</module>
<module>
datasource-xml
</module>
...
...
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论