figmar/deepdata
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

deepdata: core product code (backend, frontend, runtime, deepflow module)

Browse Source
This commit is contained in:
figmar
2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/datax/s3reader/pom.xml
+84
View File
@@ -0,0 +1,84 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-all</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<artifactId>s3reader</artifactId>
<name>s3reader</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-core</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-common</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.properties</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${jdk-version}</source>
<target>${jdk-version}</target>
<encoding>${project-sourceEncoding}</encoding>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assembly/package.xml</descriptor>
</descriptors>
<finalName>datax</finalName>
</configuration>
<executions>
<execution>
<id>dwzip</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
runtime/datax/s3reader/src/main/assembly/package.xml
+35
View File
@@ -0,0 +1,35 @@
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id></id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>plugin.json</include>
<include>plugin_job_template.json</include>
</includes>
<outputDirectory>plugin/reader/s3reader</outputDirectory>
</fileSet>
<fileSet>
<directory>target/</directory>
<includes>
<include>s3reader-0.0.1-SNAPSHOT.jar</include>
</includes>
<outputDirectory>plugin/reader/s3reader</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>plugin/reader/s3reader/libs</outputDirectory>
<scope>runtime</scope>
</dependencySet>
</dependencySets>
</assembly>
runtime/datax/s3reader/src/main/java/com/deepdata/plugin/reader/s3reader/HttpEndpointSecurity.java
+175
View File
@@ -0,0 +1,175 @@
package com.deepdata.plugin.reader.s3reader;
import java.net.IDN;
import java.net.InetAddress;
import java.net.URI;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
final class HttpEndpointSecurity {
private static final String ALLOWED_HOSTS_PROPERTY = "deepdata.ssrf.allowedHosts";
private static final String ALLOWED_HOSTS_ENV = "DEEPDATA_SSRF_ALLOWED_HOSTS";
private HttpEndpointSecurity() {
}
static URI validateExternalHttpUri(String rawUri, String fieldName) {
if (isBlank(rawUri)) {
throw new IllegalArgumentException(fieldName + " is required");
}
if (rawUri.indexOf('\r') >= 0 || rawUri.indexOf('\n') >= 0) {
throw new IllegalArgumentException(fieldName + " contains invalid control characters");
}
URI uri = URI.create(rawUri.trim()).normalize();
String scheme = uri.getScheme() == null ? null : uri.getScheme().toLowerCase(Locale.ROOT);
if (!"http".equals(scheme) && !"https".equals(scheme)) {
throw new IllegalArgumentException(fieldName + " only supports http or https");
}
if (!isBlank(uri.getUserInfo())) {
throw new IllegalArgumentException(fieldName + " must not contain user info");
}
if (uri.getFragment() != null) {
throw new IllegalArgumentException(fieldName + " must not contain fragment");
}
int port = uri.getPort();
if (port == 0 || port < -1 || port > 65535) {
throw new IllegalArgumentException(fieldName + " contains invalid port");
}
String asciiHost = toAsciiHost(uri.getHost(), fieldName);
if (isAllowedHost(asciiHost)) {
return uri;
}
InetAddress[] addresses;
try {
addresses = InetAddress.getAllByName(asciiHost);
} catch (Exception e) {
throw new IllegalArgumentException(fieldName + " host cannot be resolved", e);
}
if (addresses.length == 0) {
throw new IllegalArgumentException(fieldName + " host cannot be resolved");
}
for (InetAddress address : addresses) {
if (!isPublicRoutableAddress(address)) {
throw new IllegalArgumentException(fieldName + " points to a restricted network address");
}
}
return uri;
}
private static String toAsciiHost(String host, String fieldName) {
if (isBlank(host)) {
throw new IllegalArgumentException(fieldName + " must contain a valid host");
}
try {
String normalized = host.endsWith(".") ? host.substring(0, host.length() - 1) : host;
return IDN.toASCII(normalized, IDN.USE_STD3_ASCII_RULES).toLowerCase(Locale.ROOT);
} catch (Exception e) {
throw new IllegalArgumentException(fieldName + " contains invalid host", e);
}
}
private static boolean isAllowedHost(String host) {
return configuredAllowedHosts().contains(host);
}
private static Set<String> configuredAllowedHosts() {
Set<String> hosts = new HashSet<String>();
addConfiguredHosts(hosts, System.getProperty(ALLOWED_HOSTS_PROPERTY));
addConfiguredHosts(hosts, System.getenv(ALLOWED_HOSTS_ENV));
return hosts;
}
private static void addConfiguredHosts(Set<String> hosts, String rawHosts) {
if (isBlank(rawHosts)) {
return;
}
String[] parts = rawHosts.split(",");
for (String part : parts) {
if (!isBlank(part)) {
hosts.add(toAsciiHost(part.trim(), "allowed host"));
}
}
}
private static boolean isPublicRoutableAddress(InetAddress address) {
if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress()
|| address.isSiteLocalAddress() || address.isMulticastAddress()) {
return false;
}
byte[] bytes = address.getAddress();
if (bytes.length == 4) {
return isPublicRoutableIpv4(bytes);
}
if (bytes.length == 16) {
return isPublicRoutableIpv6(bytes);
}
return false;
}
private static boolean isPublicRoutableIpv4(byte[] bytes) {
int first = bytes[0] & 0xff;
int second = bytes[1] & 0xff;
int third = bytes[2] & 0xff;
if (first == 0 || first == 10 || first == 127 || first >= 224) {
return false;
}
if (first == 100 && second >= 64 && second <= 127) {
return false;
}
if (first == 169 && second == 254) {
return false;
}
if (first == 172 && second >= 16 && second <= 31) {
return false;
}
if (first == 192 && second == 168) {
return false;
}
if (first == 192 && second == 0 && (third == 0 || third == 2)) {
return false;
}
if (first == 198 && (second == 18 || second == 19 || (second == 51 && third == 100))) {
return false;
}
if (first == 203 && second == 0 && third == 113) {
return false;
}
return true;
}
private static boolean isPublicRoutableIpv6(byte[] bytes) {
int first = bytes[0] & 0xff;
int second = bytes[1] & 0xff;
if ((first & 0xfe) == 0xfc) {
return false;
}
if (first == 0xfe && (second & 0xc0) == 0x80) {
return false;
}
if (first == 0x20 && second == 0x01 && (bytes[2] & 0xff) == 0x0d && (bytes[3] & 0xff) == 0xb8) {
return false;
}
return !isIpv4MappedPrivateAddress(bytes);
}
private static boolean isIpv4MappedPrivateAddress(byte[] bytes) {
for (int i = 0; i < 10; i++) {
if (bytes[i] != 0) {
return false;
}
}
if ((bytes[10] & 0xff) != 0xff || (bytes[11] & 0xff) != 0xff) {
return false;
}
byte[] ipv4 = new byte[] {bytes[12], bytes[13], bytes[14], bytes[15]};
return !isPublicRoutableIpv4(ipv4);
}
private static boolean isBlank(String value) {
return value == null || value.trim().isEmpty();
}
}
runtime/datax/s3reader/src/main/java/com/deepdata/plugin/reader/s3reader/S3Reader.java
+224
View File
@@ -0,0 +1,224 @@
package com.deepdata.plugin.reader.s3reader;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.alibaba.datax.common.element.Record;
import com.alibaba.datax.common.element.StringColumn;
import com.alibaba.datax.common.exception.CommonErrorCode;
import com.alibaba.datax.common.exception.DataXException;
import com.alibaba.datax.common.plugin.RecordSender;
import com.alibaba.datax.common.spi.Reader;
import com.alibaba.datax.common.util.Configuration;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.S3Configuration;
/**
* S3兼容对象存储读取器
* 支持自定义 S3 兼容对象存储（如 MinIO、Ceph 等）
*/
public class S3Reader extends Reader {
private static final Logger LOG = LoggerFactory.getLogger(S3Reader.class);
public static class Job extends Reader.Job {
private Configuration jobConfig = null;
@Override
public void init() {
this.jobConfig = super.getPluginJobConf();
}
@Override
public void prepare() {
String endpoint = this.jobConfig.getString("endpoint");
String bucket = this.jobConfig.getString("bucket");
String accessKey = this.jobConfig.getString("accessKey");
String secretKey = this.jobConfig.getString("secretKey");
if (StringUtils.isBlank(endpoint)) {
throw new RuntimeException("endpoint is required for s3reader");
}
HttpEndpointSecurity.validateExternalHttpUri(endpoint, "endpoint");
if (StringUtils.isBlank(bucket)) {
throw new RuntimeException("bucket is required for s3reader");
}
if (StringUtils.isBlank(accessKey)) {
throw new RuntimeException("accessKey is required for s3reader");
}
if (StringUtils.isBlank(secretKey)) {
throw new RuntimeException("secretKey is required for s3reader");
}
}
@Override
public List<Configuration> split(int adviceNumber) {
return Collections.singletonList(this.jobConfig);
}
@Override
public void post() {
}
@Override
public void destroy() {
}
}
public static class Task extends Reader.Task {
private Configuration jobConfig;
private Set<String> fileType;
private String endpoint;
private String accessKey;
private String secretKey;
private String bucket;
private String prefix;
private String region;
private S3Client s3;
private String effectivePrefix;
@Override
public void init() {
this.jobConfig = super.getPluginJobConf();
this.fileType = new HashSet<>(this.jobConfig.getList("fileType", Collections.emptyList(), String.class));
this.endpoint = this.jobConfig.getString("endpoint");
this.accessKey = this.jobConfig.getString("accessKey");
this.secretKey = this.jobConfig.getString("secretKey");
this.bucket = this.jobConfig.getString("bucket");
this.prefix = this.jobConfig.getString("prefix");
// OBS也是默认us-east-1，这里保留默认值
this.region = this.jobConfig.getString("region", "us-east-1");
HttpEndpointSecurity.validateExternalHttpUri(this.endpoint, "endpoint");
this.s3 = getS3Client();
this.effectivePrefix = getEffectivePrefix();
}
@Override
public void startRead(RecordSender recordSender) {
try {
List<String> files = listFiles().stream()
.filter(file -> fileType.isEmpty() || fileType.contains(getFileSuffix(file)))
.collect(Collectors.toList());
files.forEach(filePath -> {
Record record = recordSender.createRecord();
record.addColumn(new StringColumn(filePath));
recordSender.sendToWriter(record);
});
this.jobConfig.set("columnNumber", 1);
} catch (Exception e) {
LOG.error("Error reading files from S3 compatible storage: {}", this.endpoint, e);
throw new RuntimeException(e);
}
}
/**
* 列举 S3 对象
* 非递归：只列举 prefix 当前目录下的对象（通过 delimiter="/" 实现）
*/
private List<String> listFiles() {
if (StringUtils.isBlank(endpoint) || StringUtils.isBlank(bucket)) {
throw new IllegalArgumentException("endpoint and bucket must be provided");
}
List<String> keys = new ArrayList<>();
String continuationToken = null;
try {
do {
ListObjectsV2Request.Builder reqBuilder = ListObjectsV2Request.builder()
.bucket(bucket)
.prefix(effectivePrefix)
/*.delimiter("/")*/ /* xuhuanqing 将其设置为递归 */;
if (continuationToken != null) {
reqBuilder.continuationToken(continuationToken);
}
ListObjectsV2Response res = s3.listObjectsV2(reqBuilder.build());
for (S3Object obj : res.contents()) {
String key = obj.key();
if (isInValid(key)) {
continue;
}
keys.add(key);
}
continuationToken = res.isTruncated() ? res.nextContinuationToken() : null;
} while (continuationToken != null);
} catch (Exception e) {
LOG.warn("Failed to list S3 objects: {}", e.getMessage(), e);
}
return keys;
}
private boolean isInValid(String key) {
if (!effectivePrefix.isEmpty() && !key.startsWith(effectivePrefix)) {
return true;
}
if (key.equals(effectivePrefix) || key.endsWith("/")) {
return true;
}
return false;
}
private String getEffectivePrefix() {
String effectivePrefix = "";
if (prefix != null) {
effectivePrefix = prefix.startsWith("/") ? prefix.substring(1) : prefix;
if (!effectivePrefix.isEmpty() && !effectivePrefix.endsWith("/")) {
effectivePrefix = effectivePrefix + "/";
}
}
return effectivePrefix;
}
private S3Client getS3Client() {
try {
AwsBasicCredentials creds = AwsBasicCredentials.create(accessKey, secretKey);
S3Configuration serviceConfig = S3Configuration.builder()
.pathStyleAccessEnabled(true)
.build();
return S3Client.builder()
.endpointOverride(HttpEndpointSecurity.validateExternalHttpUri(endpoint, "endpoint"))
.region(Region.of(region))
.serviceConfiguration(serviceConfig)
.credentialsProvider(StaticCredentialsProvider.create(creds))
.build();
} catch (Exception e) {
LOG.error("Error init S3 client: {}", this.endpoint, e);
throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR, e);
}
}
private String getFileSuffix(String key) {
String fileName = Paths.get(key).getFileName().toString();
int lastDotIndex = fileName.lastIndexOf('.');
if (lastDotIndex == -1 || lastDotIndex == fileName.length() - 1) {
return "";
}
return fileName.substring(lastDotIndex + 1);
}
@Override
public void destroy() {
if (s3 != null) {
try {
s3.close();
} catch (Exception ignore) {
}
}
}
}
}
runtime/datax/s3reader/src/main/resources/plugin.json
+6
View File
@@ -0,0 +1,6 @@
{
"name": "s3reader",
"class": "com.deepdata.plugin.reader.s3reader.S3Reader",
"description": "read from S3 compatible object storage",
"developer": "deepdata"
}
runtime/datax/s3reader/src/main/resources/plugin_job_template.json
+11
View File
@@ -0,0 +1,11 @@
{
"name": "s3reader",
"parameter": {
"endpoint": "https://s3.example.com",
"bucket": "test-bucket",
"accessKey": "ak-xxx",
"secretKey": "sk-xxx",
"prefix": "/test",
"region": "us-east-1"
}
}
runtime/datax/s3reader/src/test/java/com/deepdata/plugin/reader/s3reader/HttpEndpointSecurityTest.java
+84
View File
@@ -0,0 +1,84 @@
package com.deepdata.plugin.reader.s3reader;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class HttpEndpointSecurityTest {
@Test(expected = IllegalArgumentException.class)
public void shouldRejectLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://127.0.0.1:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectLocalhost() {
HttpEndpointSecurity.validateExternalHttpUri("http://localhost:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectIpv6LoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://[::1]:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectIpv4MappedIpv6LoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://[::ffff:127.0.0.1]:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectIntegerEncodedLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://2130706433:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectHexEncodedLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://0x7f000001:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectOctalEncodedLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://017700000001:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectMetadataServiceAddress() {
HttpEndpointSecurity.validateExternalHttpUri("http://169.254.169.254/latest/meta-data", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectPrivateNetworkAddress() {
HttpEndpointSecurity.validateExternalHttpUri("http://192.168.1.10:9000", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectUnsupportedScheme() {
HttpEndpointSecurity.validateExternalHttpUri("gopher://8.8.8.8:70", "endpoint");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectUserInfo() {
HttpEndpointSecurity.validateExternalHttpUri("http://user@8.8.8.8:9000", "endpoint");
}
@Test
public void shouldAllowPublicHttpAddress() {
assertEquals("8.8.8.8",
HttpEndpointSecurity.validateExternalHttpUri("https://8.8.8.8:9000", "endpoint").getHost());
}
@Test
public void shouldAllowExplicitlyConfiguredHost() {
String previous = System.getProperty("deepdata.ssrf.allowedHosts");
try {
System.setProperty("deepdata.ssrf.allowedHosts", "localhost");
assertEquals("localhost",
HttpEndpointSecurity.validateExternalHttpUri("http://localhost:9000", "endpoint").getHost());
} finally {
if (previous == null) {
System.clearProperty("deepdata.ssrf.allowedHosts");
} else {
System.setProperty("deepdata.ssrf.allowedHosts", previous);
}
}
}
}