figmar/deepdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deepdata: core product code (backend, frontend, runtime, deepflow module)

Browse Source
This commit is contained in:
figmar
2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/datax/apireader/pom.xml
+86
View File
@@ -0,0 +1,86 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-all</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<artifactId>apireader</artifactId>
<dependencies>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-core</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-common</artifactId>
<version>${datax-project-version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.83_noneautotype</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>5.3.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.properties</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${jdk-version}</source>
<target>${jdk-version}</target>
<encoding>${project-sourceEncoding}</encoding>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assembly/package.xml</descriptor>
</descriptors>
<finalName>datax</finalName>
</configuration>
<executions>
<execution>
<id>dwzip</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
runtime/datax/apireader/src/main/assembly/package.xml
+35
View File
@@ -0,0 +1,35 @@
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id></id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<includes>
<include>plugin.json</include>
<include>plugin_job_template.json</include>
</includes>
<outputDirectory>plugin/reader/apireader</outputDirectory>
</fileSet>
<fileSet>
<directory>target/</directory>
<includes>
<include>apireader-0.0.1-SNAPSHOT.jar</include>
</includes>
<outputDirectory>plugin/reader/apireader</outputDirectory>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>plugin/reader/apireader/libs</outputDirectory>
<scope>runtime</scope>
</dependencySet>
</dependencySets>
</assembly>
runtime/datax/apireader/src/main/java/com/deepdata/plugin/reader/apireader/APIReader.java
+275
View File
@@ -0,0 +1,275 @@
package com.deepdata.plugin.reader.apireader;
import com.alibaba.datax.common.element.*;
import com.alibaba.datax.common.plugin.RecordSender;
import com.alibaba.datax.common.spi.Reader;
import com.alibaba.datax.common.util.Configuration;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.charset.StandardCharsets;
import org.apache.hc.client5.http.classic.methods.HttpDelete;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.classic.methods.HttpPut;
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import java.net.URI;
import java.util.Collections;
import java.util.List;
/**
* API读取器
* 支持符合restful api的接口
*/
public class APIReader extends Reader {
private static final Logger LOG = LoggerFactory.getLogger(APIReader.class);
public static class Job extends Reader.Job {
private Configuration jobConfig = null;
@Override
public void init() {
this.jobConfig = super.getPluginJobConf();
}
@Override
public void prepare() {
String api = this.jobConfig.getString("api");
Object schema = this.jobConfig.get("schema");
HttpEndpointSecurity.validateExternalHttpUri(api, "api");
if (schema == null) {
throw new RuntimeException("schema configuration is required for APIReader");
}
}
@Override
public List<Configuration> split(int adviceNumber) {
// API通常不支持并行分片读取，除非有分页参数逻辑，此处暂按单通道处理
return Collections.singletonList(this.jobConfig);
}
@Override
public void destroy() {
}
}
public static class Task extends Reader.Task {
private Configuration jobConfig;
private String api;
private String method;
private String dataPath; // 用于指定返回json中哪个节点是数据列表，例如 "data.list"
private List<Configuration> schemaFields;
private String requestBody;
private Configuration requestHeaders;
@Override
public void init() {
this.jobConfig = super.getPluginJobConf();
this.api = this.jobConfig.getString("api");
this.method = this.jobConfig.getString("method", "GET");
this.requestBody = this.jobConfig.getString("body", "");
this.requestHeaders = this.jobConfig.getConfiguration("headers");
parseSchema();
}
private void parseSchema() {
Configuration schemaConfig = this.jobConfig.getConfiguration("schema");
if (schemaConfig != null && !schemaConfig.getKeys().isEmpty()) {
this.dataPath = schemaConfig.getString("dataPath", "");
this.schemaFields = schemaConfig.getListConfiguration("fields");
return;
}
if (this.schemaFields == null || this.schemaFields.isEmpty()) {
throw new RuntimeException("schema.fields is required and cannot be empty");
}
for (Configuration fieldConfig : this.schemaFields) {
String name = fieldConfig.getString("name"); // JSON中的字段名
String path = fieldConfig.getString("path"); // 可选：支持深层提取
if (StringUtils.isBlank(name) && StringUtils.isBlank(path)) {
throw new RuntimeException("schema.fields must contain name or path");
}
}
}
@Override
public void startRead(RecordSender recordSender) {
LOG.info("Start reading from API: [{}]", this.api);
try {
// 1. 发起HTTP请求
String responseBody = doHttpRequest(this.api, this.method, this.requestBody, this.requestHeaders);
// 2. 解析响应
JSONArray dataArray = parseResponse(responseBody);
if (dataArray.isEmpty()) {
LOG.warn("API returned empty data.");
return;
}
// 3. 添加文件表头
addDataHeader(recordSender);
// 4. 遍历数据并转换为DataX Record
addData(recordSender, dataArray);
} catch (Exception e) {
LOG.error("Error occurred while reading from API", e);
throw new RuntimeException(e);
}
}
private void addData(RecordSender recordSender, JSONArray dataArray) {
for (Object itemObj : dataArray) {
JSONObject item = (JSONObject) itemObj;
Record record = recordSender.createRecord();
for (Configuration fieldConfig : this.schemaFields) {
String name = fieldConfig.getString("name"); // JSON中的字段名
String path = fieldConfig.getString("path"); // 可选：支持深层提取
if (StringUtils.isBlank(name) && StringUtils.isBlank(path)) {
throw new RuntimeException("schema.fields must contain name or path");
}
Object val;
if (StringUtils.isNotBlank(path)) {
val = JSONPath.eval(item, path);
} else {
val = item.get(name);
}
if (val == null) {
record.addColumn(new StringColumn(""));
continue;
}
record.addColumn(new StringColumn(val.toString()));
}
recordSender.sendToWriter(record);
}
}
private void addDataHeader(RecordSender recordSender) {
Record recordHeader = recordSender.createRecord();
for (Configuration fieldConfig : this.schemaFields) {
String name = fieldConfig.getString("name"); // JSON中的字段名
String alias = fieldConfig.getString("alias"); // 自定义的字段别名
if (StringUtils.isNotBlank(alias)) {
recordHeader.addColumn(new StringColumn(alias));
} else {
recordHeader.addColumn(new StringColumn(name));
}
}
recordSender.sendToWriter(recordHeader);
}
private JSONArray parseResponse(String responseBody) {
Object responseJson = JSON.parse(responseBody);
JSONArray dataArray;
// 定位数据节点
if (StringUtils.isNotBlank(this.dataPath)) {
// 使用 JSONPath 提取
Object extracted = JSONPath.eval(responseJson, this.dataPath);
if (extracted instanceof JSONArray) {
dataArray = (JSONArray) extracted;
} else {
throw new RuntimeException("The dataPath configured [" + this.dataPath + "] does not point to a JSON Array.");
}
} else {
if (responseJson instanceof JSONArray) {
dataArray = (JSONArray) responseJson;
} else if (responseJson instanceof JSONObject) {
throw new RuntimeException("Response is an Object, please configure schema.dataPath to point to the list.");
} else {
throw new RuntimeException("Unknown response format.");
}
}
return dataArray;
}
/**
* 简单的HTTP请求实现
*/
private String doHttpRequest(String urlStr, String method, String body, Configuration headers) throws Exception {
URI safeUri = HttpEndpointSecurity.validateExternalHttpUri(urlStr, "api");
RequestConfig requestConfig = RequestConfig.custom()
.setRedirectsEnabled(false)
.build();
try (CloseableHttpClient client = HttpClients.custom()
.disableRedirectHandling()
.setDefaultRequestConfig(requestConfig)
.build()) {
HttpUriRequestBase request = buildRequest(safeUri, method, body);
if (headers != null && !headers.getKeys().isEmpty()) {
for (String key : headers.getKeys()) {
String value = headers.getString(key);
if (StringUtils.isNotBlank(key) && value != null) {
request.addHeader(key, value);
}
}
}
try (CloseableHttpResponse response = client.execute(request)) {
int status = response.getCode();
if (status >= 200 && status < 300) {
return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
}
throw new RuntimeException("API Call failed. Http Code: " + status);
}
}
}
private HttpUriRequestBase buildRequest(URI uri, String method, String body) {
String verb = StringUtils.defaultIfBlank(method, "GET").toUpperCase();
HttpUriRequestBase request;
switch (verb) {
case "POST":
request = new HttpPost(uri);
break;
case "PUT":
request = new HttpPut(uri);
break;
case "DELETE":
request = new HttpDelete(uri);
break;
default:
request = new HttpGet(uri);
break;
}
if (request instanceof HttpPost || request instanceof HttpPut) {
if (StringUtils.isNotBlank(body)) {
request.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON));
}
}
return request;
}
@Override
public void destroy() {
}
}
}
runtime/datax/apireader/src/main/java/com/deepdata/plugin/reader/apireader/HttpEndpointSecurity.java
+175
View File
@@ -0,0 +1,175 @@
package com.deepdata.plugin.reader.apireader;
import java.net.IDN;
import java.net.InetAddress;
import java.net.URI;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
final class HttpEndpointSecurity {
private static final String ALLOWED_HOSTS_PROPERTY = "deepdata.ssrf.allowedHosts";
private static final String ALLOWED_HOSTS_ENV = "DEEPDATA_SSRF_ALLOWED_HOSTS";
private HttpEndpointSecurity() {
}
static URI validateExternalHttpUri(String rawUri, String fieldName) {
if (isBlank(rawUri)) {
throw new IllegalArgumentException(fieldName + " is required");
}
if (rawUri.indexOf('\r') >= 0 || rawUri.indexOf('\n') >= 0) {
throw new IllegalArgumentException(fieldName + " contains invalid control characters");
}
URI uri = URI.create(rawUri.trim()).normalize();
String scheme = uri.getScheme() == null ? null : uri.getScheme().toLowerCase(Locale.ROOT);
if (!"http".equals(scheme) && !"https".equals(scheme)) {
throw new IllegalArgumentException(fieldName + " only supports http or https");
}
if (!isBlank(uri.getUserInfo())) {
throw new IllegalArgumentException(fieldName + " must not contain user info");
}
if (uri.getFragment() != null) {
throw new IllegalArgumentException(fieldName + " must not contain fragment");
}
int port = uri.getPort();
if (port == 0 || port < -1 || port > 65535) {
throw new IllegalArgumentException(fieldName + " contains invalid port");
}
String asciiHost = toAsciiHost(uri.getHost(), fieldName);
if (isAllowedHost(asciiHost)) {
return uri;
}
InetAddress[] addresses;
try {
addresses = InetAddress.getAllByName(asciiHost);
} catch (Exception e) {
throw new IllegalArgumentException(fieldName + " host cannot be resolved", e);
}
if (addresses.length == 0) {
throw new IllegalArgumentException(fieldName + " host cannot be resolved");
}
for (InetAddress address : addresses) {
if (!isPublicRoutableAddress(address)) {
throw new IllegalArgumentException(fieldName + " points to a restricted network address");
}
}
return uri;
}
private static String toAsciiHost(String host, String fieldName) {
if (isBlank(host)) {
throw new IllegalArgumentException(fieldName + " must contain a valid host");
}
try {
String normalized = host.endsWith(".") ? host.substring(0, host.length() - 1) : host;
return IDN.toASCII(normalized, IDN.USE_STD3_ASCII_RULES).toLowerCase(Locale.ROOT);
} catch (Exception e) {
throw new IllegalArgumentException(fieldName + " contains invalid host", e);
}
}
private static boolean isAllowedHost(String host) {
return configuredAllowedHosts().contains(host);
}
private static Set<String> configuredAllowedHosts() {
Set<String> hosts = new HashSet<String>();
addConfiguredHosts(hosts, System.getProperty(ALLOWED_HOSTS_PROPERTY));
addConfiguredHosts(hosts, System.getenv(ALLOWED_HOSTS_ENV));
return hosts;
}
private static void addConfiguredHosts(Set<String> hosts, String rawHosts) {
if (isBlank(rawHosts)) {
return;
}
String[] parts = rawHosts.split(",");
for (String part : parts) {
if (!isBlank(part)) {
hosts.add(toAsciiHost(part.trim(), "allowed host"));
}
}
}
private static boolean isPublicRoutableAddress(InetAddress address) {
if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress()
|| address.isSiteLocalAddress() || address.isMulticastAddress()) {
return false;
}
byte[] bytes = address.getAddress();
if (bytes.length == 4) {
return isPublicRoutableIpv4(bytes);
}
if (bytes.length == 16) {
return isPublicRoutableIpv6(bytes);
}
return false;
}
private static boolean isPublicRoutableIpv4(byte[] bytes) {
int first = bytes[0] & 0xff;
int second = bytes[1] & 0xff;
int third = bytes[2] & 0xff;
if (first == 0 || first == 10 || first == 127 || first >= 224) {
return false;
}
if (first == 100 && second >= 64 && second <= 127) {
return false;
}
if (first == 169 && second == 254) {
return false;
}
if (first == 172 && second >= 16 && second <= 31) {
return false;
}
if (first == 192 && second == 168) {
return false;
}
if (first == 192 && second == 0 && (third == 0 || third == 2)) {
return false;
}
if (first == 198 && (second == 18 || second == 19 || (second == 51 && third == 100))) {
return false;
}
if (first == 203 && second == 0 && third == 113) {
return false;
}
return true;
}
private static boolean isPublicRoutableIpv6(byte[] bytes) {
int first = bytes[0] & 0xff;
int second = bytes[1] & 0xff;
if ((first & 0xfe) == 0xfc) {
return false;
}
if (first == 0xfe && (second & 0xc0) == 0x80) {
return false;
}
if (first == 0x20 && second == 0x01 && (bytes[2] & 0xff) == 0x0d && (bytes[3] & 0xff) == 0xb8) {
return false;
}
return !isIpv4MappedPrivateAddress(bytes);
}
private static boolean isIpv4MappedPrivateAddress(byte[] bytes) {
for (int i = 0; i < 10; i++) {
if (bytes[i] != 0) {
return false;
}
}
if ((bytes[10] & 0xff) != 0xff || (bytes[11] & 0xff) != 0xff) {
return false;
}
byte[] ipv4 = new byte[] {bytes[12], bytes[13], bytes[14], bytes[15]};
return !isPublicRoutableIpv4(ipv4);
}
private static boolean isBlank(String value) {
return value == null || value.trim().isEmpty();
}
}
runtime/datax/apireader/src/main/resources/plugin.json
+6
View File
@@ -0,0 +1,6 @@
{
"name": "apireader",
"class": "com.deepdata.plugin.reader.apireader.APIReader",
"description": "Read the structured data returned by the api",
"developer": "deepdata"
}
runtime/datax/apireader/src/main/resources/plugin_job_template.json
+20
View File
@@ -0,0 +1,20 @@
{
"name": "apireader",
"parameter": {
"api": "https://api.example.com/data",
"method": "GET",
"body": "{\"name\": \"value\"}",
"headers": {
"Content-Type": "application/json"
},
"schema": {
"dataPath": "$.data",
"fields": [
{
"name": "name",
"path": "alias"
}
]
}
}
}
runtime/datax/apireader/src/test/java/com/deepdata/plugin/reader/apireader/HttpEndpointSecurityTest.java
+83
View File
@@ -0,0 +1,83 @@
package com.deepdata.plugin.reader.apireader;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class HttpEndpointSecurityTest {
@Test(expected = IllegalArgumentException.class)
public void shouldRejectLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://127.0.0.1:8080/internal", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectLocalhost() {
HttpEndpointSecurity.validateExternalHttpUri("http://localhost:8080/internal", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectIpv6LoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://[::1]:8080/internal", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectIpv4MappedIpv6LoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://[::ffff:127.0.0.1]:8080/internal", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectIntegerEncodedLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://2130706433/internal", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectHexEncodedLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://0x7f000001/internal", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectOctalEncodedLoopbackHost() {
HttpEndpointSecurity.validateExternalHttpUri("http://017700000001/internal", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectMetadataServiceAddress() {
HttpEndpointSecurity.validateExternalHttpUri("http://169.254.169.254/latest/meta-data", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectPrivateNetworkAddress() {
HttpEndpointSecurity.validateExternalHttpUri("http://10.0.0.1/admin", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectUnsupportedScheme() {
HttpEndpointSecurity.validateExternalHttpUri("file:///etc/passwd", "api");
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectUserInfo() {
HttpEndpointSecurity.validateExternalHttpUri("http://user@8.8.8.8/path", "api");
}
@Test
public void shouldAllowPublicHttpAddress() {
assertEquals("8.8.8.8", HttpEndpointSecurity.validateExternalHttpUri("https://8.8.8.8/data", "api").getHost());
}
@Test
public void shouldAllowExplicitlyConfiguredHost() {
String previous = System.getProperty("deepdata.ssrf.allowedHosts");
try {
System.setProperty("deepdata.ssrf.allowedHosts", "localhost");
assertEquals("localhost",
HttpEndpointSecurity.validateExternalHttpUri("http://localhost:9000/data", "api").getHost());
} finally {
if (previous == null) {
System.clearProperty("deepdata.ssrf.allowedHosts");
} else {
System.setProperty("deepdata.ssrf.allowedHosts", previous);
}
}
}
}