Develop

common/src/main/java/com/robin/comm/util/gis/GpsConvertor.java

@@ -0,0 +1,90 @@
+package com.robin.comm.util.gis;
+
+import com.esri.core.geometry.Point;
+import com.robin.core.base.exception.MissingConfigException;
+import org.springframework.util.Assert;
+
+@SuppressWarnings("unused")
+public class GpsConvertor {
+    private static final double a = 6378245.0;
+    private static final double ee = 0.00669342162296594323;
+    private static final double x_pi = 3.14159265358979324 * 3000.0 / 180.0;
+
+    public static boolean gpsInChina(Point point){
+        Assert.notNull(point,"point must not be null");
+        double latitude=point.getY();
+        double longtitude=point.getX();
+        boolean inFlag= !(longtitude < 72.004) && !(longtitude > 137.8347);
+        if(latitude<0.8293 || latitude>55.8271){
+            inFlag=false;
+        }
+        if ((119.962 < longtitude && longtitude < 121.750) && (21.586 < latitude && latitude < 25.463)){
+            inFlag=false;
+        }
+        return inFlag;
+    }
+    public static Point BAIDU_to_WGS84(Point point){
+        if(!gpsInChina(point)){
+            throw new MissingConfigException("gps not in china range! can not convert");
+        }
+        double x = point.getX() - 0.0065;
+        double y = point.getY() - 0.006;
+        double z = Math.sqrt(x * x + y * y) - 0.00002 * Math.sin(y * x_pi);
+        double theta = Math.atan2(y, x) - 0.000003 * Math.cos(x * x_pi);
+        return GCJ02_to_WGS84(new Point(z * Math.cos(theta),z * Math.sin(theta)));
+    }
+    public static Point WGS84_to_BAIDU(Point point){
+        if(!gpsInChina(point)){
+            throw new MissingConfigException("gps not in china range! can not convert");
+        }
+        double x = point.getX();
+        double y = point.getY();
+
+        double z = Math.sqrt(x * x + y * y) + 0.00002 * Math.sin(y * x_pi);
+        double theta = Math.atan2(y, x) + 0.000003 * Math.cos(x * x_pi);
+        double bdLng = z * Math.cos(theta) + 0.0065;
+        double bdLat = z * Math.sin(theta) + 0.006;
+        return new Point(bdLng,bdLat);
+    }
+    public static Point GCJ02_to_WGS84(Point point){
+        if(!gpsInChina(point)){
+            throw new MissingConfigException("gps not in china range! can not convert");
+        }
+        Point tmpLatLng = WGS84_to_GCJ02(point);
+        double tmpLat = 2 * point.getY()- tmpLatLng.getY();
+        double tmpLng = 2 * point.getX() - tmpLatLng.getX();
+
+        //第二次纠偏
+        tmpLatLng = WGS84_to_GCJ02(point);
+        tmpLat = 2 * tmpLat - tmpLatLng.getY();
+        tmpLng = 2 * tmpLng - tmpLatLng.getX();
+        return new Point(tmpLng,tmpLat);
+    }
+    private static Point WGS84_to_GCJ02(Point point){
+        if(!gpsInChina(point)){
+            throw new MissingConfigException("gps not in china range! can not convert");
+        }
+        double dLat = transformLat(point.getX() - 105.0, point.getY() - 35.0);
+        double dLon = transformLon(point.getX() - 105.0, point.getY() - 35.0);
+        double radLat = point.getY() / 180.0 * Math.PI;
+        double magic = Math.sin(radLat);
+        magic = 1 - ee * magic * magic;
+        double sqrtMagic = Math.sqrt(magic);
+        return new Point(point.getX()+(dLon * 180.0) / (a / sqrtMagic * Math.cos(radLat) * Math.PI),point.getY()+(dLat * 180.0) / ((a * (1 - ee)) / (magic * sqrtMagic) * Math.PI));
+    }
+    private static double transformLat(Double x, Double y){
+        double ret = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * Math.sqrt(Math.abs(x));
+        ret += (20.0 * Math.sin(6.0 * x * Math.PI) + 20.0 * Math.sin(2.0 * x * Math.PI)) * 2.0 / 3.0;
+        ret += (20.0 * Math.sin(y * Math.PI) + 40.0 * Math.sin(y / 3.0 * Math.PI)) * 2.0 / 3.0;
+        ret += (160.0 * Math.sin(y / 12.0 * Math.PI) + 320 * Math.sin(y * Math.PI / 30.0)) * 2.0 / 3.0;
+        return ret;
+    }
+
+    private static double transformLon(Double x, Double y){
+        double ret = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * Math.sqrt(Math.abs(x));
+        ret += (20.0 * Math.sin(6.0 * x * Math.PI) + 20.0 * Math.sin(2.0 * x * Math.PI)) * 2.0 / 3.0;
+        ret += (20.0 * Math.sin(x * Math.PI) + 40.0 * Math.sin(x / 3.0 * Math.PI)) * 2.0 / 3.0;
+        ret += (150.0 * Math.sin(x / 12.0 * Math.PI) + 300.0 * Math.sin(x / 30.0 * Math.PI)) * 2.0 / 3.0;
+        return ret;
+    }
+}

common/src/main/java/com/robin/core/fileaccess/fs/AbstractFileSystemAccessor.java

@@ -100,17 +100,7 @@ protected static InputStream wrapInputStream(InputStream instream){
 	protected static OutputStream getOutputStreamByPath(String path, OutputStream out) throws IOException{
 		return CompressEncoder.getOutputStreamByCompressType(path,out);
 	}
-	protected OutputStream getOutputStream(DataCollectionMeta meta) throws IOException {
-		OutputStream outputStream;
-		if(!ObjectUtils.isEmpty(meta.getResourceCfgMap().get(ResourceConst.USETMPFILETAG)) && "true".equalsIgnoreCase(meta.getResourceCfgMap().get(ResourceConst.USETMPFILETAG).toString())){
-			String tmpPath = com.robin.core.base.util.FileUtils.getWorkingPath(meta);
-			String tmpFilePath =  tmpPath + ResourceUtil.getProcessFileName(meta.getPath());
-			outputStream= Files.newOutputStream(Paths.get(tmpFilePath));
-		}else {
-			outputStream = new ByteArrayOutputStream();
-		}
-		return outputStream;
-	}
+
 
 	@Override
 	public void init(DataCollectionMeta meta){

common/src/main/java/com/robin/core/resaccess/iterator/JdbcResIterator.java

@@ -3,24 +3,26 @@
 import com.robin.comm.dal.holder.db.DbConnectionHolder;
 import com.robin.comm.dal.holder.db.JdbcResourceHolder;
 import com.robin.comm.dal.pool.ResourceAccessHolder;
-import com.robin.core.base.dao.SimpleJdbcDao;
+import com.robin.core.base.dao.CommJdbcUtil;
+import com.robin.core.base.exception.GenericException;
 import com.robin.core.base.exception.OperationNotSupportException;
 import com.robin.core.base.spring.SpringContextHolder;
+import com.robin.core.base.util.Const;
 import com.robin.core.fileaccess.fs.AbstractFileSystemAccessor;
 import com.robin.core.fileaccess.iterator.AbstractResIterator;
 import com.robin.core.fileaccess.meta.DataCollectionMeta;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.dbutils.DbUtils;
 import org.apache.commons.dbutils.QueryRunner;
+import org.springframework.jdbc.support.lob.DefaultLobHandler;
+import org.springframework.jdbc.support.lob.LobHandler;
 import org.springframework.retry.annotation.Backoff;
 import org.springframework.retry.annotation.Retryable;
 import org.springframework.util.Assert;
-import org.springframework.util.StringUtils;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.sql.*;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -33,9 +35,11 @@ public class JdbcResIterator extends AbstractResIterator {
     private ResultSet rs;
     private Statement statement;
     private PreparedStatement preparedStatement;
+    private LobHandler lobHandler;
 
     public JdbcResIterator() {
-
+        this.identifier= Const.ACCESSRESOURCE.JDBC.getValue();
+        lobHandler=new DefaultLobHandler();
     }
 
     public JdbcResIterator(DataCollectionMeta colmeta) {
@@ -101,16 +105,22 @@ public boolean hasNext() {
 
     @Override
     public Map<String, Object> next() {
-        return null;
-    }
-
-    @Override
-    public String getIdentifier() {
-        return "jdbc";
+        Map<String, Object> map = new HashMap<>();
+        try {
+            ResultSetMetaData rsmd = rs.getMetaData();
+            int count = rsmd.getColumnCount();
+            for (int i = 0; i < count; i++) {
+                String columnName = rsmd.getColumnName(i + 1);
+                map.put(columnName, CommJdbcUtil.getRecordValue(rsmd,rs,lobHandler,i));
+            }
+        }catch (SQLException ex){
+            throw new GenericException(ex);
+        }
+        return map;
     }
 
     @Override
     public void setAccessUtil(AbstractFileSystemAccessor accessUtil) {
-        throw new OperationNotSupportException("");
+        throw new OperationNotSupportException("jdbc can not access format data file!");
     }
 }

core/src/main/java/com/robin/core/base/dao/CommJdbcUtil.java

@@ -276,7 +276,7 @@ private static ResultSetExtractor<List<Map<String,Object>>> getDefaultExtract(fi
         };
     }
 
-    private Object getRecordValue(ResultSetMetaData rsmd, ResultSet rs, LobHandler lobHandler, int pos) throws SQLException {
+    public static Object getRecordValue(ResultSetMetaData rsmd, ResultSet rs, LobHandler lobHandler, int pos) throws SQLException {
         int columnType = rsmd.getColumnType(pos + 1);
         rs.getObject(pos + 1);
         Object retObj = null;

core/src/main/java/com/robin/core/base/datameta/BaseDataBaseMeta.java

@@ -44,7 +44,7 @@ public abstract class BaseDataBaseMeta implements DataBaseInterface, Serializabl
 	protected DataBaseParam param;
 	protected String dbType;
 	//Enum type of all support DB
-	public static final String[] DB_TYPE_ENMU ={"Oracle","Mysql","DB2","Sqlserver","Sybase","Postgresql","Phoenix4","Hive1","Hive","Oraclerac","H2","Impala"};
+	public static final String[] DB_TYPE_ENMU ={TYPE_ORACLE,TYPE_MYSQL,TYPE_DB2,TYPE_SQLSERVER,TYPE_SYBASE,TYPE_PGSQL,TYPE_PHONEIX,TYPE_HIVE,TYPE_HIVE2,TYPE_ORACLERAC,TYPE_H2,TYPE_IMPALA};
 	//jdbc Url Template like jdbc:mysql://[hostName]:[port]/[databaseName]?useUnicode=true&characterEncoding=[encode]
 	public static final Pattern PATTERN_TEMPLATE_PARAM = Pattern.compile("\\[.*?\\]");
 	@Override
@@ -125,7 +125,7 @@ public String getAddColumnStatement(String tableName, String schema,
 										DataBaseColumnMeta v) {
 		StringBuilder builder=new StringBuilder();
 		builder.append("ALTER TABLE ").append(getTableSpec(tableName,schema)).append(" ADD COLUMN ");
-		builder.append(v.getColumnName()).append(" ").append(getSqlGen().returnTypeDef(v.getColumnType().toString(),v));
+		builder.append(v.getColumnName()).append(" ").append(getSqlGen().returnTypeDef(v.getColumnType(),v));
 		if(v.isIncrement() && supportAutoInc()){
 			builder.append(" ").append(getSqlGen().getAutoIncrementDef());
 		}

core/src/main/java/com/robin/core/base/util/Const.java

@@ -393,7 +393,10 @@ public enum FILESYSTEM{
         S3("s3"),
         ALIYUN("oss"),
         TENCENT("cos"),
-        QINIU("qiniu");
+        QINIU("qiniu"),
+        BAIDU_BOS("bos"),
+        HUAWEI_OBS("obs"),
+        MINIO("minio");
         private String value;
         FILESYSTEM(String value){
             this.value=value;
@@ -409,6 +412,7 @@ public enum ACCESSRESOURCE{
         RABBITMQ("rabbitmq"),
         ROCKETMQ("rocket"),
         ZEROQ("zero"),
+        HBASE("hbase"),
         CLICKHOUSE("clickhouse"),
         CASSANDRA("cassandra");
 

core/src/main/java/com/robin/core/base/util/ResourceConst.java

@@ -5,6 +5,8 @@ public class ResourceConst {
     public static final String WORKINGPATHPARAM="output.workingPath";
     public static final String USETMPFILETAG="output.usingTmpFiles";
     public static final String BUCKETNAME="bucketName";
+    public static final String ALLOWOFFHEAPKEY="allowOffHeapMemLimit";
+    public static final Double ALLOWOUFHEAPMEMLIMIT=4000.0;
 
     public enum IngestType {
         TYPE_HDFS(1L,"HDFS"),
@@ -196,4 +198,18 @@ public String getValue() {
         }
 
     }
+    public enum OBSPARAM{
+        ENDPOIN("endpoint"),
+        REGION("region"),
+        ACESSSKEYID("accessKeyId"),
+        SECURITYACCESSKEY("securityAccessKey");
+        private String value;
+        OBSPARAM(String value){
+            this.value=value;
+        }
+
+        public String getValue() {
+            return value;
+        }
+    }
 }

core/src/main/java/com/robin/core/sql/util/AbstractSqlGen.java

@@ -75,15 +75,9 @@ private String doFilterSql(List<FilterCondition> paramList,String linkOper){
             if (ObjectUtils.isEmpty(param.getValue()) || CollectionUtils.isEmpty(param.getConditions())) {
                 continue;
             }
-            String prevoper = param.getPrefixOper();
-            String nextoper = param.getSuffixOper();
+            String prevoper = !ObjectUtils.isEmpty(param.getPrefixOper())?param.getPrefixOper():"";
+            String nextoper = !ObjectUtils.isEmpty(param.getSuffixOper())?param.getSuffixOper():"";
 
-            if (prevoper == null || prevoper.length() == 0) {
-                prevoper = "";
-            }
-            if (nextoper == null || nextoper.length() == 0) {
-                nextoper = "";
-            }
             if(org.apache.commons.lang3.StringUtils.isEmpty(linkOper)) {
                 replaceStr = Const.OPERATOR_AND;
             }
@@ -284,7 +278,7 @@ public String getFieldDefineSqlPart(FieldContent field) {
     }
     @Override
     public String getFieldDefineSqlByMeta(DataBaseColumnMeta columnMeta){
-        String datatype = columnMeta.getColumnType().toString();
+        String datatype = columnMeta.getColumnType();
         StringBuilder builder = new StringBuilder();
         String name = columnMeta.getColumnName();
         builder.append(name).append(" ").append(returnTypeDef(datatype, columnMeta));
@@ -620,7 +614,7 @@ public String getSchemaName(String schema) {
             return "\"" + schema + "\"";
         }
     }
-    protected String getNoPageSql(String sql,PageQuery pageQuery){
+    protected String getNoPageSql(String sql,PageQuery<?> pageQuery){
         Assert.isTrue(pageQuery.getPageSize()==0,"");
         StringBuilder builder=new StringBuilder(sql);
         if(!StringUtils.isEmpty(pageQuery.getOrder()) && !StringUtils.isEmpty(pageQuery.getOrderDirection())){
@@ -629,7 +623,7 @@ protected String getNoPageSql(String sql,PageQuery pageQuery){
         return builder.toString();
     }
 
-    protected Integer[] getStartEndRecord(PageQuery pageQuery) {
+    protected Integer[] getStartEndRecord(PageQuery<?> pageQuery) {
         int nBegin = (pageQuery.getPageNumber() - 1) * pageQuery.getPageSize();
         int tonums = nBegin + pageQuery.getPageSize();
         if (!ObjectUtils.isEmpty(pageQuery.getRecordCount()) && pageQuery.getRecordCount() < tonums) {
@@ -718,7 +712,7 @@ else if(!org.springframework.util.StringUtils.isEmpty(pageQuery.getOrder())){
         pagingSelect.append(" )r) r ");
         return pagingSelect;
     }
-    protected void checkSqlAndPage(String sql,PageQuery pageQuery){
+    protected void checkSqlAndPage(String sql,PageQuery<?> pageQuery){
         Assert.isTrue(!ObjectUtils.isEmpty(sql),"querySql not null");
         //Assert.notNull(pageQuery,"pageQuery is Null");
     }