Skip to content

成鋆 / sohu.port

Go to a project
Toggle navigation
Toggle navigation pinning
Commit a844c69f a844c69f6fc8b446255097556682c6e05b2b6ef8 by chengyun
Options

initialize

0 parents
Inline Side-by-side
Showing 97 changed files with 3136 additions and 0 deletions
#java相关
libs/*
*.iml
*.stackdump
*.orig
.idea
.settings
.classpath
.project
*/src/main/webapp/build/*
# */src/main/webapp/ext/*
bootstrap.*
*/target
*/*/target
*/*.zip
*/*.tar
*.m3u8
/prepared/m3u8/20180608/drama/4730_1.m3u8
/prepared/m3u8/20180608/drama/4946_1.m3u8
/prepared/m3u8/20180608/drama/4734_1.m3u8
/prepared/m3u8/20180608/drama/4948_1.m3u8
/prepared/m3u8/20180608/drama/4760_1.m3u8
/prepared/m3u8/20180608/drama/4772_1.m3u8
/prepared/m3u8/20180608/drama/4806_1.m3u8
/prepared/m3u8/20180608/drama/4976_1.m3u8
No preview for this file type
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.topdraw.sohu</groupId>
<artifactId>port</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>port Maven Webapp</name>
<dependencies>
<dependency>
<groupId>org.afflatus</groupId>
<artifactId>utility</artifactId>
<version>1.0.3</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.1.6</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.21</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.20</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.26</version>
</dependency>
<dependency>
<groupId>com.thoughtworks.xstream</groupId>
<artifactId>xstream</artifactId>
<version>1.4.7</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
<version>2.2.3</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz-jobs</artifactId>
<version>2.2.3</version>
</dependency>
<dependency>
<groupId>javax.transaction</groupId>
<artifactId>jta</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.3</version>
</dependency>
<dependency>
<groupId>org.dom4j</groupId>
<artifactId>dom4j</artifactId>
<version>2.0.0</version>
</dependency>
</dependencies>
<build>
<finalName>port</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>
./target/lib
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_war_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20.1</version>
</plugin>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<version>3.2.0</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
package com.topdraw.sohu.listener;
import com.mysql.jdbc.AbandonedConnectionCleanupThread;
import com.topdraw.sohu.utils.PropertiesUtil;
import org.afflatus.utility.DruidUtil;
import org.afflatus.utility.MemcachedClient;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.impl.StdSchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import javax.servlet.annotation.WebListener;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
import java.util.Enumeration;
@WebListener
public class ServerListener implements ServletContextListener {
private static Logger logger;
private static final String CATALINA_BASE = "catalina.base";
private static final String JETTY_HOME = "jetty.home";
private static final String PROJECT_NAME = "project.name";
private static final String PROJECT_VERSION = "project.version";
public static Scheduler scheduler = null;
public static Boolean bStarted = true;
@Override
public void contextDestroyed(ServletContextEvent event) {
logger.debug("contextDestroyed");
if (null != scheduler) {
try {
// scheduler.pauseAll();
bStarted = false;
scheduler.shutdown(true);
} catch (SchedulerException e) {
logger.error("scheduler shutdown error", e);
}
}
// try {
// Thread.sleep(2000);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
try {
Enumeration<Driver> drivers = DriverManager.getDrivers();
while (drivers.hasMoreElements()) {
Driver driver = drivers.nextElement();
DriverManager.deregisterDriver(driver);
}
AbandonedConnectionCleanupThread.shutdown();
DruidUtil.closeAllDataSource();
MemcachedClient.shutdown();
} catch (Exception ex) {
ex.printStackTrace();
}
}
@Override
public void contextInitialized(ServletContextEvent event) {
if (null == System.getProperty(CATALINA_BASE)) {
String home = System.getProperty(JETTY_HOME);
if (null != home) {
System.setProperty(CATALINA_BASE, home);
} else {
System.setProperty(CATALINA_BASE, ".");
}
}
System.setProperty(PROJECT_NAME, PropertiesUtil.get(PROJECT_NAME));
System.setProperty("druid.registerToSysProperty", "true");
logger = LoggerFactory.getLogger(ServerListener.class);
logger.debug("contextInitialized [name:"
+ PropertiesUtil.get(PROJECT_NAME) + ", version:"
+ PropertiesUtil.get(PROJECT_VERSION) + "]");
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
} catch (Exception ex) {
logger.error("Connection init error", ex);
} finally {
if (null != readConnection) {
DruidUtil.close(readConnection);
}
if (null != writeConnection) {
DruidUtil.close(writeConnection);
}
}
try {
SchedulerFactory sf = new StdSchedulerFactory("quartz.properties");
scheduler = sf.getScheduler();
scheduler.start();
} catch (Exception e) {
logger.error("start quartz error", e);
}
}
}
package com.topdraw.sohu.port.job;
import com.topdraw.sohu.listener.ServerListener;
import com.topdraw.sohu.utils.SohuUtil;
import org.afflatus.utility.DruidUtil;
import org.quartz.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@DisallowConcurrentExecution
public class M3U8Job implements Job {
private static final Logger log = LoggerFactory.getLogger(M3U8Job.class);
private List<Map<String, Object>> listCmd = new ArrayList<>();
public void execute(JobExecutionContext context) throws JobExecutionException {
Connection writeConnection = null;
try {
writeConnection = DruidUtil.getRandomWriteConnection();
log.info("M3U8Job start");
listCmd = getTodoCmd();
if (listCmd.size() == 0) {
log.info("no cmd to download m3u8");
return;
} else {
if (null != ServerListener.scheduler) {
ServerListener.scheduler.pauseJob(context.getJobDetail().getKey());
}
}
while (listCmd.size() != 0) {
if (!ServerListener.bStarted) {
// tomcat 是否正在停止
log.info("tomcat is shutting down, the excuting job break");
break;
}
for (Map<String, Object> mapCmd : listCmd) {
SohuUtil util = new SohuUtil();
util.downloadM3U8ByVideoId((Long) mapCmd.get("video_id"));
}
listCmd = getTodoCmd();
}
if (null != ServerListener.scheduler) {
ServerListener.scheduler.resumeJob(context.getJobDetail().getKey());
}
} catch (Exception e) {
log.error("M3U8Job error", e);
if (null != ServerListener.scheduler) {
try {
ServerListener.scheduler.resumeJob(context.getJobDetail().getKey());
} catch (SchedulerException se) {
// TODO Auto-generated catch block
se.printStackTrace();
}
}
} finally {
DruidUtil.close(writeConnection);
}
}
private static List<Map<String, Object>> getTodoCmd() {
log.info("start get cmd to do...");
Connection readConnection = null;
List<Map<String, Object>> listRet = new ArrayList<>();
try {
readConnection = DruidUtil.getRandomReadConnection();
String sql = "SELECT * FROM sohu_video WHERE" +
//" tvPlayType <> 1 AND" +
" m3u8_path IS NULL" +
" AND definition IS NOT NULL LIMIT 0, 20";
listRet = DruidUtil.queryList(readConnection, sql);
} catch (Exception ex) {
ex.printStackTrace();
} finally {
DruidUtil.close(readConnection);
}
return listRet;
}
}
package com.topdraw.sohu.port.job;
import com.topdraw.sohu.listener.ServerListener;
import com.topdraw.sohu.utils.SohuUtil;
import org.quartz.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@DisallowConcurrentExecution
public class MetaDataJob implements Job {
private static final Logger log = LoggerFactory.getLogger(MetaDataJob.class);
public void execute(JobExecutionContext context) throws JobExecutionException {
try {
log.info("MetaDataJob start");
ServerListener.scheduler.pauseJob(context.getJobDetail().getKey());
SohuUtil util = new SohuUtil();
util.metaDataIncrementInjection();
//for (int i = 9; i <= 9; i++) {
// util.metaDataIncrementInjectionByXml("2018070" + i);
//}
if (null != ServerListener.scheduler) {
ServerListener.scheduler.resumeJob(context.getJobDetail().getKey());
}
} catch (Exception e) {
log.error("MetaDataJob error", e);
if (null != ServerListener.scheduler) {
try {
ServerListener.scheduler.resumeJob(context.getJobDetail().getKey());
} catch (SchedulerException se) {
// TODO Auto-generated catch block
se.printStackTrace();
}
}
}
}
}
package com.topdraw.sohu.utils;
import java.sql.Connection;
import java.util.HashMap;
import java.util.Map;
import org.afflatus.utility.DruidUtil;
import com.alibaba.fastjson.JSON;
public class BIInterfaceUtil {
private static final String BI_SWITCH = PropertiesUtil.get("BI_SWITCH");
public static void createMediaAssetsPublish(String strAction, String strPlatform, String strType, String strName,
Map<String, Object> mapData, Connection writeConnection) {
try {
if (BI_SWITCH.equals("off")) {
return;
}
Map<String, Object> mapSave = new HashMap<String, Object>();
mapSave.put("action", strAction);
mapSave.put("platform", strPlatform);
mapSave.put("type", strType);
mapSave.put("name", strName);
if (mapData.containsKey("create_time")) {
mapData.remove("create_time");
}
if (mapData.containsKey("update_time")) {
mapData.remove("update_time");
}
if (JSON.toJSONString(mapData).length() > 4096) {
if (mapData.containsKey("image")) {
mapData.remove("image");
}
if (mapData.containsKey("images")) {
mapData.remove("images");
}
}
mapSave.put("data", JSON.toJSONString(mapData));
DruidUtil.save(writeConnection, mapSave, "x_media_assets_publish");
} catch (Exception ex) {
ex.printStackTrace();
} finally {
}
}
}
package com.topdraw.sohu.utils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
public class FTPUtils {
private static final Logger logger = LoggerFactory
.getLogger(FTPUtils.class);
private final String PROJECT_PATH = this.getClass().getClassLoader().getResource("../../") == null ?
(System.getProperty("user.dir").endsWith("/") ? System.getProperty("user.dir")
: System.getProperty("user.dir") + "/")
: this.getClass().getClassLoader().getResource("../../").getPath();
/**
* Description: 向FTP服务器上传文件
* @param url FTP服务器hostname
* @param port FTP服务器端口
* @param username FTP登录账号
* @param password FTP登录密码
* @param path FTP服务器保存目录
* @param filename 上传到FTP服务器上的文件名
* @param input 输入流
* @return 成功返回true，否则返回false
*/
public boolean uploadFile(String url, int port, String username, String password, String path, String filename, InputStream input) {
boolean success = false;
FTPClient ftp = new FTPClient();
try {
int reply;
ftp.connect(url, port);//连接FTP服务器
//如果采用默认端口，可以使用ftp.connect(url)的方式直接连接FTP服务器
ftp.login(username, password);//登录
reply = ftp.getReplyCode();
System.out.println("ReplyCode: " + reply);
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
return success;
}
ftp.changeWorkingDirectory(path);
ftp.storeFile(filename, input);
input.close();
ftp.logout();
success = true;
} catch (IOException e) {
e.printStackTrace();
} finally {
if (ftp.isConnected()) {
try {
ftp.disconnect();
} catch (IOException ioe) {
}
}
}
return success;
}
/**
* Description: 从FTP服务器下载文件
* @param url FTP服务器hostname
* @param port FTP服务器端口
* @param username FTP登录账号
* @param password FTP登录密码
* @param remotePath FTP服务器上的相对路径
* @param fileName 要下载的文件名
* @param localPath 下载后保存到本地的路径
* @return
*/
public static boolean downFile(
String url, int port, String username, String password, String remotePath, String fileName,
String localPath) {
boolean success = false;
FTPClient ftp = new FTPClient();
try {
int reply;
ftp.setConnectTimeout(10 * 1000);
ftp.connect(url, port);
//如果采用默认端口，可以使用ftp.connect(url)的方式直接连接FTP服务器
ftp.login(username, password);//登录
ftp.setBufferSize(102400);
ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
reply = ftp.getReplyCode();
System.out.println("ReplyCode: " + reply);
if (!FTPReply.isPositiveCompletion(reply)) {
logger.info("isPositiveCompletion");
ftp.disconnect();
return success;
}
logger.info("remotePath: " + remotePath);
ftp.changeWorkingDirectory(remotePath);//转移到FTP服务器目录
ftp.enterLocalPassiveMode();
FTPFile[] fs = ftp.listFiles();
for (FTPFile ff:fs) {
if (ff.getName().equals(fileName)) {
File localFile = new File(localPath + "/" + ff.getName());
logger.info("file size: " + ff.getSize());
OutputStream is = new FileOutputStream(localFile);
ftp.retrieveFile(ff.getName(), is);
is.close();
logger.info("download file size: " + localFile.length());
if (ff.getSize() == localFile.length()) {
success = true;
} else {
logger.info("file size mismatch: " + ff.getSize() + " -> " + localPath.length());
}
break;
}
}
ftp.logout();
} catch (IOException e) {
e.printStackTrace();
success = false;
} finally {
if (ftp.isConnected()) {
try {
ftp.disconnect();
} catch (IOException ioe) {
ioe.printStackTrace();
}
}
}
return success;
}
public String downFile(String strFtpPath, String strRelativePath) {
String strUrl = "", strUserName = "", strPassWord = "", strRemotePath = "", strFileName = "", strLocalPath = "";
int iPort = 21;
// 截取FTP地址
final String strFtpFlag = "ftp://";
if (strFtpPath != null && strFtpPath.length() > 0 && strFtpPath.toLowerCase().contains(strFtpFlag)) {
// 首先去掉FTP
final String cutedFtp = strFtpPath.substring(strFtpPath.indexOf(strFtpFlag) + strFtpFlag.length());
// 首先截取用户名、密码、ip、端口
String str4 = "";
if (cutedFtp.indexOf("/") != -1) {
str4 = cutedFtp.substring(0, cutedFtp.indexOf("/"));
} else {
str4 = cutedFtp;
}
// 截取用户名、密码
String strUsernameAndPwd = str4.substring(0, str4.indexOf("@"));
// 截取ip、端口
String strIpAndPort = str4.substring(str4.indexOf("@") + 1);
// 开始获取ip和端口
if (!"".equals(strIpAndPort)) {
if (strIpAndPort.indexOf(":") != -1) {
strUrl = strIpAndPort.substring(0, strIpAndPort.indexOf(":"));
String strPort = strIpAndPort.substring(strIpAndPort.indexOf(":") + 1, strIpAndPort.length());
if (strPort != null) {
iPort = Integer.parseInt(strPort);
}
} else {
// 如果没有端口只获取IP
strUrl = strIpAndPort;
}
}
// 开始获取用户名和密码
if (!"".equals(strUsernameAndPwd)) {
strUserName = strUsernameAndPwd.substring(0, strUsernameAndPwd.indexOf(":"));
strPassWord = strUsernameAndPwd.substring(strUsernameAndPwd.indexOf(":") + 1);
}
// 截取ftp文件路径和文件名
String strFileNameAndPath = "";
if (cutedFtp.indexOf("/") != -1) {
strFileNameAndPath = cutedFtp.substring(cutedFtp.indexOf("/") + 1, cutedFtp.length());
} else {
strFileNameAndPath = "";
}
// 开始获取ftp文件路径和文件名
if (!"".equals(strIpAndPort)) {
if (strFileNameAndPath.indexOf("/") != -1) {
strRemotePath = strFileNameAndPath.substring(0, strFileNameAndPath.lastIndexOf("/"));
strFileName = strFileNameAndPath.substring(strFileNameAndPath.lastIndexOf("/") + 1,
strFileNameAndPath.length());
} else {
strFileName = strFileNameAndPath;
}
}
}
strLocalPath = PROJECT_PATH + strRelativePath + strFileName;
logger.info("file download target path: " + strLocalPath);
File file = new File(strLocalPath);
if (!file.isDirectory()) { // 目录不存在
String[] aPathSegments = strLocalPath.split("/");
String strWalkThroughPath = "/";
for (int i = 0; i < aPathSegments.length - 1; i++) {
strWalkThroughPath = strWalkThroughPath + "/" + aPathSegments[i];
file = new File(strWalkThroughPath);
if (!file.isDirectory()) {
file.mkdir();
}
}
}
boolean b = FTPUtils.downFile(strUrl, iPort, strUserName, strPassWord, strRemotePath, strFileName,
PROJECT_PATH + strRelativePath);
if (!b) {
// 下载失败
logger.info("file download error: " + strLocalPath);
return "";
} else {
logger.info("file saved: " + strLocalPath);
}
return strLocalPath;
}
public static void main(String[] args) {
try {
//FileInputStream in = new FileInputStream(new File("D:/adv.txt"));
//boolean flag = uploadFile("10.50.127.181", 21, "root", "bestvwin", "/tmp", "adv.txt", in);
//boolean flag = downFile("172.25.44.26", 21, "wacos", "wacos", "/opt/wacos/CTMSData/picture/2018/02/07",
// "20180207112330_320029.jpg", "");
//System.out.println(flag);
} catch (Exception e) {
e.printStackTrace();
}
}
}
package com.topdraw.sohu.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
public class HttpConnectionUtil {
private static final Logger log = LoggerFactory
.getLogger(HttpConnectionUtil.class);
/**
* @param strUrlPath 下载路径
* @param strDownloadDir 下载存放目录
* @param strDownloadDir 下载文件重命名（null保持原有名字）
* @return 返回下载文件
*/
public static File downloadFile(String strUrlPath, String strDownloadDir, String strFileName) {
File file = null;
try {
// 统一资源
URL url = new URL(strUrlPath);
// 连接类的父类，抽象类
URLConnection urlConnection = url.openConnection();
// http的连接类
HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection;
// 设定请求的方法，默认是GET
httpURLConnection.setRequestMethod("GET");
// 设置字符编码
httpURLConnection.setRequestProperty("Charset", "UTF-8");
httpURLConnection.setRequestProperty("Accept-Encoding", "identity");
httpURLConnection.setReadTimeout(10000);
// 打开到此 URL 引用的资源的通信链接（如果尚未建立这样的连接）。
httpURLConnection.connect();
// 文件大小
int fileLength = httpURLConnection.getContentLength();
// 文件名
String filePathUrl = strFileName == null ? httpURLConnection.getURL().getFile() : strFileName;
String fileFullName = filePathUrl.substring(filePathUrl.lastIndexOf(File.separatorChar) + 1);
//log.info("file length -> " + fileLength);
URLConnection con = url.openConnection();
BufferedInputStream bin = new BufferedInputStream(httpURLConnection.getInputStream());
String path = strDownloadDir + File.separatorChar + fileFullName;
file = new File(path);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
OutputStream out = new FileOutputStream(file);
int size = 0;
int len = 0;
byte[] buf = new byte[1024];
while ((size = bin.read(buf)) != -1) {
len += size;
out.write(buf, 0, size);
// 打印下载百分比
// log.info("下载了 -> " + len * 100 / fileLength);
}
bin.close();
out.close();
if (file.length() != fileLength && fileLength != -1) {
log.info("file size mismatch: " + fileLength + " -> " + file.length());
file = null;
}
} catch (MalformedURLException e) {
e.printStackTrace();
file = null;
} catch (IOException e) {
e.printStackTrace();
file = null;
}
return file;
}
public static void main(String[] args) {
}
}
package com.topdraw.sohu.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ImageUtil {
public static Map<String, Integer> mapImageType = new HashMap<>();
static {
mapImageType.put("normal", -1);
mapImageType.put("thumbnail", 0);
mapImageType.put("poster", 1);
mapImageType.put("stills", 2);
mapImageType.put("icon", 3);
mapImageType.put("title", 4);
mapImageType.put("ad", 5);
mapImageType.put("sketch", 6);
mapImageType.put("background", 7);
mapImageType.put("channel", 9);
mapImageType.put("channel_bw", 10);
mapImageType.put("channel_logo", 11);
mapImageType.put("channel_name", 12);
mapImageType.put("other", 99);
}
public static String convertStrImages2SpecialFormatJSON(String strImages) {
JSONArray jsonArr;
if (strImages != null) {
jsonArr = JSONArray.parseArray(strImages);
} else {
jsonArr = new JSONArray();
}
Map<String, Object> mapImages = new HashMap<String, Object>();
mapImages.put("map", new HashMap<String, Object>());
mapImages.put("list", new ArrayList<Map<String, Object>>());
for (int i = 0; i < jsonArr.size(); i++) {
JSONObject jsonObj = jsonArr.getJSONObject(i);
String strKey = ResourceUtil.getImageKeyByType(jsonObj.containsKey("type") ? jsonObj.getIntValue("type") : -1);
if (((Map<String, Object>) mapImages.get("map")).containsKey(strKey)) {
((ArrayList<Integer>) ((Map<String, Object>) mapImages.get("map")).get(strKey)).add(i);
} else {
List<Integer> listIndex = new ArrayList<Integer>();
listIndex.add(i);
((Map<String, Object>) mapImages.get("map")).put(strKey, listIndex);
}
Map<String, Object> mapImageDetail = new HashMap<String, Object>();
mapImageDetail.put("id", jsonObj.getLong("id"));
mapImageDetail.put("fileUrl", jsonObj.getString("fileUrl"));
mapImageDetail.put("height", jsonObj.getInteger("height"));
mapImageDetail.put("width", jsonObj.getInteger("width"));
mapImageDetail.put("size", jsonObj.getInteger("size"));
mapImageDetail.put("extension", jsonObj.getString("extension"));
mapImageDetail.put("enable", jsonObj.getBoolean("enable"));
mapImageDetail.put("name", jsonObj.getString("name"));
mapImageDetail.put("csp_code", jsonObj.getString("csp_code"));
((ArrayList<Map<String, Object>>) mapImages.get("list")).add(mapImageDetail);
}
return JSON.toJSONString(mapImages);
}
}
package com.topdraw.sohu.utils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class M3U8 {
private String strBasePath;
private List<Ts> listTs = new ArrayList<>();
private long lStartTime;// 开始时间
private long lEndTime;// 结束时间
private long lStartDownloadTime;// 开始下载时间
private long lEndDownloadTime;// 结束下载时间
public String getStrBasePath() {
return strBasePath;
}
public void setStrBasePath(String strBasePath) {
this.strBasePath = strBasePath;
}
public List<Ts> getListTs() {
return listTs;
}
public void setListTs(List<Ts> listTs) {
this.listTs = listTs;
}
public void addTs(Ts ts) {
this.listTs.add(ts);
}
public long getLStartDownloadTime() {
return lStartDownloadTime;
}
public void setLStartDownloadTime(long lStartDownloadTime) {
this.lStartDownloadTime = lStartDownloadTime;
}
public long getLEndDownloadTime() {
return lEndDownloadTime;
}
public void setLEndDownloadTime(long lEndDownloadTime) {
this.lEndDownloadTime = lEndDownloadTime;
}
/**
* 获取开始时间
*
* @return
*/
public long getLStartTime() {
if (listTs.size() > 0) {
Collections.sort(listTs);
lStartTime = listTs.get(0).getLongDate();
return lStartTime;
}
return 0;
}
/**
* 获取结束时间(加上了最后一段时间的持续时间)
*
* @return
*/
public long getLEndTime() {
if (listTs.size() > 0) {
Ts tsM3U8 = listTs.get(listTs.size() - 1);
lEndTime = tsM3U8.getLongDate() + (long) (tsM3U8.getFSeconds() * 1000);
return lEndTime;
}
return 0;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("strBasePath: " + strBasePath);
for (Ts ts : listTs) {
sb.append("\nts_file_name = " + ts);
}
sb.append("\n\nlStartTime = " + lStartTime);
sb.append("\n\nlEndTime = " + lEndTime);
sb.append("\n\nlStartDownloadTime = " + lStartDownloadTime);
sb.append("\n\nlEndDownloadTime = " + lEndDownloadTime);
return sb.toString();
}
public static class Ts implements Comparable<Ts> {
private String strFilePath;
private float fSeconds;
public Ts(String strFilePath, float fSeconds) {
this.strFilePath = strFilePath;
this.fSeconds = fSeconds;
}
public String getStrFilePath() {
return strFilePath;
}
public void setStrFilePath(String strFilePath) {
this.strFilePath = strFilePath;
}
public float getFSeconds() {
return fSeconds;
}
public void setFSeconds(float fSeconds) {
this.fSeconds = fSeconds;
}
@Override
public String toString() {
return strFilePath + " (" + fSeconds + "sec)";
}
/**
* 获取时间
*/
public long getLongDate() {
try {
return Long.parseLong(strFilePath.substring(0, strFilePath.lastIndexOf(".")));
} catch (Exception e) {
return 0;
}
}
@Override
public int compareTo(Ts o) {
return strFilePath.compareTo(o.strFilePath);
}
}
}
\ No newline at end of file
package com.topdraw.sohu.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
public class PropertiesUtil {
private static final String NAME = "app.properties";
private static Properties p;
static {
p = new Properties();
InputStream is = PropertiesUtil.class.getClassLoader()
.getResourceAsStream(NAME);
try {
p.load(is);
} catch (IOException e) {
e.printStackTrace();
}
}
public static String get(String key) {
return p.getProperty(key);
}
}
package com.topdraw.sohu.utils;
import java.sql.Connection;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import org.afflatus.utility.DruidUtil;
public class ResourceUtil {
public static void CreateResourceRecycle(String strFileUrl) {
Connection writeConnection = null;
try {
writeConnection = DruidUtil.getRandomWriteConnection();
Map<String, Object> map = new HashMap<String, Object>();
map.put("path", strFileUrl);
map.put("id", UUID.randomUUID().toString());
DruidUtil.beginTransaction(writeConnection);
DruidUtil.save(writeConnection, map, "cm_resource_recycle");
DruidUtil.commitTransaction(writeConnection);
} catch(Exception ex) {
ex.printStackTrace();
} finally {
DruidUtil.close(writeConnection);
writeConnection = null;
}
}
public static String getImageKeyByType(int iType) {
String strType = "normal";
switch (iType) {
case 0:
strType = "thumbnail";
break;
case 1:
strType = "poster";
break;
case 2:
strType = "stills";
break;
case 3:
strType = "icon";
break;
case 4:
strType = "title";
break;
case 5:
strType = "ad";
break;
case 6:
strType = "sketch";
break;
case 7:
strType = "background";
break;
case 9:
strType = "channel";
break;
case 10:
strType = "channel_bw";
break;
case 11:
strType = "channel_logo";
break;
case 12:
strType = "channel_name";
break;
case 99:
strType = "other";
break;
}
return strType;
}
}
package com.topdraw.sohu.utils;
import com.alibaba.fastjson.JSONObject;
import org.afflatus.utility.DruidUtil;
import org.afflatus.utility.WebUtil;
import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import java.io.*;
import java.sql.Connection;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.*;
public class SohuUtil {
//private final String PROJECT_PATH = this.getClass().getClassLoader().getResource("../../") == null ?
// (System.getProperty("user.dir").endsWith("/") ? System.getProperty("user.dir")
// : System.getProperty("user.dir") + "/")
// : this.getClass().getClassLoader().getResource("../../").getPath();
private final String PROJECT_PATH = "/prepared/";
private final String M3U8_RELATIVE_PATH = "m3u8" + File.separatorChar;
private final String XML_RELATIVE_PATH = "xml" + File.separatorChar;
private final String TS_HLS_RELATIVE_PATH = "ts" + File.separatorChar;
private final String API_KEY = "a1ab29b59ea9f581011f1eb6506c7ada";
private static final String FILM_XML_URL = "http://ott.hd.sohu.com/hd/houyi/film.xml";
private static final String DRAMA_XML_URL = "http://ott.hd.sohu.com/hd/houyi/drama.xml";
private static final String VARIETY_XML_URL = "http://ott.hd.sohu.com/hd/houyi/variety.xml";
private static final String DOCUMENTARY_XML_URL = "http://ott.hd.sohu.com/hd/houyi/documentary.xml";
private static final String CARTOON_XML_URL = "http://ott.hd.sohu.com/hd/houyi/cartoon.xml";
private static final String FEEFILM_XML_URL = "http://ott.hd.sohu.com/hd/houyi/feeFilm.xml";
private static final String FILM_ALL_XML_URL = "http://ott.hd.sohu.com/hd/all/film_all_";
private static final String DRAMA_ALL_XML_URL = "http://ott.hd.sohu.com/hd/all/drama_all_";
private static final String VARIETY_ALL_XML_URL = "http://ott.hd.sohu.com/hd/all/variety_all_";
private static final String DOCUMENTARY_ALL_XML_URL = "http://ott.hd.sohu.com/hd/all/documentary_all_";
private static final String CARTOON_ALL_XML_URL = "http://ott.hd.sohu.com/hd/all/cartoon_all_";
private static final String FEEFILM_ALL_XML_URL = "http://ott.hd.sohu.com/hd/all/fee_film_all_";
private static final String M3U8_URL = "http://h5.ott.tv.sohu.com/api/partner/getM3u8VideoUrls.json";
private Logger log = Logger.getLogger(this.getClass().getName());
private void fullMetaDataInjection(String strUrl) {
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
File file = HttpConnectionUtil.downloadFile(strUrl, PROJECT_PATH, null);
if (file != null) {
SAXReader reader = new SAXReader();
Document document = reader.read(file.getPath());
Element root = document.getRootElement();
int category_id = 0;
int cate_code = 0;
Date updated = new Date();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
for (Iterator<?> it = root.elementIterator(); it.hasNext();) {
DruidUtil.beginTransaction(writeConnection);
Element element = (Element) it.next();
if ("category_id".equals(element.getName())) {
category_id = Integer.parseInt(element.getText());
}
if ("cate_code".equals(element.getName())) {
cate_code = Integer.parseInt(element.getText());
}
if ("updated".equals(element.getName())) {
updated = sdf.parse(element.getText());
}
if ("album".equals(element.getName())) {
Map<String, Object> mapAlbum = new HashMap<>();
for (Iterator<?> itAlbum = element.elementIterator(); itAlbum.hasNext();) {
Element elAlbum = (Element) itAlbum.next();
if (elAlbum.getName().equals("publish_time")) {
mapAlbum.put(elAlbum.getName(), elAlbum.getText().equals("") ? null : sdf.parse(
elAlbum.getText()));
} else if (elAlbum.getName().equals("episode_updated") ||
elAlbum.getName().equals("episode_total") || elAlbum.getName().equals("year") ||
elAlbum.getName().equals("fee") || elAlbum.getName().equals("is_clip") ||
elAlbum.getName().equals("is_show") || elAlbum.getName().equals("is_early")) {
mapAlbum.put(elAlbum.getName(), elAlbum.getText().equals("") ? 0 : Integer.parseInt(
elAlbum.getText()
));
} else if (!elAlbum.getName().equals("videos")) {
mapAlbum.put(elAlbum.getName().equals("update_time") ? "album_update_time" :
elAlbum.getName(), elAlbum.getText().equals("") ? null : elAlbum.getText());
} else {
for (Iterator<?> itVideos = elAlbum.elementIterator(); itVideos.hasNext();) {
Element elVideos = (Element) itVideos.next();
if (elVideos.getName().equals("video")) {
Map<String, Object> mapVideo = new HashMap<>();
for (Iterator<?> itVideo = elVideos.elementIterator(); itVideo.hasNext();) {
Element elVideo = (Element) itVideo.next();
if (elVideo.getName().equals("issue_time")) {
mapVideo.put(elVideo.getName(), elVideo.getText().equals("") ? null :
sdf.parse(elVideo.getText()));
} else if (elVideo.getName().equals("play_order") ||
elVideo.getName().equals("time_length") ||
elVideo.getName().equals("is_show") ||
elVideo.getName().equals("fee") ||
elVideo.getName().equals("tvPlayType")) {
//log.info(elVideo.getName() + " " + mapAlbum.get("name"));
mapVideo.put(elVideo.getName(), elVideo.getText().equals("") ? 0 :
Integer.parseInt(elVideo.getText()));
} else if (elVideo.getName().equals("start_time") ||
elVideo.getName().equals("end_time")) {
} else if (!elVideo.getName().equals("logoInfo")) {
mapVideo.put(elVideo.getName().equals("update_time") ?
"video_update_time" : elVideo.getName(),
elVideo.getText().equals("") ? null : elVideo.getText());
} else {
for (Iterator<?> itLog = elVideo.elementIterator(); itLog.hasNext();) {
Element elLog = (Element) itLog.next();
if (elLog.getName().equals("dimension")) {
mapVideo.put(elLog.getName(), elLog.getText());
} else {
mapVideo.put(elLog.getName(), elLog.getText().equals("") ?
0 : Integer.parseInt(elLog.getText()));
}
}
}
}
mapVideo.put("album_id", mapAlbum.get("album_id"));
String sql = "SELECT id FROM sohu_video WHERE video_id = ?";
Map<String, Object> mapExist = DruidUtil.queryUniqueResult(readConnection,
sql, mapVideo.get("video_id"));
if (mapExist == null) {
DruidUtil.save(writeConnection, mapVideo, "sohu_video");
log.info("---" + mapVideo.toString());
} else {
log.info("Video: " + mapVideo.get("video_id") + " already exists, just update");
mapVideo.put("id", mapExist.get("id"));
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
}
}
}
}
}
mapAlbum.put("category_id", category_id);
mapAlbum.put("cate_code", cate_code);
mapAlbum.put("updated", updated);
String sql = "SELECT id FROM sohu_album WHERE album_id = ?";
Map<String, Object> mapExist = DruidUtil.queryUniqueResult(readConnection, sql,
mapAlbum.get("album_id"));
if (mapExist == null) {
DruidUtil.save(writeConnection, mapAlbum, "sohu_album");
log.info(mapAlbum.toString());
} else {
mapAlbum.put("id", mapExist.get("id"));
DruidUtil.update(writeConnection, mapAlbum, "sohu_album", "id");
log.info("Album: " + mapAlbum.get("album_id") + " already exists, just update");
}
}
DruidUtil.commitTransaction(writeConnection);
}
} else {
log.info("Download xml failed");
}
} catch (Exception ex) {
ex.printStackTrace();
log.error("failure in fullMetaDataInjection", ex);
} finally {
DruidUtil.close(readConnection);
DruidUtil.close(writeConnection);
}
}
public void metaDataIncrementInjection() {
log.info("------ start injecting film metadata ------");
metaDataIncrementInjection(FILM_XML_URL);
log.info("------ film metadata updated ------");
log.info("------ start injecting drama metadata ------");
metaDataIncrementInjection(DRAMA_XML_URL);
log.info("------ drama metadata updated ------");
log.info("------ start injecting variety metadata ------");
metaDataIncrementInjection(VARIETY_XML_URL);
log.info("------ variety metadata updated ------");
log.info("------ start injecting documentary metadata ------");
metaDataIncrementInjection(DOCUMENTARY_XML_URL);
log.info("------ documentary metadata updated ------");
log.info("------ start injecting cartoon metadata ------");
metaDataIncrementInjection(CARTOON_XML_URL);
log.info("------ cartoon metadata updated ------");
log.info("------ start injecting feeFilm metadata ------");
metaDataIncrementInjection(FEEFILM_XML_URL);
log.info("------ feeFilm metadata updated ------");
log.info("****************** 华丽的分割线 ******************");
log.info("------ start self_produced documentary update ------");
fullSelfProducedMetaDataUpdate("http://ott.hd.sohu.com/hd/houyi/sohu_documentary_update.xml");
log.info("------ documentary metadata updated ------");
log.info("------ start self_produced drama update ------");
fullSelfProducedMetaDataUpdate("http://ott.hd.sohu.com/hd/houyi/sohu_drama_update.xml");
log.info("------ drama metadata updated ------");
log.info("------ start self_produced film update ------");
fullSelfProducedMetaDataUpdate("http://ott.hd.sohu.com/hd/houyi/sohu_film_update.xml");
log.info("------ film metadata updated ------");
log.info("------ start self_produced variety update ------");
fullSelfProducedMetaDataUpdate("http://ott.hd.sohu.com/hd/houyi/sohu_variety_update.xml");
log.info("------ variety metadata updated ------");
}
// 搜狐增量接口(自制内容标识)
private void fullSelfProducedMetaDataUpdate(String strUrl) {
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
Date dNow = new Date();
SimpleDateFormat sdf_ymdh = new SimpleDateFormat("yyyyMMddHH");
SimpleDateFormat sdf_ymd = new SimpleDateFormat("yyyyMMdd");
String strDate = sdf_ymdh.format(dNow);
String strDatePath = sdf_ymd.format(dNow) + File.separatorChar;
String strCategory = strUrl.substring(strUrl.lastIndexOf("/") + 1, strUrl.lastIndexOf("."));
File file = HttpConnectionUtil.downloadFile(strUrl, PROJECT_PATH + XML_RELATIVE_PATH + strDatePath +
strCategory + File.separatorChar, strCategory + "_" + strDate + ".xml");
if (file != null) {
SAXReader reader = new SAXReader();
Document document = reader.read(file.getPath());
Element root = document.getRootElement();
long lAlbumId = 0;
for (Iterator<?> it = root.elementIterator(); it.hasNext();) {
DruidUtil.beginTransaction(writeConnection);
Element element = (Element) it.next();
if ("album".equals(element.getName())) {
int i = 0;
for (Iterator<?> itAlbum = element.elementIterator(); itAlbum.hasNext();) {
Element elAlbum = (Element) itAlbum.next();
if (elAlbum.getName().equals("album_id")) {
lAlbumId = Long.parseLong(elAlbum.getText());
break;
}
}
String sql = "SELECT id, self_produced FROM sohu_album WHERE album_id = ?";
Map<String, Object> mapExist = DruidUtil.queryUniqueResult(readConnection, sql,
lAlbumId);
if (mapExist == null) {
log.info("no such album: " + lAlbumId);
} else {
if ((Integer) mapExist.get("self_produced") == 1) {
log.info("unnecessary to update: " + lAlbumId);
} else {
i++;
mapExist.put("self_produced", 1);
DruidUtil.update(writeConnection, mapExist, "sohu_album", "id");
log.info("Album: " + lAlbumId + " self_produced updated");
}
}
if (i == 0) {
log.info("no records updated");
} else {
log.info(i + " records updated");
}
}
DruidUtil.commitTransaction(writeConnection);
}
} else {
log.info("Download xml failed");
}
} catch (Exception ex) {
ex.printStackTrace();
log.error("failure in fullSelfProducedMetaDataUpdate: " + strUrl, ex);
} finally {
DruidUtil.close(readConnection);
DruidUtil.close(writeConnection);
}
}
private void metaDataIncrementInjection(String strURL) {
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
Date dNow = new Date();
strURL = strURL + "?api_key=" + API_KEY + "&t=" + dNow.getTime();
SimpleDateFormat sdf_ymdh = new SimpleDateFormat("yyyyMMddHH");
SimpleDateFormat sdf_ymd = new SimpleDateFormat("yyyyMMdd");
String strDate = sdf_ymdh.format(dNow);
String strDatePath = sdf_ymd.format(dNow) + File.separatorChar;
String strCategory = strURL.substring(strURL.lastIndexOf("/") + 1, strURL.lastIndexOf("."));
File file = HttpConnectionUtil.downloadFile(strURL, PROJECT_PATH + XML_RELATIVE_PATH + strDatePath +
strCategory + File.separatorChar, strCategory + "_" + strDate + ".xml");
if (file != null) {
SAXReader reader = new SAXReader();
Document document = reader.read(file.getPath());
Element root = document.getRootElement();
int category_id = 0;
int cate_code = 0;
Date updated = new Date();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
DruidUtil.beginTransaction(writeConnection);
for (Iterator<?> it = root.elementIterator(); it.hasNext();) {
Element element = (Element) it.next();
if ("category_id".equals(element.getName())) {
category_id = Integer.parseInt(element.getText());
}
if ("cate_code".equals(element.getName())) {
cate_code = Integer.parseInt(element.getText());
}
if ("updated".equals(element.getName())) {
updated = sdf.parse(element.getText());
}
if ("album".equals(element.getName())) {
Map<String, Object> mapAlbum = new HashMap<>();
for (Iterator<?> itAlbum = element.elementIterator(); itAlbum.hasNext();) {
Element elAlbum = (Element) itAlbum.next();
if (elAlbum.getName().equals("publish_time")) {
mapAlbum.put(elAlbum.getName(), elAlbum.getText().equals("") ? null : sdf.parse(
elAlbum.getText()));
} else if (elAlbum.getName().equals("episode_updated") ||
elAlbum.getName().equals("episode_total") || elAlbum.getName().equals("year") ||
elAlbum.getName().equals("fee") || elAlbum.getName().equals("is_clip") ||
elAlbum.getName().equals("is_show") || elAlbum.getName().equals("is_early")) {
mapAlbum.put(elAlbum.getName(), elAlbum.getText().equals("") ? 0 : Integer.parseInt(
elAlbum.getText()
));
} else if (!elAlbum.getName().equals("videos")) {
mapAlbum.put(elAlbum.getName().equals("update_time") ? "album_update_time" :
elAlbum.getName(), elAlbum.getText().equals("") ? null : elAlbum.getText());
} else {
for (Iterator<?> itVideos = elAlbum.elementIterator(); itVideos.hasNext();) {
Element elVideos = (Element) itVideos.next();
if (elVideos.getName().equals("video")) {
Map<String, Object> mapVideo = new HashMap<>();
for (Iterator<?> itVideo = elVideos.elementIterator(); itVideo.hasNext();) {
Element elVideo = (Element) itVideo.next();
if (elVideo.getName().equals("issue_time")) {
mapVideo.put(elVideo.getName(), elVideo.getText().equals("") ? null :
sdf.parse(elVideo.getText()));
} else if (elVideo.getName().equals("play_order") ||
elVideo.getName().equals("time_length") ||
elVideo.getName().equals("is_show") ||
elVideo.getName().equals("fee") ||
elVideo.getName().equals("tvPlayType")) {
//log.info(elVideo.getName() + " " + mapAlbum.get("name"));
mapVideo.put(elVideo.getName(), elVideo.getText().equals("") ? 0 :
Integer.parseInt(elVideo.getText()));
} else if (elVideo.getName().equals("start_time") ||
elVideo.getName().equals("end_time")) {
} else if (!elVideo.getName().equals("logoInfo")) {
mapVideo.put(elVideo.getName().equals("update_time") ?
"video_update_time" : elVideo.getName(),
elVideo.getText().equals("") ? null : elVideo.getText());
} else {
for (Iterator<?> itLog = elVideo.elementIterator(); itLog.hasNext();) {
Element elLog = (Element) itLog.next();
if (elLog.getName().equals("dimension")) {
mapVideo.put(elLog.getName(), elLog.getText());
} else {
mapVideo.put(elLog.getName(), elLog.getText().equals("") ?
0 : Integer.parseInt(elLog.getText()));
}
}
}
}
mapVideo.put("album_id", mapAlbum.get("album_id"));
String sql = "SELECT id, is_show, m3u8_path FROM sohu_video WHERE video_id = ?";
Map<String, Object> mapExist = DruidUtil.queryUniqueResult(readConnection,
sql, mapVideo.get("video_id"));
if (mapExist == null) {
mapVideo.put("xml_path", file.getPath());
try {
DruidUtil.save(writeConnection, mapVideo, "sohu_video");
log.info("video saved: " + mapVideo.get("video_id") + " [" +
mapVideo.get("video_name") + "]");
} catch (SQLException sqle) {
log.error("fail to save video", sqle);
}
} else {
mapVideo.put("id", mapExist.get("id"));
if (mapExist.get("m3u8_path") != null &&
mapExist.get("m3u8_path").toString().equals("500")) {
// 重新触发下载m3u8_path
mapVideo.put("m3u8_path", null);
}
mapVideo.put("xml_path", file.getPath());
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
log.info("video: " + mapVideo.get("video_id") +
" already exists, just update");
if (mapVideo.get("is_show") != mapExist.get("is_show")) {
Map<String, Object> mapOnlineInfo = new HashMap<>();
mapOnlineInfo.put("video_id", mapVideo.get("video_id"));
mapOnlineInfo.put("is_online", mapVideo.get("is_show"));
DruidUtil.save(writeConnection, mapOnlineInfo, "sohu_online_info");
}
}
}
}
}
}
mapAlbum.put("category_id", category_id);
mapAlbum.put("cate_code", cate_code);
mapAlbum.put("updated", updated);
String sql = "SELECT id, episode_updated FROM sohu_album WHERE album_id = ?";
Map<String, Object> mapExist = DruidUtil.queryUniqueResult(readConnection, sql,
mapAlbum.get("album_id"));
if (mapExist == null) {
mapAlbum.put("xml_path", file.getPath());
DruidUtil.save(writeConnection, mapAlbum, "sohu_album");
log.info("album saved: " + mapAlbum.get("album_id") + " [" +
mapAlbum.get("album_name") + "]");
} else {
mapAlbum.put("id", mapExist.get("id"));
if ((Integer) mapAlbum.get("episode_updated") < (Integer) mapExist.get("episode_updated")) {
mapAlbum.put("episode_updated", mapExist.get("episode_updated"));
}
mapAlbum.put("xml_path", file.getPath());
DruidUtil.update(writeConnection, mapAlbum, "sohu_album", "id");
log.info("Album: " + mapAlbum.get("album_id") + " already exists, just update");
if (mapAlbum.get("is_show") != mapExist.get("is_show")) {
Map<String, Object> mapOnlineInfo = new HashMap<>();
mapOnlineInfo.put("album_id", mapAlbum.get("album_id"));
mapOnlineInfo.put("is_online", mapAlbum.get("is_show"));
DruidUtil.save(writeConnection, mapOnlineInfo, "sohu_online_info");
}
}
}
}
DruidUtil.commitTransaction(writeConnection);
} else {
log.info("Download xml failed");
}
} catch (Exception ex) {
ex.printStackTrace();
log.error("failure in metaDataIncrementInjection", ex);
} finally {
DruidUtil.close(readConnection);
DruidUtil.close(writeConnection);
}
}
private String getCategoryByVideo(Map<String, Object> map) {
String str = "";
Connection readConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
String sql = "SELECT cate_code FROM sohu_album WHERE album_id = ?";
Map<String, Object> mapAlbum = DruidUtil.queryUniqueResult(readConnection, sql, map.get("album_id"));
int iCateCode = (Integer) mapAlbum.get("cate_code");
switch (iCateCode) {
case 100:
if ((Integer) map.get("fee") == 0) {
str = "film";
} else {
str = "feeFilm";
}
break;
case 101:
str = "drama";
break;
case 106:
str = "variety";
break;
case 107:
str = "documentary";
break;
case 115:
str = "cartoon";
break;
}
} catch (Exception ex) {
ex.printStackTrace();
} finally {
DruidUtil.close(readConnection);
}
return str;
}
public boolean downloadM3U8ByVideoId(long lVideoId) {
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
String sql = "SELECT id, album_id, video_id, definition, tvPlayType, fee FROM sohu_video WHERE video_id = ?";
Map<String, Object> mapVideo = DruidUtil.queryUniqueResult(readConnection, sql, lVideoId);
String strDefinition = mapVideo.get("definition").toString();
String[] arr = strDefinition.split(",");
String strCategory = getCategoryByVideo(mapVideo) + File.separatorChar;
int iDefinition = 2;
if (!strDefinition.contains(",")) {
iDefinition = Integer.parseInt(strDefinition);
} else {
for (int i = 0; i < arr.length; i++) {
if (i == arr.length - 1) {
iDefinition = Integer.parseInt(arr[arr.length - 1]);
break;
}
if (Integer.parseInt(arr[i]) > Integer.parseInt(arr[i + 1])) {
if (Integer.parseInt(arr[i]) != 2) {
arr[i + 1] = arr[i];
}
}
}
}
if (mapVideo.get("m3u8_path") == null) {
String strParam = "albumId=" + mapVideo.get("album_id") + "&versionId=" + iDefinition +
"&tvVerId=" + lVideoId;
String strResponse = WebUtil.sendGet(M3U8_URL, strParam, "UTF-8");
JSONObject jsonObject = JSONObject.parseObject(strResponse);
if (jsonObject.getIntValue("status") != 200) {
log.info("fail to get m3u8 info from sohu by video_id: " + lVideoId);
mapVideo.put("m3u8_path", jsonObject.getIntValue("status"));
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
return false;
} else {
if (jsonObject.get("data") == null) {
log.info("fail to get m3u8 [200 null] info from sohu by video_id: " + lVideoId);
mapVideo.put("m3u8_path", "200 null");
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
return false;
}
String str = jsonObject.getString("data");
ByteArrayInputStream is = new ByteArrayInputStream(str.getBytes());
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
String strDate = sdf.format(new Date()) + File.separatorChar;
log.info("start downloading m3u8...");
String strPath = PROJECT_PATH + M3U8_RELATIVE_PATH + strDate + strCategory +
lVideoId + "_" + iDefinition + ".m3u8";
File file = new File(strPath);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
OutputStream os = new FileOutputStream(file);
int iBytesRead;
byte[] buffer = new byte[8192];
while ((iBytesRead = is.read(buffer, 0, 8192)) != -1) {
os.write(buffer, 0, iBytesRead);
}
os.close();
is.close();
log.info("downloaded successfully: " + file.getPath());
mapVideo.put("m3u8_path", strPath);
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
}
} else {
log.info("m3u8 already downloaded");
}
} catch (Exception ex) {
ex.printStackTrace();
log.error("fail to downloadM3U8ByVideoId: " + lVideoId, ex);
} finally {
DruidUtil.close(readConnection);
DruidUtil.close(writeConnection);
}
return true;
}
/*
private M3U8 getM3U8ByURL(String strM3U8URL) {
try {
File file = new File(strM3U8URL);
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
String strBasePath = strM3U8URL.substring(0, strM3U8URL.lastIndexOf(File.separatorChar) + 1);
M3U8 m3u8Ret = new M3U8();
m3u8Ret.setStrBasePath(strBasePath);
String strLine;
float fSeconds = 0;
int iIndex;
while ((strLine = reader.readLine()) != null) {
if (strLine.startsWith("#")) {
if (strLine.startsWith("#EXTINF:")) {
strLine = strLine.substring(8);
if ((iIndex = strLine.indexOf(",")) != -1) {
strLine = strLine.substring(0, iIndex);
}
try {
fSeconds = Float.parseFloat(strLine);
} catch (Exception e) {
fSeconds = 0;
}
}
continue;
}
if (strLine.endsWith("m3u8")) {
return getM3U8ByURL(strBasePath + strLine);
}
m3u8Ret.addTs(new M3U8.Ts(strLine, fSeconds));
fSeconds = 0;
}
reader.close();
return m3u8Ret;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
public void downloadTSHLSByVideoId(long lVideoId) {
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
String sql = "SELECT id, m3u8_path FROM sohu_video WHERE video_id = ?";
Map<String, Object> mapVideo = DruidUtil.queryUniqueResult(readConnection, sql, lVideoId);
if (mapVideo == null || mapVideo.get("m3u8_path") == null || mapVideo.get("m3u8_path").toString().equals("")) {
log.info("invalid video_id: " + lVideoId);
return;
}
SohuUtil util = new SohuUtil();
M3U8 m3u8 = util.getM3U8ByURL(mapVideo.get("m3u8_path").toString());
String strBasePath = m3u8.getStrBasePath();
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
String strDate = sdf.format(new Date()) + File.separatorChar;
m3u8.getListTs().parallelStream().forEachOrdered(ts -> {
//log.info(m3u8.getListTs().indexOf(ts) + ".ts");
File file = new File(PROJECT_PATH + TS_HLS_RELATIVE_PATH + strDate + lVideoId + File.separatorChar +
m3u8.getListTs().indexOf(ts) + ".ts");
if (!file.exists()) {// 下载过的就不管了
File fileTs = HttpConnectionUtil.downloadFile(ts.getStrFilePath(),
PROJECT_PATH + TS_HLS_RELATIVE_PATH + strDate + lVideoId + File.separatorChar,
m3u8.getListTs().indexOf(ts) + ".ts");
if (fileTs == null) {
log.info("fail to Download ts: " + ts.getStrFilePath());
} else {
log.info("ts hls downloaded: " + m3u8.getListTs().indexOf(ts) + ".ts");
}
}
});
log.info("Video Downloaded: " + lVideoId);
// mapVideo.put("m3u8_path", strPath);
// DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
} catch (Exception ex) {
ex.printStackTrace();
log.error("fail to downloadTSHLSByVideoId", ex);
} finally {
DruidUtil.close(readConnection);
DruidUtil.close(writeConnection);
}
}
public void downloadM3U8AndTSHLS(long lVideoId) {
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
String sql = "SELECT id, album_id, video_id, definition, m3u8_path FROM sohu_video WHERE video_id = ?";
Map<String, Object> mapVideo = DruidUtil.queryUniqueResult(readConnection, sql, lVideoId);
if (mapVideo == null) {
log.info("invalid video_id: " + lVideoId);
return;
}
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
String strDate = sdf.format(new Date()) + File.separatorChar;
String strDefinition = mapVideo.get("definition").toString();
String[] arr = strDefinition.split(",");
int iDefinition = 2;
if (!strDefinition.contains(",")) {
iDefinition = Integer.parseInt(strDefinition);
} else {
for (int i = 0; i < arr.length; i++) {
if (i == arr.length - 1) {
iDefinition = Integer.parseInt(arr[arr.length - 1]);
break;
}
if (Integer.parseInt(arr[i]) > Integer.parseInt(arr[i + 1])) {
if (Integer.parseInt(arr[i]) != 2) {
arr[i + 1] = arr[i];
}
}
}
}
if (mapVideo.get("m3u8_path") == null) {
String strParam = "albumId=" + mapVideo.get("album_id") + "&versionId=" + iDefinition +
"&tvVerId=" + lVideoId;
String strResponse = WebUtil.sendGet(M3U8_URL, strParam, "UTF-8");
JSONObject jsonObject = JSONObject.parseObject(strResponse);
if (jsonObject.getIntValue("status") != 200) {
log.info("fail to get m3u8 info from sohu by video_id: " + lVideoId);
return;
} else {
String str = jsonObject.getString("data");
ByteArrayInputStream is = new ByteArrayInputStream(str.getBytes());
log.info("start downloading m3u8...");
String strPath = PROJECT_PATH + M3U8_RELATIVE_PATH + strDate + lVideoId + "_" + iDefinition + ".m3u8";
File file = new File(strPath);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
OutputStream os = new FileOutputStream(file);
int iBytesRead;
byte[] buffer = new byte[8192];
while ((iBytesRead = is.read(buffer, 0, 8192)) != -1) {
os.write(buffer, 0, iBytesRead);
}
os.close();
is.close();
log.info("downloaded successfully: " + file.getPath());
mapVideo.put("m3u8_path", strPath);
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
}
} else {
log.info("m3u8 already downloaded, ready to download ts hls...");
}
SohuUtil util = new SohuUtil();
M3U8 m3u8 = util.getM3U8ByURL(mapVideo.get("m3u8_path").toString());
String strBasePath = m3u8.getStrBasePath();
String strTsBaseUrl = PROJECT_PATH + TS_HLS_RELATIVE_PATH + strDate + lVideoId + "_" + iDefinition +
File.separatorChar;
m3u8.getListTs().parallelStream().forEachOrdered(ts -> {
//log.info(m3u8.getListTs().indexOf(ts) + ".ts");
File fileExists = new File(strTsBaseUrl + "ts_" + fillZero(m3u8.getListTs().indexOf(ts), 4) + ".ts");
if (!fileExists.exists()) {// 下载过的就不管了
File fileTs = null;
while (fileTs == null) {
fileTs = HttpConnectionUtil.downloadFile(ts.getStrFilePath(),
strTsBaseUrl, "ts_" + fillZero(m3u8.getListTs().indexOf(ts), 4) + ".ts");
if (fileTs == null) {
log.info("fail to Download ts: " + ts.getStrFilePath() + ", ready to retry");
} else {
log.info("ts hls downloaded: " + fileTs.getPath());
}
}
}
});
log.info("Video Downloaded: " + lVideoId);
mapVideo.put("ts_path", strTsBaseUrl);
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
//log.info("start merging...");
//mergeFiles(new File(mapVideo.get("ts_path").toString()), lVideoId + ".ts");
} catch (Exception ex) {
ex.printStackTrace();
log.error("fail to downloadM3U8TS", ex);
} finally {
DruidUtil.close(readConnection);
DruidUtil.close(writeConnection);
}
}
public boolean mergeFiles(File file, String strResultPath) {
File[] fPaths = file.listFiles();
if (fPaths == null || fPaths.length < 1) {
return false;
}
if (fPaths.length == 1) {
return fPaths[0].renameTo(new File(strResultPath));
}
for (int i = 0; i < fPaths.length; i++) {
if (!fPaths[i].exists() || !fPaths[i].isFile()) {
return false;
}
}
List<File> listFile = Arrays.asList(fPaths);
//Collections.sort(listFile, new Comparator<File>() {
// @Override
// public int compare(File o1, File o2) {
// if (o1.isDirectory() && o2.isFile())
// return -1;
// if (o1.isFile() && o2.isDirectory())
// return 1;
// return Integer.parseInt(o1.getName().split("\\.")[0]) - Integer.parseInt(o2.getName().split("\\.")[0]);
// }
//});
File fileResult = new File(file.getPath() + File.separatorChar + "full_" + strResultPath);
try {
//FileOutputStream fs = new FileOutputStream(fileResult, true);
//FileChannel fileResultChannel = fs.getChannel();
//FileInputStream fis;
//for (int i = 0; i < listFile.size(); i++) {
// fis = new FileInputStream(listFile.get(i));
// FileChannel blk = fis.getChannel();
// fileResultChannel.transferFrom(blk, fileResultChannel.size(), blk.size());
// fis.close();
// blk.close();
// if (i % 20 == 0) {
// log.info(i + " ts merged");
// }
//}
//fs.close();
//fileResultChannel.close();
fileResult.createNewFile();
RandomAccessFile raf = new RandomAccessFile(fileResult, "rw");
raf.setLength(0);
raf.seek(0);
byte[] bytes = new byte[1024];
int len = -1;
for(int i = 0; i < listFile.size(); i++) {
RandomAccessFile out = new RandomAccessFile(listFile.get(i), "rw");
while((len = out.read(bytes)) != -1) {
raf.write(bytes, 0, len);
}
out.close();
if (i != 1 && (i + 1) % 20 == 0) {
log.info((i + 1) + " ts merged");
}
}
raf.close();
log.info("merged successfully: " + fileResult.getPath());
} catch (Exception e) {
e.printStackTrace();
return false;
}
// for (int i = 0; i < fpaths.length; i ++) {
// fpaths[i].delete();
// }
return true;
}
private static String fillZero (Integer i, Integer iTotal) {
String strRet = "";
String str = i.toString();
if (str.length() >= iTotal) {
strRet = str;
} else {
for (int j = 0; j < iTotal - str.length(); j++) {
strRet += "0";
}
strRet += str;
}
return strRet;
}
public void metaDataIncrementInjectionByXml(String strDatePath) {
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, FILM_XML_URL, strDatePath + "09");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, FILM_XML_URL, strDatePath + "21");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, DRAMA_XML_URL, strDatePath + "09");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, DRAMA_XML_URL, strDatePath + "21");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, VARIETY_XML_URL, strDatePath + "09");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, VARIETY_XML_URL, strDatePath + "21");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, DOCUMENTARY_XML_URL, strDatePath + "09");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, DOCUMENTARY_XML_URL, strDatePath + "21");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, CARTOON_XML_URL, strDatePath + "09");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, CARTOON_XML_URL, strDatePath + "21");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, FEEFILM_XML_URL, strDatePath + "09");
metaDataIncrementInjectionByXml(strDatePath + File.separatorChar, FEEFILM_XML_URL, strDatePath + "21");
}
private void metaDataIncrementInjectionByXml(String strDatePath, String strURL, String strDate) {
Connection readConnection = null;
Connection writeConnection = null;
try {
readConnection = DruidUtil.getRandomReadConnection();
writeConnection = DruidUtil.getRandomWriteConnection();
String strCategory = strURL.substring(strURL.lastIndexOf("/") + 1, strURL.lastIndexOf("."));
File file = new File(PROJECT_PATH + XML_RELATIVE_PATH + strDatePath +
strCategory + File.separatorChar, strCategory + "_" + strDate + ".xml");
if (file != null) {
SAXReader reader = new SAXReader();
Document document = reader.read(file.getPath());
Element root = document.getRootElement();
int category_id = 0;
int cate_code = 0;
Date updated = new Date();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
DruidUtil.beginTransaction(writeConnection);
for (Iterator<?> it = root.elementIterator(); it.hasNext();) {
Element element = (Element) it.next();
if ("category_id".equals(element.getName())) {
category_id = Integer.parseInt(element.getText());
}
if ("cate_code".equals(element.getName())) {
cate_code = Integer.parseInt(element.getText());
}
if ("updated".equals(element.getName())) {
updated = sdf.parse(element.getText());
}
if ("album".equals(element.getName())) {
Map<String, Object> mapAlbum = new HashMap<>();
for (Iterator<?> itAlbum = element.elementIterator(); itAlbum.hasNext();) {
Element elAlbum = (Element) itAlbum.next();
if (elAlbum.getName().equals("publish_time")) {
mapAlbum.put(elAlbum.getName(), elAlbum.getText().equals("") ? null : sdf.parse(
elAlbum.getText()));
} else if (elAlbum.getName().equals("episode_updated") ||
elAlbum.getName().equals("episode_total") || elAlbum.getName().equals("year") ||
elAlbum.getName().equals("fee") || elAlbum.getName().equals("is_clip") ||
elAlbum.getName().equals("is_show") || elAlbum.getName().equals("is_early")) {
mapAlbum.put(elAlbum.getName(), elAlbum.getText().equals("") ? 0 : Integer.parseInt(
elAlbum.getText()
));
} else if (!elAlbum.getName().equals("videos")) {
mapAlbum.put(elAlbum.getName().equals("update_time") ? "album_update_time" :
elAlbum.getName(), elAlbum.getText().equals("") ? null : elAlbum.getText());
} else {
for (Iterator<?> itVideos = elAlbum.elementIterator(); itVideos.hasNext();) {
Element elVideos = (Element) itVideos.next();
if (elVideos.getName().equals("video")) {
Map<String, Object> mapVideo = new HashMap<>();
for (Iterator<?> itVideo = elVideos.elementIterator(); itVideo.hasNext();) {
Element elVideo = (Element) itVideo.next();
if (elVideo.getName().equals("issue_time")) {
mapVideo.put(elVideo.getName(), elVideo.getText().equals("") ? null :
sdf.parse(elVideo.getText()));
} else if (elVideo.getName().equals("play_order") ||
elVideo.getName().equals("time_length") ||
elVideo.getName().equals("is_show") ||
elVideo.getName().equals("fee") ||
elVideo.getName().equals("tvPlayType")) {
//log.info(elVideo.getName() + " " + mapAlbum.get("name"));
mapVideo.put(elVideo.getName(), elVideo.getText().equals("") ? 0 :
Integer.parseInt(elVideo.getText()));
} else if (elVideo.getName().equals("start_time") ||
elVideo.getName().equals("end_time")) {
} else if (!elVideo.getName().equals("logoInfo")) {
mapVideo.put(elVideo.getName().equals("update_time") ?
"video_update_time" : elVideo.getName(),
elVideo.getText().equals("") ? null : elVideo.getText());
} else {
for (Iterator<?> itLog = elVideo.elementIterator(); itLog.hasNext();) {
Element elLog = (Element) itLog.next();
if (elLog.getName().equals("dimension")) {
mapVideo.put(elLog.getName(), elLog.getText());
} else {
mapVideo.put(elLog.getName(), elLog.getText().equals("") ?
0 : Integer.parseInt(elLog.getText()));
}
}
}
}
mapVideo.put("album_id", mapAlbum.get("album_id"));
String sql = "SELECT id, is_show, m3u8_path FROM sohu_video WHERE video_id = ?";
Map<String, Object> mapExist = DruidUtil.queryUniqueResult(readConnection,
sql, mapVideo.get("video_id"));
if (mapExist == null) {
mapVideo.put("xml_path", file.getPath());
DruidUtil.save(writeConnection, mapVideo, "sohu_video");
log.info("video saved: " + mapVideo.get("video_id") + " [" +
mapVideo.get("video_name") + "]");
} else {
mapVideo.put("id", mapExist.get("id"));
if (mapExist.get("m3u8_path") != null &&
mapExist.get("m3u8_path").toString().equals("500")) {
// 重新触发下载m3u8_path
mapVideo.put("m3u8_path", null);
}
mapVideo.put("xml_path", file.getPath());
DruidUtil.update(writeConnection, mapVideo, "sohu_video", "id");
log.info("video: " + mapVideo.get("video_id") +
" already exists, just update");
if (mapVideo.get("is_show") != mapExist.get("is_show")) {
Map<String, Object> mapOnlineInfo = new HashMap<>();
mapOnlineInfo.put("video_id", mapVideo.get("video_id"));
mapOnlineInfo.put("is_online", mapVideo.get("is_show"));
DruidUtil.save(writeConnection, mapOnlineInfo, "sohu_online_info");
}
}
}
}
}
}
mapAlbum.put("category_id", category_id);
mapAlbum.put("cate_code", cate_code);
mapAlbum.put("updated", updated);
String sql = "SELECT id, episode_updated FROM sohu_album WHERE album_id = ?";
Map<String, Object> mapExist = DruidUtil.queryUniqueResult(readConnection, sql,
mapAlbum.get("album_id"));
if (mapExist == null) {
mapAlbum.put("xml_path", file.getPath());
DruidUtil.save(writeConnection, mapAlbum, "sohu_album");
log.info("album saved: " + mapAlbum.get("album_id") + " [" +
mapAlbum.get("album_name") + "]");
} else {
mapAlbum.put("id", mapExist.get("id"));
if ((Integer) mapAlbum.get("episode_updated") < (Integer) mapExist.get("episode_updated")) {
mapAlbum.put("episode_updated", mapExist.get("episode_updated"));
}
mapAlbum.put("xml_path", file.getPath());
DruidUtil.update(writeConnection, mapAlbum, "sohu_album", "id");
log.info("Album: " + mapAlbum.get("album_id") + " already exists, just update");
if (mapAlbum.get("is_show") != mapExist.get("is_show")) {
Map<String, Object> mapOnlineInfo = new HashMap<>();
mapOnlineInfo.put("album_id", mapAlbum.get("album_id"));
mapOnlineInfo.put("is_online", mapAlbum.get("is_show"));
DruidUtil.save(writeConnection, mapOnlineInfo, "sohu_online_info");
}
}
}
}
DruidUtil.commitTransaction(writeConnection);
} else {
log.info("Download xml failed");
}
} catch (Exception ex) {
ex.printStackTrace();
log.error("failure in metaDataIncrementInjection", ex);
} finally {
DruidUtil.close(readConnection);
DruidUtil.close(writeConnection);
}
}
*/
private static void fullMetaDataInjection(String strAllPath, int iTotal) {
switch (strAllPath) {
case "film":
strAllPath = FILM_ALL_XML_URL;
break;
case "drama":
strAllPath = DRAMA_ALL_XML_URL;
break;
case "variety":
strAllPath = VARIETY_ALL_XML_URL;
break;
case "documentary":
strAllPath = DOCUMENTARY_ALL_XML_URL;
break;
case "cartoon":
strAllPath = CARTOON_ALL_XML_URL;
break;
case "feeFilm":
strAllPath = FEEFILM_ALL_XML_URL;
break;
default:
System.out.println("uncorrect sohu category");
return;
}
SohuUtil util = new SohuUtil();
for (int i = 1; i <= iTotal; i++) {
String strUrl = strAllPath + i + ".xml";
System.out.println("start injection: " + strUrl);
util.fullMetaDataInjection(strUrl);
}
}
public static void main(String[] args) {
//SohuUtil util = new SohuUtil();
//util.fullSelfProducedMetaDataUpdate("http://ott.hd.sohu.com/hd/houyi/sohu_film.xml");
if (args.length != 2) {
System.out.println("wrong number of arguments");
} else {
String strAllPath = args[0];
String strTotal = args[1];
fullMetaDataInjection(strAllPath, Integer.parseInt(strTotal));
}
//SohuUtil util = new SohuUtil();
//for (int i = 1; i <= 2; i++) {
// String strUrl = "http://ott.hd.sohu.com/hd/all/fee_film_all_" + i + ".xml";
// System.out.println("start injection: " + strUrl);
// util.fullMetaDataInjection(strUrl);
//}
//util.downloadM3U8TS(4372);
//util.downloadTSHLSByVideoId(4372);
//util.downloadM3U8AndTSHLS(3000504);
//util.mergeFiles(new File("E:\\MyWorkSpace\\Eclipse\\hyperion\\hyperion.admin/ts\\20180607\\3000504_21\\"),
// "3000504.ts");
//util.metaDataIncrementInjection(DRAMA_XML_URL);
//Connection readConnection = null;
//try {
// readConnection = DruidUtil.getRandomReadConnection();
// String sql = "SELECT * FROM sohu_video WHERE album_id = ?";
// List<Map<String, Object>> list = DruidUtil.queryList(readConnection, sql, 225);
// File file = new File("225.txt");
// if (!file.exists()) {
// file.createNewFile();
// }
// FileWriter fw = new FileWriter(file.getAbsoluteFile());
// BufferedWriter bw = new BufferedWriter(fw);
// for (Map<String, Object> map : list) {
// bw.write(map.get("m3u8_path") + "\r\n");
// }
// bw.close();
//
//} catch (Exception ex) {
// ex.printStackTrace();
//} finally {
// DruidUtil.close(readConnection);
//}
//util.downloadM3U8ByVideoId(683180);
//util.metaDataIncrementInjection(VARIETY_XML_URL);
}
}
package com.topdraw.sohu.utils;
import org.dom4j.Document;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import org.dom4j.io.OutputFormat;
import org.dom4j.io.XMLWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.UUID;
public class XmlUtils {
private static final Logger logger = LoggerFactory.getLogger(XmlUtils.class);
public static final String FTP_PATH = "/app/proftpd/smp/";
public static final String XML_RELATIVE_PATH = "notify/xml/";
public String generateNotifyXml(int iResult, String strErrorDescripiton) {
String strRet = "";
Date date = new Date();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String strDate = sdf.format(date) + "/";
String strRelativePath = XML_RELATIVE_PATH + strDate + UUID.randomUUID() + ".xml";
String strPath = FTP_PATH + strRelativePath;
File file = new File(strPath);
if (!file.isDirectory()) { // 目录不存在
String[] aPathSegments = strPath.split("/");
String strWalkThroughPath = "/";
for (int i = 0; i < aPathSegments.length - 1; i++) {
strWalkThroughPath = strWalkThroughPath + "/" + aPathSegments[i];
file = new File(strWalkThroughPath);
if (!file.isDirectory()) {
file.mkdir();
}
}
}
Document document = DocumentHelper.createDocument();
Element el = document.addElement("xsi:ADI");
el.addAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
Element elReply = el.addElement("Reply");
Element elPropertyResult = elReply.addElement("Property");
Element elPropertyDescription = elReply.addElement("Property");
elPropertyResult.addAttribute("Name", "Result");
elPropertyDescription.addAttribute("Name", "Description");
elPropertyResult.setText(iResult + "");
elPropertyDescription.setText(strErrorDescripiton);
OutputFormat format = OutputFormat.createPrettyPrint();
format.setEncoding("UTF-8");
format.setNewLineAfterDeclaration(false);
try {
XMLWriter writer = new XMLWriter(new FileOutputStream(strPath), format);
writer.setEscapeText(false);
writer.write(document);
writer.close();
} catch (Exception ex) {
ex.printStackTrace();
}
return PropertiesUtil.get("project.ftp_base_url") + strRelativePath;
}
public static void main(String[] args) {
}
}
Manifest-Version: 1.0
Main-Class: com.topdraw.sohu.utils.SohuUtil
#project
project.name = sohu.port
project.version = v0.0.1
#### DB Configuration #################
db.read_datasource_count=1
db.write_datasource_count=1
#### Global Configuration #############
Global.CrossDomainAllowOrigin=*
Global.CircumstanceLevel=1
\ No newline at end of file
name=druid_read_01
driverClassName = com.mysql.jdbc.Driver
url = jdbc:mysql://139.196.188.223:3306/hyperion?useUnicode=true&characterEncoding=UTF-8&useOldAliasMetadataBehavior=true&useSSL=false&autoReconnect=true
username = druid_read
password = zyRead!
initialSize = 5
maxActive = 20
minIdle = 3
autoReconnect=true
#获取连接时最大等待时间，单位毫秒。配置了maxWait之后，
#缺省启用公平锁，并发效率会有所下降，
#如果需要可以通过配置useUnfairLock属性为true使用非公平锁。
maxWait = 60000
removeAbandoned = true
removeAbandonedTimeout = 3600
#有两个含义：
#1) Destroy线程会检测连接的间隔时间
#2) testWhileIdle的判断依据，详细看testWhileIdle属性的说明
timeBetweenEvictionRunsMillis = 60000
minEvictableIdleTimeMillis = 300000
validationQuery = SELECT 1 FROM DUAL
#空闲时测试
#如果validationQuery为null，testOnBorrow、testOnReturn、testWhileIdle都不会其作用。
testWhileIdle = true
testOnBorrow = false
testOnReturn = false
poolPreparedStatements = false
maxPoolPreparedStatementPerConnectionSize = 50
#属性类型是字符串，通过别名的方式配置扩展插件，
#常用的插件有：监控统计用的filter:stat 日志用的filter:log4j 防御sql注入的filter:wall
filters = stat
#要启用PSCache，必须配置大于0，当大于0时，
#poolPreparedStatements自动触发修改为true。
maxOpenPreparedStatements = -1
#物理连接初始化的时候执行的sql
#connectionInitSqls=
#根据dbType自动识别 当数据库抛出一些不可恢复的异常时，抛弃连接
#exceptionSorter=
name=druid_write_01
driverClassName=com.mysql.jdbc.Driver
url = jdbc:mysql://139.196.188.223:3306/hyperion?useUnicode=true&characterEncoding=UTF-8&useOldAliasMetadataBehavior=true&useSSL=false&autoReconnect=true
username = druid_write
password = zyWrite!
initialSize=5
maxActive=20
minIdle=3
autoReconnect=true
#获取连接时最大等待时间，单位毫秒。配置了maxWait之后，
#缺省启用公平锁，并发效率会有所下降，
#如果需要可以通过配置useUnfairLock属性为true使用非公平锁。
maxWait=60000
removeAbandoned=true
removeAbandonedTimeout=3600
#有两个含义：
#1) Destroy线程会检测连接的间隔时间
#2) testWhileIdle的判断依据，详细看testWhileIdle属性的说明
timeBetweenEvictionRunsMillis=60000
minEvictableIdleTimeMillis=300000
validationQuery=SELECT 1 FROM DUAL
#空闲时测试
#如果validationQuery为null，testOnBorrow、testOnReturn、testWhileIdle都不会其作用。
testWhileIdle = true
testOnBorrow=false
testOnReturn=false
poolPreparedStatements=false
maxPoolPreparedStatementPerConnectionSize=50
#属性类型是字符串，通过别名的方式配置扩展插件，
#常用的插件有：监控统计用的filter:stat 日志用的filter:log4j 防御sql注入的filter:wall
filters=stat
#要启用PSCache，必须配置大于0，当大于0时，
#poolPreparedStatements自动触发修改为true。
maxOpenPreparedStatements = -1
#物理连接初始化的时候执行的sql
#connectionInitSqls=
#根据dbType自动识别 当数据库抛出一些不可恢复的异常时，抛弃连接
#exceptionSorter=
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<!-- ========================== 自定义输出格式说明=================test=============== -->
<!-- %p 输出优先级，即DEBUG，INFO，WARN，ERROR，FATAL -->
<!-- %r 输出自应用启动到输出该log信息耗费的毫秒数 -->
<!-- %c 输出所属的类目，通常就是所在类的全名 -->
<!-- %t 输出产生该日志事件的线程名 -->
<!-- %n 输出一个回车换行符，Windows平台为“/r/n”，Unix平台为“/n” -->
<!-- %d 输出日志时间点的日期或时间，默认格式为ISO8601，也可以在其后指定格式，比如：%d{yyy MMM dd HH:mm:ss,SSS}，输出类似：2002年10月18日
22:10:28,921 -->
<!-- %l 输出日志事件的发生位置，包括类目名、发生的线程，以及在代码中的行数。举例：Testlog4.main(TestLog4.java:10) -->
<!-- ========================================================================== -->
<!-- ========================== 输出方式说明================================ -->
<!-- Log4j提供的appender有以下几种: -->
<!-- org.apache.log4j.ConsoleAppender(控制台), -->
<!-- org.apache.log4j.FileAppender(文件), -->
<!-- org.apache.log4j.DailyRollingFileAppender(每天产生一个日志文件), -->
<!-- org.apache.log4j.RollingFileAppender(文件大小到达指定尺寸的时候产生一个新的文件), -->
<!-- org.apache.log4j.WriterAppender(将日志信息以流格式发送到任意指定的地方) -->
<!-- ========================================================================== -->
<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
<appender name="cAppender" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="[%d{dd HH:mm:ss,SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<!--过滤器设置输出的级别 -->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="debug" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<!-- <appender name="rfAppender" class="org.apache.log4j.RollingFileAppender">
<param name="File" value="./log/sohu.port.log" /> <param name="Append"
value="true" /> <param name="MaxBackupIndex" value="10" /> <layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%p (%c:%L)- %m%n" /> </layout> </appender> -->
<appender name="drfInfoAppender" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${catalina.base}/logs/sohu.port.info" />
<param name="DatePattern" value="'.'yyyy-MM-dd'.log'" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="[%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="info" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<appender name="drfErrorAppender" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${catalina.base}/logs/sohu.port.error" />
<param name="DatePattern" value="'.'yyyy-MM-dd'.log'" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="[%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="warning" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<!-- 指定logger的设置，additivity指示是否遵循缺省的继承机制 <logger name="test.TaskCommandHandler"
additivity="false"> <priority value ="info"/> <appender-ref ref="cAppender"
/> <appender-ref ref="drfAppender" /> </logger> -->
<!-- 根logger的设置 -->
<root>
<priority value="info" />
<appender-ref ref="cAppender" />
<appender-ref ref="drfInfoAppender" />
<!--<appender-ref ref="drfErrorAppender"/> -->
</root>
</log4j:configuration>
\ No newline at end of file
org.quartz.scheduler.instanceName = Sohu Interface Scheduler
org.quartz.scheduler.instanceId = 01
org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
org.quartz.threadPool.threadCount = 10
org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore
org.quartz.plugin.jobInitializer.class = org.quartz.plugins.xml.XMLSchedulingDataProcessorPlugin
org.quartz.plugin.jobInitializer.fileNames = quartz_jobs.xml
org.quartz.plugin.jobInitializer.failOnFileNotFound = true
org.quartz.scheduler.jmx.export = true
\ No newline at end of file
<?xml version='1.0' encoding='utf-8'?>
<job-scheduling-data
xmlns="http://www.quartz-scheduler.org/xml/JobSchedulingData"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.quartz-scheduler.org/xml/JobSchedulingData http://www.quartz-scheduler.org/xml/job_scheduling_data_2_0.xsd"
version="2.0">
<schedule>
<job>
<name>M3U8Job</name>
<group>M3U8JobGroup</group>
<description>M3U8Job</description>
<job-class>com.topdraw.sohu.port.job.M3U8Job</job-class>
</job>
<trigger>
<simple>
<name>M3U8JobTrigger</name>
<group>M3U8JobGroup</group>
<job-name>M3U8Job</job-name>
<job-group>M3U8JobGroup</job-group>
<start-time-seconds-in-future>5</start-time-seconds-in-future>
<repeat-count>-1</repeat-count>
<repeat-interval>30000</repeat-interval>
</simple>
</trigger>
<job>
<name>MetaDataJob</name>
<group>MetaDataJobGroup</group>
<description>MetaDataJob</description>
<job-class>com.topdraw.sohu.port.job.MetaDataJob</job-class>
</job>
<trigger>
<cron>
<name>MetaDataJobTrigger</name>
<group>MetaDataJobGroup</group>
<job-name>MetaDataJob</job-name>
<job-group>MetaDataJobGroup</job-group>
<cron-expression>0 55 * * * ?</cron-expression>
</cron>
</trigger>
</schedule>
</job-scheduling-data>
\ No newline at end of file
<!DOCTYPE web-app PUBLIC
"-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
"http://java.sun.com/dtd/web-app_2_3.dtd" >
<web-app>
<display-name>Archetype Created Web Application</display-name>
</web-app>
<html>
<body>
<h2>Hello World!</h2>
</body>
</html>
Manifest-Version: 1.0
Main-Class: com.topdraw.sohu.utils.SohuUtil
#project
project.name = sohu.port
project.version = v0.0.1
#### DB Configuration #################
db.read_datasource_count=1
db.write_datasource_count=1
#### Global Configuration #############
Global.CrossDomainAllowOrigin=*
Global.CircumstanceLevel=1
\ No newline at end of file
name=druid_read_01
driverClassName = com.mysql.jdbc.Driver
url = jdbc:mysql://139.196.188.223:3306/hyperion?useUnicode=true&characterEncoding=UTF-8&useOldAliasMetadataBehavior=true&useSSL=false&autoReconnect=true
username = druid_read
password = zyRead!
initialSize = 5
maxActive = 20
minIdle = 3
autoReconnect=true
#获取连接时最大等待时间，单位毫秒。配置了maxWait之后，
#缺省启用公平锁，并发效率会有所下降，
#如果需要可以通过配置useUnfairLock属性为true使用非公平锁。
maxWait = 60000
removeAbandoned = true
removeAbandonedTimeout = 3600
#有两个含义：
#1) Destroy线程会检测连接的间隔时间
#2) testWhileIdle的判断依据，详细看testWhileIdle属性的说明
timeBetweenEvictionRunsMillis = 60000
minEvictableIdleTimeMillis = 300000
validationQuery = SELECT 1 FROM DUAL
#空闲时测试
#如果validationQuery为null，testOnBorrow、testOnReturn、testWhileIdle都不会其作用。
testWhileIdle = true
testOnBorrow = false
testOnReturn = false
poolPreparedStatements = false
maxPoolPreparedStatementPerConnectionSize = 50
#属性类型是字符串，通过别名的方式配置扩展插件，
#常用的插件有：监控统计用的filter:stat 日志用的filter:log4j 防御sql注入的filter:wall
filters = stat
#要启用PSCache，必须配置大于0，当大于0时，
#poolPreparedStatements自动触发修改为true。
maxOpenPreparedStatements = -1
#物理连接初始化的时候执行的sql
#connectionInitSqls=
#根据dbType自动识别 当数据库抛出一些不可恢复的异常时，抛弃连接
#exceptionSorter=
name=druid_write_01
driverClassName=com.mysql.jdbc.Driver
url = jdbc:mysql://139.196.188.223:3306/hyperion?useUnicode=true&characterEncoding=UTF-8&useOldAliasMetadataBehavior=true&useSSL=false&autoReconnect=true
username = druid_write
password = zyWrite!
initialSize=5
maxActive=20
minIdle=3
autoReconnect=true
#获取连接时最大等待时间，单位毫秒。配置了maxWait之后，
#缺省启用公平锁，并发效率会有所下降，
#如果需要可以通过配置useUnfairLock属性为true使用非公平锁。
maxWait=60000
removeAbandoned=true
removeAbandonedTimeout=3600
#有两个含义：
#1) Destroy线程会检测连接的间隔时间
#2) testWhileIdle的判断依据，详细看testWhileIdle属性的说明
timeBetweenEvictionRunsMillis=60000
minEvictableIdleTimeMillis=300000
validationQuery=SELECT 1 FROM DUAL
#空闲时测试
#如果validationQuery为null，testOnBorrow、testOnReturn、testWhileIdle都不会其作用。
testWhileIdle = true
testOnBorrow=false
testOnReturn=false
poolPreparedStatements=false
maxPoolPreparedStatementPerConnectionSize=50
#属性类型是字符串，通过别名的方式配置扩展插件，
#常用的插件有：监控统计用的filter:stat 日志用的filter:log4j 防御sql注入的filter:wall
filters=stat
#要启用PSCache，必须配置大于0，当大于0时，
#poolPreparedStatements自动触发修改为true。
maxOpenPreparedStatements = -1
#物理连接初始化的时候执行的sql
#connectionInitSqls=
#根据dbType自动识别 当数据库抛出一些不可恢复的异常时，抛弃连接
#exceptionSorter=
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<!-- ========================== 自定义输出格式说明=================test=============== -->
<!-- %p 输出优先级，即DEBUG，INFO，WARN，ERROR，FATAL -->
<!-- %r 输出自应用启动到输出该log信息耗费的毫秒数 -->
<!-- %c 输出所属的类目，通常就是所在类的全名 -->
<!-- %t 输出产生该日志事件的线程名 -->
<!-- %n 输出一个回车换行符，Windows平台为“/r/n”，Unix平台为“/n” -->
<!-- %d 输出日志时间点的日期或时间，默认格式为ISO8601，也可以在其后指定格式，比如：%d{yyy MMM dd HH:mm:ss,SSS}，输出类似：2002年10月18日
22:10:28,921 -->
<!-- %l 输出日志事件的发生位置，包括类目名、发生的线程，以及在代码中的行数。举例：Testlog4.main(TestLog4.java:10) -->
<!-- ========================================================================== -->
<!-- ========================== 输出方式说明================================ -->
<!-- Log4j提供的appender有以下几种: -->
<!-- org.apache.log4j.ConsoleAppender(控制台), -->
<!-- org.apache.log4j.FileAppender(文件), -->
<!-- org.apache.log4j.DailyRollingFileAppender(每天产生一个日志文件), -->
<!-- org.apache.log4j.RollingFileAppender(文件大小到达指定尺寸的时候产生一个新的文件), -->
<!-- org.apache.log4j.WriterAppender(将日志信息以流格式发送到任意指定的地方) -->
<!-- ========================================================================== -->
<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
<appender name="cAppender" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="[%d{dd HH:mm:ss,SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<!--过滤器设置输出的级别 -->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="debug" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<!-- <appender name="rfAppender" class="org.apache.log4j.RollingFileAppender">
<param name="File" value="./log/sohu.port.log" /> <param name="Append"
value="true" /> <param name="MaxBackupIndex" value="10" /> <layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%p (%c:%L)- %m%n" /> </layout> </appender> -->
<appender name="drfInfoAppender" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${catalina.base}/logs/sohu.port.info" />
<param name="DatePattern" value="'.'yyyy-MM-dd'.log'" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="[%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="info" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<appender name="drfErrorAppender" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${catalina.base}/logs/sohu.port.error" />
<param name="DatePattern" value="'.'yyyy-MM-dd'.log'" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="[%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="warning" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<!-- 指定logger的设置，additivity指示是否遵循缺省的继承机制 <logger name="test.TaskCommandHandler"
additivity="false"> <priority value ="info"/> <appender-ref ref="cAppender"
/> <appender-ref ref="drfAppender" /> </logger> -->
<!-- 根logger的设置 -->
<root>
<priority value="info" />
<appender-ref ref="cAppender" />
<appender-ref ref="drfInfoAppender" />
<!--<appender-ref ref="drfErrorAppender"/> -->
</root>
</log4j:configuration>
\ No newline at end of file
org.quartz.scheduler.instanceName = Sohu Interface Scheduler
org.quartz.scheduler.instanceId = 01
org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
org.quartz.threadPool.threadCount = 10
org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore
org.quartz.plugin.jobInitializer.class = org.quartz.plugins.xml.XMLSchedulingDataProcessorPlugin
org.quartz.plugin.jobInitializer.fileNames = quartz_jobs.xml
org.quartz.plugin.jobInitializer.failOnFileNotFound = true
org.quartz.scheduler.jmx.export = true
\ No newline at end of file
<?xml version='1.0' encoding='utf-8'?>
<job-scheduling-data
xmlns="http://www.quartz-scheduler.org/xml/JobSchedulingData"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.quartz-scheduler.org/xml/JobSchedulingData http://www.quartz-scheduler.org/xml/job_scheduling_data_2_0.xsd"
version="2.0">
<schedule>
<job>
<name>M3U8Job</name>
<group>M3U8JobGroup</group>
<description>M3U8Job</description>
<job-class>com.topdraw.sohu.port.job.M3U8Job</job-class>
</job>
<trigger>
<simple>
<name>M3U8JobTrigger</name>
<group>M3U8JobGroup</group>
<job-name>M3U8Job</job-name>
<job-group>M3U8JobGroup</job-group>
<start-time-seconds-in-future>5</start-time-seconds-in-future>
<repeat-count>-1</repeat-count>
<repeat-interval>30000</repeat-interval>
</simple>
</trigger>
<job>
<name>MetaDataJob</name>
<group>MetaDataJobGroup</group>
<description>MetaDataJob</description>
<job-class>com.topdraw.sohu.port.job.MetaDataJob</job-class>
</job>
<trigger>
<cron>
<name>MetaDataJobTrigger</name>
<group>MetaDataJobGroup</group>
<job-name>MetaDataJob</job-name>
<job-group>MetaDataJobGroup</job-group>
<cron-expression>0 55 * * * ?</cron-expression>
</cron>
</trigger>
</schedule>
</job-scheduling-data>
\ No newline at end of file
No preview for this file type
Manifest-Version: 1.0
Built-By: cheng
Created-By: IntelliJ IDEA
Build-Jdk: 1.8.0_121
Manifest-Version: 1.0
Main-Class: com.topdraw.sohu.utils.SohuUtil
#project
project.name = sohu.port
project.version = v0.0.1
#### DB Configuration #################
db.read_datasource_count=1
db.write_datasource_count=1
#### Global Configuration #############
Global.CrossDomainAllowOrigin=*
Global.CircumstanceLevel=1
\ No newline at end of file
name=druid_read_01
driverClassName = com.mysql.jdbc.Driver
url = jdbc:mysql://139.196.188.223:3306/hyperion?useUnicode=true&characterEncoding=UTF-8&useOldAliasMetadataBehavior=true&useSSL=false&autoReconnect=true
username = druid_read
password = zyRead!
initialSize = 5
maxActive = 20
minIdle = 3
autoReconnect=true
#获取连接时最大等待时间，单位毫秒。配置了maxWait之后，
#缺省启用公平锁，并发效率会有所下降，
#如果需要可以通过配置useUnfairLock属性为true使用非公平锁。
maxWait = 60000
removeAbandoned = true
removeAbandonedTimeout = 3600
#有两个含义：
#1) Destroy线程会检测连接的间隔时间
#2) testWhileIdle的判断依据，详细看testWhileIdle属性的说明
timeBetweenEvictionRunsMillis = 60000
minEvictableIdleTimeMillis = 300000
validationQuery = SELECT 1 FROM DUAL
#空闲时测试
#如果validationQuery为null，testOnBorrow、testOnReturn、testWhileIdle都不会其作用。
testWhileIdle = true
testOnBorrow = false
testOnReturn = false
poolPreparedStatements = false
maxPoolPreparedStatementPerConnectionSize = 50
#属性类型是字符串，通过别名的方式配置扩展插件，
#常用的插件有：监控统计用的filter:stat 日志用的filter:log4j 防御sql注入的filter:wall
filters = stat
#要启用PSCache，必须配置大于0，当大于0时，
#poolPreparedStatements自动触发修改为true。
maxOpenPreparedStatements = -1
#物理连接初始化的时候执行的sql
#connectionInitSqls=
#根据dbType自动识别 当数据库抛出一些不可恢复的异常时，抛弃连接
#exceptionSorter=
name=druid_write_01
driverClassName=com.mysql.jdbc.Driver
url = jdbc:mysql://139.196.188.223:3306/hyperion?useUnicode=true&characterEncoding=UTF-8&useOldAliasMetadataBehavior=true&useSSL=false&autoReconnect=true
username = druid_write
password = zyWrite!
initialSize=5
maxActive=20
minIdle=3
autoReconnect=true
#获取连接时最大等待时间，单位毫秒。配置了maxWait之后，
#缺省启用公平锁，并发效率会有所下降，
#如果需要可以通过配置useUnfairLock属性为true使用非公平锁。
maxWait=60000
removeAbandoned=true
removeAbandonedTimeout=3600
#有两个含义：
#1) Destroy线程会检测连接的间隔时间
#2) testWhileIdle的判断依据，详细看testWhileIdle属性的说明
timeBetweenEvictionRunsMillis=60000
minEvictableIdleTimeMillis=300000
validationQuery=SELECT 1 FROM DUAL
#空闲时测试
#如果validationQuery为null，testOnBorrow、testOnReturn、testWhileIdle都不会其作用。
testWhileIdle = true
testOnBorrow=false
testOnReturn=false
poolPreparedStatements=false
maxPoolPreparedStatementPerConnectionSize=50
#属性类型是字符串，通过别名的方式配置扩展插件，
#常用的插件有：监控统计用的filter:stat 日志用的filter:log4j 防御sql注入的filter:wall
filters=stat
#要启用PSCache，必须配置大于0，当大于0时，
#poolPreparedStatements自动触发修改为true。
maxOpenPreparedStatements = -1
#物理连接初始化的时候执行的sql
#connectionInitSqls=
#根据dbType自动识别 当数据库抛出一些不可恢复的异常时，抛弃连接
#exceptionSorter=
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<!-- ========================== 自定义输出格式说明=================test=============== -->
<!-- %p 输出优先级，即DEBUG，INFO，WARN，ERROR，FATAL -->
<!-- %r 输出自应用启动到输出该log信息耗费的毫秒数 -->
<!-- %c 输出所属的类目，通常就是所在类的全名 -->
<!-- %t 输出产生该日志事件的线程名 -->
<!-- %n 输出一个回车换行符，Windows平台为“/r/n”，Unix平台为“/n” -->
<!-- %d 输出日志时间点的日期或时间，默认格式为ISO8601，也可以在其后指定格式，比如：%d{yyy MMM dd HH:mm:ss,SSS}，输出类似：2002年10月18日
22:10:28,921 -->
<!-- %l 输出日志事件的发生位置，包括类目名、发生的线程，以及在代码中的行数。举例：Testlog4.main(TestLog4.java:10) -->
<!-- ========================================================================== -->
<!-- ========================== 输出方式说明================================ -->
<!-- Log4j提供的appender有以下几种: -->
<!-- org.apache.log4j.ConsoleAppender(控制台), -->
<!-- org.apache.log4j.FileAppender(文件), -->
<!-- org.apache.log4j.DailyRollingFileAppender(每天产生一个日志文件), -->
<!-- org.apache.log4j.RollingFileAppender(文件大小到达指定尺寸的时候产生一个新的文件), -->
<!-- org.apache.log4j.WriterAppender(将日志信息以流格式发送到任意指定的地方) -->
<!-- ========================================================================== -->
<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
<appender name="cAppender" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="[%d{dd HH:mm:ss,SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<!--过滤器设置输出的级别 -->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="debug" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<!-- <appender name="rfAppender" class="org.apache.log4j.RollingFileAppender">
<param name="File" value="./log/sohu.port.log" /> <param name="Append"
value="true" /> <param name="MaxBackupIndex" value="10" /> <layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%p (%c:%L)- %m%n" /> </layout> </appender> -->
<appender name="drfInfoAppender" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${catalina.base}/logs/sohu.port.info" />
<param name="DatePattern" value="'.'yyyy-MM-dd'.log'" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="[%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="info" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<appender name="drfErrorAppender" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${catalina.base}/logs/sohu.port.error" />
<param name="DatePattern" value="'.'yyyy-MM-dd'.log'" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="[%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p] [%t] %c{1}:%m%n" />
</layout>
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="levelMin" value="warning" />
<param name="levelMax" value="error" />
<param name="AcceptOnMatch" value="true" />
</filter>
</appender>
<!-- 指定logger的设置，additivity指示是否遵循缺省的继承机制 <logger name="test.TaskCommandHandler"
additivity="false"> <priority value ="info"/> <appender-ref ref="cAppender"
/> <appender-ref ref="drfAppender" /> </logger> -->
<!-- 根logger的设置 -->
<root>
<priority value="info" />
<appender-ref ref="cAppender" />
<appender-ref ref="drfInfoAppender" />
<!--<appender-ref ref="drfErrorAppender"/> -->
</root>
</log4j:configuration>
\ No newline at end of file
org.quartz.scheduler.instanceName = Sohu Interface Scheduler
org.quartz.scheduler.instanceId = 01
org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
org.quartz.threadPool.threadCount = 10
org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore
org.quartz.plugin.jobInitializer.class = org.quartz.plugins.xml.XMLSchedulingDataProcessorPlugin
org.quartz.plugin.jobInitializer.fileNames = quartz_jobs.xml
org.quartz.plugin.jobInitializer.failOnFileNotFound = true
org.quartz.scheduler.jmx.export = true
\ No newline at end of file
<?xml version='1.0' encoding='utf-8'?>
<job-scheduling-data
xmlns="http://www.quartz-scheduler.org/xml/JobSchedulingData"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.quartz-scheduler.org/xml/JobSchedulingData http://www.quartz-scheduler.org/xml/job_scheduling_data_2_0.xsd"
version="2.0">
<schedule>
<job>
<name>M3U8Job</name>
<group>M3U8JobGroup</group>
<description>M3U8Job</description>
<job-class>com.topdraw.sohu.port.job.M3U8Job</job-class>
</job>
<trigger>
<simple>
<name>M3U8JobTrigger</name>
<group>M3U8JobGroup</group>
<job-name>M3U8Job</job-name>
<job-group>M3U8JobGroup</job-group>
<start-time-seconds-in-future>5</start-time-seconds-in-future>
<repeat-count>-1</repeat-count>
<repeat-interval>30000</repeat-interval>
</simple>
</trigger>
<job>
<name>MetaDataJob</name>
<group>MetaDataJobGroup</group>
<description>MetaDataJob</description>
<job-class>com.topdraw.sohu.port.job.MetaDataJob</job-class>
</job>
<trigger>
<cron>
<name>MetaDataJobTrigger</name>
<group>MetaDataJobGroup</group>
<job-name>MetaDataJob</job-name>
<job-group>MetaDataJobGroup</job-group>
<cron-expression>0 55 * * * ?</cron-expression>
</cron>
</trigger>
</schedule>
</job-scheduling-data>
\ No newline at end of file
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
<!DOCTYPE web-app PUBLIC
"-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
"http://java.sun.com/dtd/web-app_2_3.dtd" >
<web-app>
<display-name>Archetype Created Web Application</display-name>
</web-app>
<html>
<body>
<h2>Hello World!</h2>
</body>
</html>