Skip to content

谢俊文 / Job.slice.push_cdn

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 8db33c07 8db33c07233e6c62bede72c46ab1de4a512301cb by 谢俊文
Options

Initial commit

0 parents
Inline Side-by-side
Showing 16 changed files with 432 additions and 0 deletions
File mode changed
No preview for this file type
No preview for this file type
1 #!/x/app/python3/bin/python3
2 # -*- coding: utf-8 -*-
3
4 PATH={
5 'SLICE_OUTPUT': "/storage/SliceFiles/m3u8/",
6 'CDN_SOURCE': "/storage/m3u8/",
7 'LOG_PATH': "task.log"
8 }
9
10 URL={
11 'GET': "http://172.0.31.15:9085/leviathan.admin/FileTask/getTask",
12 'SLICE_POST': "http://172.0.31.15:9085/leviathan.admin/FileTask/sliceNotify",
13 'PUSH_POST':"http://172.0.31.15:9085/leviathan.admin/FileTask/sourceCdnPublishNotify"
14 }
15
16 SERVERID={
17 'ID':"1"
18 }
1 #!/bin/bash
2
3 #保留文件数
4 ReservedNum=7
5 FileDir="/data/bak /data/bak2"
6 date=$(date "+%Y%m%d-%H%M%S")
7
8
9 for i in $FileDir
10 do
11 FileNum=$(ls -l $i|grep ^d |wc -l)
12 while(( $FileNum > $ReservedNum))
13 do
14 OldFile=$(ls -rt $i| head -1)
15 echo $date "Delete File:"$OldFile
16 rm -rf $i/$OldFile
17 let "FileNum--"
18 done
19
20 done
1 #!/x/app/python3/bin/python3
2 # -*- coding: utf-8 -*-
3 import os
4 import re
5 import config
6 import socket
7 import taskmanage
8 import logging
9
10 logging.basicConfig(level=logging.DEBUG,
11 format='%(asctime)s %(filename)s %(levelname)s %(message)s',
12 datefmt='%a, %d %b %Y %H:%M:%S',
13 filename=config.PATH["LOG_PATH"],
14 filemode='a+')
15
16 class PushTasks:
17 #初始化属性，导入配置
18 def __init__(self):
19 self.__GetUrl=config.URL["GET"]
20 self.__Post_Push_Url=config.URL["PUSH_POST"]
21 self.__outPath = config.PATH["SLICE_OUTPUT"]
22 self.__CDNPath = config.PATH["CDN_SOURCE"]
23 self.__serverID = config.SERVERID["ID"]
24 self.__header = {"Content-Type": "application/json-rpc"}
25
26 #判断奇偶，判断运行主机
27 def job_balance(self,num):
28 int(num)
29 if (num % 2 == self.__serverID):
30 return True
31 else:
32 return False
33
34
35
36 #检查输入文件是否存在
37 def check_file(self,inputfile):
38 if os.path.exists(inputfile):
39 return (True,"")
40 else:
41 return (False,"丢失切片文件:{0}.".format(inputfile))
42
43 #复制迁移
44 def mv_CDN(self,url):
45 base=os.path.basename(url)
46 fileName=os.path.splitext(base)[0]
47 pattern=re.compile(r'\d+-\d+-\d+')
48 directory = pattern.findall(url)
49 pushPath=config.PATH["SLICE_OUTPUT"]+directory[0]+"/"+fileName
50 CDNPath=config.PATH["CDN_SOURCE"]+directory[0]+"/"+fileName
51 try:
52 os.makedirs(CDNPath)
53 except Exception as e:
54 logging.debug(str(e))
55 logging.debug("cp -vrf {0} {1}".format(pushPath,CDNPath))
56 os.system("cp -vrf {0} {1}".format(pushPath,CDNPath))
57
58
59 #主程序
60 def main(address):
61 while 1<2:
62 Job=PushTasks()
63 pushJob=taskmanage.TaskManage()
64 #get方法接任务
65 try:
66 ResponseGet=pushJob.get_request(config.URL["GET"],"?start=0&limit=1&status=500&source_cdn_status=0")
67 except Exception as e:
68 logging.error(str(e))
69 continue
70 resultSet=ResponseGet['resultSet']
71 if resultSet:
72 pass
73 else:
74 continue
75 #获取任务ID，待发布文件目录
76 TaskId=ResponseGet['resultSet'][0]['task_id']
77 url=ResponseGet['resultSet'][0]['url']
78 resultm3u8=Job.check_file(config.PATH["SLICE_OUTPUT"]+os.path.splitext(url)[0])
79 if resultm3u8[0]:
80 pass
81 else:
82 try:
83 pushJob.change_push_status(config.URL["PUSH_POST"],TaskId,-1,address,resultm3u8[1])
84 logging.error(resultm3u8[1])
85 except Exception as e:
86 logging.error(str(e))
87 continue
88 continue
89 #改任务状态为发布中
90 try:
91 pushJob.change_push_status(config.URL["PUSH_POST"],TaskId,10,address,"")
92 except Exception as e:
93 logging.error(str(e))
94 continue
95 Job.mv_CDN(url)
96 #改任务状态为发布完成
97 try:
98 pushJob.change_push_status(config.URL["PUSH_POST"],TaskId,100,address,"")
99 except Exception as e:
100 logging.error(str(e))
101 continue
102
103 #实例化
104 if __name__ == '__main__':
105 address=socket.gethostbyname(socket.getfqdn(socket.gethostname()))
106 try:
107 main(address)
108 except Exception as e:
109 pushJob.change_push_status(config.URL["PUSH_POST"],TaskId,-1,address,str(e))
110 logging.error(str(e))
1 #!/x/app/python3/bin/python3
2 # -*- coding: utf-8 -*-
3 import urllib.request
4 import urllib.parse
5 import urllib.error
6 import json
7
8
9 class TaskManage:
10
11 def __init__(self):
12 self.__header = {"Content-Type": "application/json-rpc"}
13
14
15 #get请求
16 def get_request(self,GetUrl,Param):
17 request = urllib.request.Request(GetUrl+Param)
18 result = urllib.request.urlopen(request).read().decode('utf-8')
19 json=eval(result)
20 return json
21
22 #post请求
23 def post_request(self,PostUrl,data):
24 request = urllib.request.Request(PostUrl,json.dumps(data).encode('utf-8'),self.__header)
25 result = urllib.request.urlopen(request).read().decode('utf-8')
26
27 #改切片状态
28 def change_slice_status(self,PostUrl,TaskId,Status,slice_server,slice_error_desc):
29 data={"task_id" : TaskId, "status": Status, "slice_server": slice_server, "slice_error_desc": slice_error_desc}
30 return self.post_request(PostUrl,data)
31
32 #改发布状态
33 def change_push_status(self,PostUrl,TaskId,Status,slice_server,source_cdn_error_desc):
34 data={"task_id" : TaskId, "source_cdn_status": Status, "source_cdn_server": slice_server,"source_cdn_error_desc":source_cdn_error_desc}
35 return self.post_request(PostUrl,data)
File mode changed
No preview for this file type
No preview for this file type
1 #!/x/app/python3/bin/python3
2 # -*- coding: utf-8 -*-
3
4 PATH={
5 'SLICE_OUTPUT': "/storage/SliceFiles/m3u8/",
6 'SLICE_INPUT': "/storage/smp/download/ts/",
7 'ZIP_PACKAGE': "/storage/SliceFiles/zip/",
8 'LOG_PATH': "task.log"
9 }
10
11 URL={
12 'GET': "http://172.0.31.15:9085/leviathan.admin/FileTask/getTask",
13 'SLICE_POST': "http://172.0.31.15:9085/leviathan.admin/FileTask/sliceNotify",
14 'PUSH_POST':"http://172.0.31.15:9085/leviathan.admin/FileTask/sourceCdnPublishNotify"
15 }
16
17 SERVERID={
18 'ID':"1"
19 }
1 #!/bin/bash
2
3 #保留文件数
4 ReservedNum=7
5 FileDir="/data/bak /data/bak2"
6 date=$(date "+%Y%m%d-%H%M%S")
7
8
9 for i in $FileDir
10 do
11 FileNum=$(ls -l $i|grep ^d |wc -l)
12 while(( $FileNum > $ReservedNum))
13 do
14 OldFile=$(ls -rt $i| head -1)
15 echo $date "Delete File:"$OldFile
16 rm -rf $i/$OldFile
17 let "FileNum--"
18 done
19
20 done
File mode changed
1 #!/x/app/python3/bin/python3
2 # -*- coding: utf-8 -*-
3 import taskmanage
4 import os
5 import re
6 import ffmpy3
7 import socket
8 import config
9 import logging
10 import time
11
12 logging.basicConfig(level=logging.DEBUG,
13 format='%(asctime)s %(filename)s %(levelname)s %(message)s',
14 datefmt='%a, %d %b %Y %H:%M:%S',
15 filename=config.PATH["LOG_PATH"],
16 filemode='a+')
17
18
19 class FfmpegTasks:
20 #初始化属性，导入配置
21 def __init__(self):
22 self.__GetUrl=config.URL["GET"]
23 self.__Post_Slice_Url=config.URL["SLICE_POST"]
24 self.__Post_Push_Url=config.URL["PUSH_POST"]
25 self.__outputPath = config.PATH["SLICE_OUTPUT"]
26 self.__inputPath = config.PATH["SLICE_INPUT"]
27 self.__zipPath = config.PATH["ZIP_PACKAGE"]
28 self.__serverID = int(config.SERVERID["ID"])
29
30 #判断奇偶，判断运行主机
31 def job_balance(self,num):
32 int(num)
33 if (num % 2 == self.__serverID):
34 return True
35 else:
36 return False
37
38 #检查输入文件是否存在
39 def check_file(self,filetype,inputfile):
40 if os.path.isfile(inputfile):
41 return (True,"")
42 else:
43 return (False,"丢失{0}文件:{1}.".format(filetype,inputfile))
44
45
46 #切片
47 def slice_task(self,inputfile):
48 #get方法获取输入文件名，去后缀
49 base=os.path.basename(inputfile)
50 fileName=os.path.splitext(base)[0]
51 #正则匹配日期字符
52 pattern=re.compile(r'\d+-\d+-\d+')
53 directory = pattern.findall(inputfile)
54 #拼接输出目录
55 outputPath=self.__outputPath + directory[0] +'/'+ fileName +'/'
56 #拼接打包目录
57 zipPath=self.__zipPath+directory[0] +'/'
58 #创建输出目录
59 try:
60 os.makedirs(outputPath)
61 except Exception as e:
62 logging.debug(str(e))
63 #创建zip包目录
64 try:
65 os.makedirs(zipPath)
66 except Exception as e:
67 logging.debug(str(e))
68 #拼接输出文件名
69 outputfile=outputPath+fileName+'%03d.ts'
70 #封装ffmpeg 命令
71 ff = ffmpy3.FFmpeg(
72 inputs={self.__inputPath+inputfile: None},
73 outputs={outputfile: '-c copy -map 0 -f segment -segment_list {0} -segment_time 10'.format(outputPath + "playlist.m3u8")})
74 #执行命令
75 logging.info("start slice {0}".format(self.__inputPath+inputfile))
76 ff.run()
77 logging.info("finish slice {0}".format(self.__inputPath+inputfile))
78 #返回输出打包目录，输出目录
79 return (self.__zipPath+directory[0] +'/'+ fileName+".zip",outputPath)
80
81 #打zip包
82 def zip_m3u8(self,Pathzip,outputPath):
83 logging.info("start zip {0}".format(outputPath))
84 os.system("zip -jr {0} {1}".format(Pathzip,outputPath))
85 logging.info("finish zip {0}".format(outputPath))
86
87
88 #主程序
89 def main(address):
90 while 1<2:
91 Job=FfmpegTasks()
92 pushJob=taskmanage.TaskManage()
93 #改200状态： 200=待切片 300=切片中 301=切片失败 500=全部完成
94 #get方法接任务
95 try:
96 ResponseGet=pushJob.get_request(config.URL["GET"],"?start=0&limit=1&status=200")
97 except Exception as e:
98 logging.error(str(e))
99 continue
100 #获取任务ID，待切片文件目录
101 resultSet=ResponseGet['resultSet']
102 if resultSet:
103 pass
104 else:
105 time.sleep(30)
106 continue
107 TaskId=ResponseGet['resultSet'][0]['task_id']
108 inputfile=ResponseGet['resultSet'][0]['url']
109 #检查download文件状态
110 resultDownload=Job.check_file("download",config.PATH["SLICE_INPUT"]+inputfile)
111 if resultDownload[0]:
112 pass
113 else:
114 try:
115 pushJob.change_slice_status(config.URL["SLICE_POST"],TaskId,305,address,resultDownload[1])
116 logging.error(resultDownload[1])
117 except Exception as e:
118 logging.error(str(e),resultDownload[1])
119 break
120 continue
121 #改任务状态为切片中
122 try:
123 pushJob.change_slice_status(config.URL["SLICE_POST"],TaskId,300,address,'')
124 except Exception as e:
125 logging.error(str(e))
126 continue
127 #执行切片
128 try:
129 outPut=Job.slice_task(inputfile)
130 #改任务状态为切片完成
131 #try:
132 # pushJob.change_slice_status(config.URL["SLICE_POST"],TaskId,500,address,'')
133 #except Exception as e:
134 # logging.error(str(e))
135 # break
136 except Exception as e:
137 try:
138 pushJob.change_slice_status(config.URL["SLICE_POST"],TaskId,305,address,'')
139 except Exception as e:
140 logging.error(str(e))
141 continue
142
143 #打包
144 try:
145 Job.zip_m3u8(outPut[0],outPut[1])
146 #改任务状态为待发布
147 try:
148 pushJob.change_slice_status(config.URL["SLICE_POST"],TaskId,500,address,'')
149 except Exception as e:
150 logging.error(str(e))
151 continue
152 except Exception as e:
153 try:
154 pushJob.change_slice_status(config.URL["SLICE_POST"],TaskId,305,address,'')
155 except Exception as e:
156 logging.error(str(e))
157 continue
158
159 #检查zip文件状态
160 resultZip=Job.check_file("zip",outPut[0])
161 if resultZip[0]:
162 pass
163 else:
164 try:
165 pushJob.change_slice_status(config.URL["SLICE_POST"],TaskId,305,address,resultZip[1])
166 logging.error(resultZip[1])
167 except Exception as e:
168 logging.error(str(e))
169 continue
170 continue
171
172 #实例化
173 if __name__ == '__main__':
174 address="172.25.50.2"
175 main(address)
This diff could not be displayed because it is too large.
1 #!/x/app/python3/bin/python3
2 # -*- coding: utf-8 -*-
3 import urllib.request
4 import urllib.parse
5 import urllib.error
6 import json
7
8
9 class TaskManage:
10
11 def __init__(self):
12 self.__header = {"Content-Type": "application/json-rpc"}
13
14
15 #get请求
16 def get_request(self,GetUrl,Param):
17 request = urllib.request.Request(GetUrl+Param)
18 result = urllib.request.urlopen(request).read().decode('utf-8')
19 json=eval(result)
20 return json
21
22 #post请求
23 def post_request(self,PostUrl,data):
24 request = urllib.request.Request(PostUrl,json.dumps(data).encode('utf-8'),self.__header)
25 result = urllib.request.urlopen(request).read().decode('utf-8')
26
27 #改切片状态
28 def change_slice_status(self,PostUrl,TaskId,Status,slice_server,slice_error_desc):
29 data={"task_id" : TaskId, "status": Status, "slice_server": slice_server, "slice_error_desc": slice_error_desc}
30 return self.post_request(PostUrl,data)
31
32 #改发布状态
33 def change_push_status(self,PostUrl,TaskId,Status,slice_server,source_cdn_error_desc):
34 data={"task_id" : TaskId, "source_cdn_status": Status, "source_cdn_server": slice_server,"source_cdn_error_desc":source_cdn_error_desc}
35 return self.post_request(PostUrl,data)