diff --git a/mpegts/atschuff.py b/mpegts/atschuff.py new file mode 100644 index 0000000..0083f98 --- /dev/null +++ b/mpegts/atschuff.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# +# Copyright 2006 John-Mark Gurney. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $Id$ +# + +__all__ = [ 'decode_title', 'decode_description' ] + +try: + from ctypes import * + import ctypes.util + import os.path + + path = os.path.dirname(__file__) + if not path: + path = '.' + hd = ctypes.cdll.LoadLibrary(os.path.join(path, 'libhuffdecode.so.1')) + hd.decodetitle.restype = c_int + hd.decodetitle.argtypes = [ c_char_p, c_char_p, c_int ] + hd.decodedescription.restype = c_int + hd.decodedescription.argtypes = [ c_char_p, c_char_p, c_int ] + + def docall(fun, s): + buflen = 256 + while True: + buf = ctypes.create_string_buffer(buflen) + cnt = fun(s, buf, buflen) + if cnt < buflen: + break + + buflen *= 2 + + return buf.value.decode('iso8859-1') + + decode_title = lambda x: docall(hd.decodetitle, x) + decode_description = lambda x: docall(hd.decodedescription, x) + +except ImportError: + def foo(*args): + raise NotImplementedError, 'Failed to import ctypes' + decode_title = decode_description = foo + +except OSError: + def foo(*args): + raise NotImplementedError, 'Failed to find library huffdecode' + decode_title = decode_description = foo diff --git a/mpegts/mpegts.py b/mpegts/mpegts.py index 5d47e7c..94ec6c2 100755 --- a/mpegts/mpegts.py +++ b/mpegts/mpegts.py @@ -27,10 +27,12 @@ # $Id$ # +import atschuff import itertools import os import sets import struct +import time import traceback TSSYNC = '\x47' @@ -38,7 +40,8 @@ TSPKTLEN = 188 READBLK = 1024 def attribreprlist(obj, attrs): - return map(lambda x, y = obj: '%s: %s' % (x, repr(getattr(y, x))), itertools.ifilter(lambda x, y = obj: hasattr(y, x), attrs)) + return map(lambda x, y = obj: '%s: %s' % (x, repr(getattr(y, x))), + itertools.ifilter(lambda x, y = obj: hasattr(y, x), attrs)) class UnReadSTR: def __init__(self, s): @@ -277,7 +280,8 @@ class PES: self.DTS = read_timestamp(buf[i:i + 5]) i += 5 elif ptsdts_flag == 0x1: - raise ValueError, "ptsdts flag forbidden: %d" % ptsdts_flag + raise ValueError, \ + "ptsdts flag forbidden: %d" % ptsdts_flag if escr_flag: self.ESCR = read_escr(buf[i:i + 6]) i += 6 @@ -297,12 +301,15 @@ class PES: self.additional_copy_info = ord(buf[i]) & 0x7f i += 1 if crc_flag: - self.prev_crc = (ord(buf[i]) << 8) | ord(buf[i + 1]) + self.prev_crc = (ord(buf[i]) << 8) | \ + ord(buf[i + 1]) i += 2 if extension_flag: private_data_flag = bool(ord(buf[i]) & 0x80) - pack_header_field_flag = bool(ord(buf[i]) & 0x40) - program_packet_sequence_counter_flag = bool(ord(buf[i]) & 0x20) + pack_header_field_flag = bool(ord(buf[i]) & \ + 0x40) + program_packet_sequence_counter_flag = \ + bool(ord(buf[i]) & 0x20) pstd_buffer_flag = bool(ord(buf[i]) & 0x10) pes_extension_flag_2 = bool(ord(buf[i]) & 0x01) i += 1 @@ -311,25 +318,34 @@ class PES: i += 16 if pack_header_field_flag: pack_field_length = ord(buf[i]) - self.pack_header = buf[i + 1:i + 1 + pack_field_length] + self.pack_header = buf[i + 1:i + 1 + + pack_field_length] i += 1 + pack_field_length if program_packet_sequence_counter_flag: assert (ord(buf[i]) & 0x80) == 0x80 - self.sequence_counter = ord(buf[i]) & 0x7f + self.sequence_counter = \ + ord(buf[i]) & 0x7f i += 1 assert (ord(buf[i]) & 0x80) == 0x80 - self.mpeg1_mpeg2_ident = bool(ord(buf[i]) & 0x4) - self.original_stuff_len = ord(buf[i]) & 0x3f + self.mpeg1_mpeg2_ident = \ + bool(ord(buf[i]) & 0x4) + self.original_stuff_len = \ + ord(buf[i]) & 0x3f i += 1 if pstd_buffer_flag: assert (ord(buf[i]) & 0xc0) == 0x40 - self.pstd_buffer_scale = bool(ord(buf[i]) & 0x20) - self.pstd_buffer_size = ((ord(buf[i]) & 0x1f) << 8) | ord(buf[i + 1]) + self.pstd_buffer_scale = \ + bool(ord(buf[i]) & 0x20) + self.pstd_buffer_size = \ + ((ord(buf[i]) & 0x1f) << 8) | \ + ord(buf[i + 1]) i += 2 if pes_extension_flag_2: assert (ord(buf[i]) & 0x80) == 0x80 - extension_field_length = ord(buf[i]) & 0x7f - self.extension_field = buf[i + 1: i + 1 + extension_field_length] + extension_field_length = \ + ord(buf[i]) & 0x7f + self.extension_field = buf[i + 1:i + \ + 1 + extension_field_length] i += 1 + extension_field_length assert i <= header_end @@ -337,15 +353,16 @@ class PES: def __repr__(self): # XXX - data length - v = [ 'length', 'scrambling_control', + v = ( 'length', 'scrambling_control', 'priority', 'data_alignment', 'copyright', 'originalcopy', 'PTS', 'DTS', 'ESCR', 'ES_rate', 'trick_mode_control', 'trick_mode_bits', 'additional_copy_info', 'pack_header', 'sequence_counter', 'mpeg1_mpeg2_ident', 'original_stuff_len', 'pstd_buffer_scale', - 'pstd_buffer_size', 'extension_field', ] - return '' % (self.stream_id, ', '.join(attribreprlist(self, v)), ) + 'pstd_buffer_size', 'extension_field', ) + return '' % (self.stream_id, + ', '.join(attribreprlist(self, v)), ) class Pack(list): def __init__(self, f = None, **keyw): @@ -368,8 +385,10 @@ class Pack(list): hlen = (ord(d[4]) << 8) | ord(d[5]) header = f.read(hlen) oh = map(ord, header) - assert (oh[0] & 0x80) == 0x80 and (oh[2] & 0x1) == 0x1 - self.rate_bound = ((oh[0] & 0x7f) << 15) | (oh[1] << 7) | (oh[2] >> 1) + assert (oh[0] & 0x80) == 0x80 and \ + (oh[2] & 0x1) == 0x1 + self.rate_bound = ((oh[0] & 0x7f) << 15) | \ + (oh[1] << 7) | (oh[2] >> 1) self.audio_bound = oh[3] >> 2 self.fixed = bool(oh[3] & 0x2) self.CSPS = bool(oh[3] & 0x1) @@ -377,14 +396,16 @@ class Pack(list): self.system_video_lock = bool(oh[4] & 0x40) assert (oh[4] & 0x20) == 0x20 self.video_bound = oh[4] & 0x1f - self.packet_rate_restriction = bool(oh[5] & 0x80) + self.packet_rate_restriction = \ + bool(oh[5] & 0x80) d = f.peek(1) self.streams = {} while ord(d) & 0x80: d = map(ord, f.read(3)) assert (d[1] & 0xc0) == 0xc0 scaleflag = bool(d[1] & 0x20) - self.streams[d[0]] = (((d[1] & 0x1f) << 8) | d[2]) * (128, 1024)[scaleflag] + self.streams[d[0]] = (((d[1] & 0x1f) << + 8) | d[2]) * (128, 1024)[scaleflag] d = f.peek(1) # PES packets d = f.peek(3) @@ -410,12 +431,16 @@ class Pack(list): 'video_bound', 'packet_rate_restriction', 'streams', ] - return '' % (', '.join(attribreprlist(self, v)), list.__repr__(self)) + return '' % (', '.join(attribreprlist(self, v)), + list.__repr__(self)) def __str__(self): buf = [] buf.append('\x00\x00\x01\xba') - clock = (1l << 46) | (((self.SCR[0] >> 30) & 0x7) << 43) | (1l << 42) | (((self.SCR[0] >> 15) & 0x7ffff) << 27) | (1 << 26) | ((self.SCR[0] & 0x7fff) << 11) | (1 << 10) | ((self.SCR[1] << 1) & 0x3fe) | 0x1 + clock = (1l << 46) | (((self.SCR[0] >> 30) & 0x7) << 43) | \ + (1l << 42) | (((self.SCR[0] >> 15) & 0x7ffff) << 27) | \ + (1 << 26) | ((self.SCR[0] & 0x7fff) << 11) | (1 << 10) | \ + ((self.SCR[1] << 1) & 0x3fe) | 0x1 for i in range(6): buf.append(chr(clock >> ((5 - i) * 8) & 0xff)) muxr = self.muxr / 50 / 8 @@ -451,7 +476,8 @@ class BitRate(int): def max_bitrate_descriptor(b): assert len(b) == 3 - return BitRate((((ord(b[0]) & 0x3f) << 16) | ((ord(b[1]) & 0xff) << 8) | (ord(b[0]) & 0xff)) * 50 * 8) + return BitRate((((ord(b[0]) & 0x3f) << 16) | + ((ord(b[1]) & 0xff) << 8) | (ord(b[0]) & 0xff)) * 50 * 8) class ISO639LangDescriptor(list): atypedict = { @@ -471,11 +497,20 @@ class ISO639LangDescriptor(list): class VStreamDescriptor: def __init__(self, b): + if not b: + self.mpeg2 = None + self.multiple_frame_rate = None + self.frame_rate_code = None + self.constrained_parameter = None + self.still_picture = None + return + fb = ord(b[0]) # XXX - libdvbpsi says no not for mpeg2 flag, but my data # seems to say otherwise. self.mpeg2 = not bool(fb & 0x04) - assert (not self.mpeg2 and len(b) == 1) or (self.mpeg2 and len(b) == 3) + assert (not self.mpeg2 and len(b) == 1) or (self.mpeg2 and + len(b) == 3) self.multiple_frame_rate = bool(fb & 0x80) self.frame_rate_code = frame_rate_code[(fb & 0x78) >> 3] self.constrained_parameter = bool(fb & 0x02) @@ -553,6 +588,10 @@ class AC3Descriptor: else: self.asvcflags = ord(data[i]) i += 1 + if i >= len(data): + self.text = '' + return + textlangcode = ord(data[i]) textlen = textlangcode >> 1 i += 1 @@ -560,7 +599,8 @@ class AC3Descriptor: if textlangcode & 1: self.text = txt.decode('latin-1') else: - assert NotImplementedError, 'the following code is untested' + assert NotImplementedError, \ + 'the following code is untested' self.text = ''.join(map(lambda x: unichr(ord(x[0]) * 256 + ord(x[1])), [txt[i:i+2] for i in range(0, len(txt), 2)])) @@ -573,6 +613,26 @@ class AC3Descriptor: return '' % (', '.join(attribreprlist(self, v)), ) +class ContentAdvisory(list): + def __init__(self, data): + list.__init__(self) + + cnt = ord(data[0]) & 0x3f + i = 1 + for j in xrange(cnt): + region, dim = struct.unpack('>BB', data[i: i + 2]) + i += 2 + d = {} + for j in xrange(dim): + d[ord(data[i])] = ord(data[i + 1]) & 0xf + i += 2 + desclen = ord(data[i]) + desc = MultiStringStruct(data[i + 1: i + 1 + desclen]) + self.append((region, d, desc)) + + def __repr__(self): + return '' % list.__repr__(self) + class ServiceLocationDescriptor(list): tag = 0xa1 sldel = '>BH3c' @@ -599,13 +659,31 @@ class ServiceLocationDescriptor(list): (self.pcr_pid, list.__repr__(self)) class MultiStringStruct(dict): + '''Conforms to Section 6.10 of A/65b.''' + def decode(self, comp, mode, data): - assert (mode == 0 and comp in (1, 2)) or comp == 0 + assert (mode == 0 and comp in (1, 2)) or comp == 0, \ + 'mode: %#x, comp: %#x' % (mode, comp) if comp == 0: + if mode == 0x3f: + return data.decode('UTF-16-BE') + elif mode == 0x3e: + # http://www.unicode.org/reports/tr6/ + raise NotImplementedError, 'Unicode Technical Report #6, A Standard Compression Scheme for Unicode' + + # There are additional limitations + assert mode < 0x34, 'Invalid mode: %#x' % mode + return ''.join(map(lambda x, y = mode * 256: unichr(ord(x) + y), data)) - raise NotImplementedError, 'no code for comp type %d' % comp + assert (comp == 1 or comp == 2) and mode == 0xff, \ + 'Invalid comp: %#x, mode: %#x' % (comp, mode) + + if comp == 1: + return atschuff.decode_title(data) + else: + return atschuff.decode_description(data) def __init__(self, data): cnt = ord(data[0]) @@ -628,13 +706,23 @@ class ComponentNameDescriptor(MultiStringStruct): return '' % \ MultiStringStruct.__repr__(self) +def FindMe(data): + raise RuntimeError, 'found me' + Descriptors = { - 2: VStreamDescriptor, - 10: ISO639LangDescriptor, - 14: max_bitrate_descriptor, - 0x81: AC3Descriptor, - 0xa1: ServiceLocationDescriptor, - 0xa3: ComponentNameDescriptor, + # 0-63 Are listed in ISO 13818-1 Table 2-40 + 2: VStreamDescriptor, # ISO 13818-1 Section 2.6.2 + # 3: Audio, # ISO 13818-1 Section 2.6.3 + 10: ISO639LangDescriptor, # ISO 13818-1 Section 2.6.18 + 14: max_bitrate_descriptor, # ISO 13818-1 Section 2.6.26 + 0x81: AC3Descriptor, # A/53d Section 5.7.3.1 + 0x87: ContentAdvisory, # A/65b Section 6.9.4 + # 0xa0: ExtendedChannelName, # A/65b Section 6.9.5 + 0xa1: ServiceLocationDescriptor, # A/65b Section 6.9.6 + # 0xa2: TimeShiftedService, # A/65b Section 6.9.7 + 0xa3: ComponentNameDescriptor, # A/65b Section 6.9.8 + # 0xad: undefined, # A/53d Section 5.7.3.4 + 0xb6: FindMe, # A/57a Section 7 (ContentId) } PIDs = { @@ -720,10 +808,12 @@ def psip_calc_crc32(data, verbose = False, table = ( if verbose: i_crc = 0xffffffffl for i in data: - i_crc = ((i_crc << 8) & 0xffffffffl) ^ table[(i_crc >> 24) ^ ord(i)] + i_crc = ((i_crc << 8) & 0xffffffffl) ^ table[(i_crc >> + 24) ^ ord(i)] print hex(i_crc) - - i_crc = reduce(lambda x, y: ((x << 8) & 0xffffffffl) ^ table[(x >> 24) ^ ord(y)], data, 0xffffffffl) + else: + i_crc = reduce(lambda x, y: ((x << 8) & 0xffffffffl) ^ + table[(x >> 24) ^ ord(y)], data, 0xffffffffl) return i_crc def psip_crc32(data): @@ -790,7 +880,8 @@ and the key is the table number.''' self._table_id = ord(payload[i]) self.syntax = bool(ord(payload[i + 1]) & 0x80) self.private = bool(ord(payload[i + 1]) & 0x40) - self.sect_len = (((ord(payload[i + 1]) & 0xf) << 8) | ord(payload[i + 2])) + 3 + self.sect_len = (((ord(payload[i + 1]) & 0xf) << 8) | \ + ord(payload[i + 2])) + 3 self.stored_sects = [ payload[i:] ] #print 'bar', i, repr(payload) self.stored_len = len(self.stored_sects[0]) @@ -807,7 +898,8 @@ and the key is the table number.''' i = ord(payload[0]) + 1 self.decode_section_header(payload, i) else: - if self.discontinuity or self.next_continuity(p.continuity): + if self.discontinuity or \ + self.next_continuity(p.continuity): self.discontinuity = True return self.stored_sects.append(p.payload) @@ -823,11 +915,14 @@ and the key is the table number.''' assert len(payload) == self.stored_len if self.syntax: - # XXX I may need to include the skipped part above in - # the crc calculations. + # XXX I may need to include the skipped part + # above in the crc calculations. if not psip_crc32(payload[:self.sect_len]): - raise ValueError, 'CRC check failed: %s' % `payload[:self.sect_len]` - self.extension = (ord(payload[3]) << 8) | ord(payload[4]) + raise ValueError, \ + 'CRC check failed: %s' % \ + `payload[:self.sect_len]` + self.extension = (ord(payload[3]) << 8) | \ + ord(payload[4]) self.version = (ord(payload[5]) & 0x3e) >> 1 self.current_next = bool(ord(payload[5]) & 1) self.section_number = ord(payload[6]) @@ -852,28 +947,59 @@ and the key is the table number.''' pass # No handler, ignore or raise exception? # hmm. I had a packet with some low bits clear - # the spec seems to imply that there can be multiple sections, - # but every case I've seen in the world there isn't. + # the spec seems to imply that there can be multiple + # sections, but every case I've seen in the world + # there isn't. if ord(payload[self.sect_len]) != 0xff: #print 'prev:', self.last_section_number # I should make sure there is enough data - self.decode_section_header(payload, self.sect_len) + self.decode_section_header(payload, + self.sect_len) #print 'starting next section:', repr(self), repr(payload) continue else: break - def __repr__(self): - v = ('table_id', 'syntax', 'private', 'table', + def __repr__(self, v=('table_id', 'syntax', 'private', 'table', 'extension', 'version', 'current_next', 'section_number', - 'last_section_number', 'protocol_version', ) - return '' % (', '.join(attribreprlist(self, v)), super(TSPSIPHandler, self).__repr__()) + 'last_section_number', 'protocol_version', )): + return '' % \ + (', '.join(attribreprlist(self, v)), super(TSPSIPHandler, + self).__repr__()) + +class PSIPObject(object): + def parse_table(self, tbl): + raise NotImplementedError + + def repr_part(self): + return [] + + def __call__(self, psip): + if psip.syntax: + self.version = psip.version + self.current_next = psip.current_next + self.section_number = psip.section_number + self.last_section_number = psip.last_section_number + else: + self.version = None + self.current_next = None + self.section_number = None + self.last_section_number = None + + self.parse_table(psip) -class PAT(dict): + def __repr__(self, v=('version', 'current_next', 'section_number', + 'last_section_number', )): + return '<%s: %s>' % (self.__class__.__name__, + ', '.join(attribreprlist(self, v) + self.repr_part())) + +class PAT(PSIPObject, dict): def __init__(self): '''In order to prevent confusion, you can't init w/ a packet.''' - super(PAT, self).__init__() + PSIPObject.__init__(self) + dict.__init__(self) + self.pid_dict = {} def clean_up(self): @@ -886,17 +1012,18 @@ class PAT(dict): def get_prog(self, pid): return self.pid_dict[pid] - def __call__(self, psip, s = '>HH', sl = struct.calcsize('>HH')): + def parse_table(self, psip, s = '>HH', sl = struct.calcsize('>HH')): assert psip.table_id == 0x00 for i in range(len(psip.table) / sl): - prog, pid = struct.unpack(s, psip.table[i * sl:(i + 1) * sl]) + prog, pid = struct.unpack(s, psip.table[i * sl:(i + + 1) * sl]) pid &= 0x1fff self.pid_dict[pid] = prog self[prog] = pid - def __repr__(self): - return '' % dict.__repr__(self) + def repr_part(self): + return [ dict.__repr__(self) ] def getaudiovideopids(pmt, lang = None): anapid = None @@ -907,8 +1034,10 @@ def getaudiovideopids(pmt, lang = None): j = i[2] if i[0] == 2: vpids.append(cpid) - elif j.has_key(5): - assert 'AC-3' in map(lambda x: x[:4], j[5]) + elif i[0] == 129: + apids.append(cpid) + elif j.has_key(5) and i[0] != 5: + assert 'AC-3' in map(lambda x: x[:4], j[5]), (i, j) if lang is None or lang == j[10][0][0]: apids.append(cpid) else: @@ -926,9 +1055,11 @@ def iteravpids(stream, avpids): if SimpleTSPacket(i).pid in avpids: yield i -class PMT(dict): +class PMT(PSIPObject, dict): def __init__(self): - super(PMT, self).__init__() + PSIPObject.__init__(self) + dict.__init__(self) + self.pcrpid = None self.es = [] @@ -939,7 +1070,7 @@ class PMT(dict): def __nonzero__(self): return len(self) or bool(self.es) - def __call__(self, psip): + def parse_table(self, psip): assert psip.table_id == 0x02 tb = psip.table @@ -959,8 +1090,9 @@ class PMT(dict): i += l es.append((t, p, d)) - def __repr__(self): - return '' % (dict.__repr__(self), repr(self.es)) + def repr_part(self): + return [ 'PCRpid: %d' % self.pcrpid, dict.__repr__(self), + 'ES: %s' % `self.es` ] def channelmajorminorsort(x, y): if x['major'] != y['major']: @@ -968,16 +1100,147 @@ def channelmajorminorsort(x, y): return cmp(x['minor'], y['minor']) -class TVCT(dict): +def gpstoutc(gps, utcoff): + gpstrue = gps - utcoff + return gpstrue + 315990000 + +class STT(PSIPObject): def __init__(self): '''In order to prevent confusion, you can't init w/ a packet.''' - super(TVCT, self).__init__() + PSIPObject.__init__(self) def clean_up(self): - self.clear() + self.utc = None + self.ds_status = None + self.ds_day_of_month = None + self.ds_hour = None + + def parse_table(self, psip): + assert psip.table_id == 0xcd and psip.table[0] == '\x00' + + system_time, gps_utc_offset, daylight_savings = \ + struct.unpack('>IBH', psip.table[1:8]) + ds_status = daylight_savings >> 15 + ds_day_of_month = (daylight_savings >> 8) & 0x1f + ds_hour = daylight_savings & 0xff + utc = gpstoutc(system_time, gps_utc_offset) + self.utc = utc + self.ds_status = ds_status + self.ds_day_of_month = ds_day_of_month + self.ds_hour = ds_hour + + def repr_part(self, v=('ds_status', 'ds_day_of_month', 'ds_hour', )): + return [ `time.ctime(self.utc)`, ] + attribreprlist(self, v) + +class MGT(list): + def __init__(self, pidtable): + '''In order to prevent confusion, you can't init w/ a packet.''' + + super(MGT, self).__init__() + self.pidtable = pidtable + self.watch = {} + + def clean_up(self): + del self[:] def __call__(self, psip): + assert psip.table_id == 0xc7 and psip.table[0] == '\x00' + + ntables = struct.unpack('>H', psip.table[1:3])[0] + i = 3 + for foo in xrange(ntables): + type, pid, version, nbytes, desclen = \ + struct.unpack('>HHBIH', psip.table[i:i + 11]) + i += 11 + pid &= 0x1fff + version &= 0x1f + desclen &= 0xfff + desc = getdescriptors(psip.table[i:i + desclen]) + self.append((type, pid, version, nbytes, desc)) + i += desclen + + # start watch + if type >= 0x100 and type <= 0x17f: + if self.pidtable.has_key(pid): + # XXX - check that it's in watch + pass + else: + self.watch[type] = { 'pid': pid, + 'version': version, } + self.pidtable[pid] = TSPSIPHandler({ + 0xcb: EIT() }) + elif type >= 0x200 and type <= 0x27f: + if self.pidtable.has_key(pid): + # XXX - check that it's in watch + pass + else: + #print 'adding ett', pid + self.watch[type] = { 'pid': pid, + 'version': version, } + self.pidtable[pid] = TSPSIPHandler({ + 0xcc: ETT() }) + + + desclen = struct.unpack('>H', psip.table[i:i + 2])[0] + desclen &= 0xfff + desc = getdescriptors(psip.table[i:i + desclen]) + self.desc = desc + #print `self` + + def __repr__(self): + return '' % (`self.desc`, + list.__repr__(self)) + +class EIT(list): + def __init__(self): + '''In order to prevent confusion, you can't init w/ a packet.''' + + super(EIT, self).__init__() + + def clean_up(self): + del self[:] + + def __call__(self, psip): + assert psip.table_id == 0xcb and psip.table[0] == '\x00' + + ntables = ord(psip.table[1]) + i = 2 + for foo in xrange(ntables): + event_id, start_time, lochilen, lolength, titlelen = \ + struct.unpack('>HIBHB', psip.table[i:i + 10]) + i += 10 + event_id &= 0x3fff + etm_location = (lochilen >> 4) & 0x3 + length = ((lochilen & 0xf) << 16) | lolength + title = MultiStringStruct(psip.table[i:i + titlelen]) + i += titlelen + desclen = struct.unpack('>H', psip.table[i:i + 2])[0] + i += 2 + desclen &= 0xfff + desc = getdescriptors(psip.table[i:i + desclen]) + i += desclen + + # XXX - UTC offset should be what? + self.append((event_id, etm_location, + gpstoutc(start_time, 0), length, title, desc)) + + #print `self` + + def __repr__(self): + return '' % list.__repr__(self) + +class TVCT(PSIPObject, dict): + def __init__(self): + '''In order to prevent confusion, you can't init w/ a packet.''' + + PSIPObject.__init__(self) + dict.__init__(self) + + def clean_up(self): + self.clear() + + def parse_table(self, psip): assert psip.table_id == 0xc8 self['channels'] = [] @@ -986,13 +1249,17 @@ class TVCT(dict): chancnt = ord(tb[i]) i += 1 for foo in range(chancnt): - shrtnm = ''.join(map(lambda x: unichr((ord(x[0]) << 8) | ord(x[1])), [tb[i + x * 2:i + (x + 1) * 2] for x in range(7)])).rstrip(unichr(0)) + shrtnm = ''.join(map(lambda x: unichr((ord(x[0]) << + 8) | ord(x[1])), [tb[i + x * 2:i + (x + 1) * 2] for + x in range(7)])).rstrip(unichr(0)) i += 7 * 2 - major = (((ord(tb[i]) << 8) | ord(tb[i + 1])) >> 2) & 0x3ff + major = (((ord(tb[i]) << 8) | ord(tb[i + 1])) >> 2) & \ + 0x3ff minor = ((ord(tb[i + 1]) & 0x3) << 8) | ord(tb[i + 2]) mode = ord(tb[i + 3]) i += 4 - carrier, tsid, prog_num, flagsa, source, desc_len = struct.unpack('>IHHHHH', tb[i:i + 14]) + carrier, tsid, prog_num, flagsa, source, desc_len = \ + struct.unpack('>IHHHHH', tb[i:i + 14]) i += 14 etm_loc = (flagsa & 0xc000) >> 14 access_control = bool(flagsa & 0x2000) @@ -1014,8 +1281,30 @@ class TVCT(dict): i += 2 self['descriptors'] = getdescriptors(tb[i:i + desc_len]) + def repr_part(self): + return [ dict.__repr__(self), ] + +class ETT(dict): + def __init__(self): + '''In order to prevent confusion, you can't init w/ a packet.''' + + super(ETT, self).__init__() + + def clean_up(self): + pass + + def __call__(self, psip): + assert psip.table_id == 0xcc and psip.table[0] == '\x00' + + id, event = struct.unpack('>HH', psip.table[1:5]) + event >>= 2 + desc = MultiStringStruct(psip.table[5:]) + self[(id, event)] = desc + + #print `self` + def __repr__(self): - return '' % (dict.__repr__(self), ) + return '' % dict.__repr__(self) class TSPESHandler: def __init__(self, cb): @@ -1065,7 +1354,8 @@ class TSPESHandler: self.stored_len = len(self.stored_sects[0]) self.discontinuity = False else: - if self.discontinuity or not self.next_continuity(p.continuity): + if self.discontinuity or \ + not self.next_continuity(p.continuity): self.discontinuity = True return self.stored_sects.append(p.payload) @@ -1076,7 +1366,8 @@ class TSPESHandler: return ps = ''.join(self.stored_sects) - assert self.stored_len == self.pes_len and self.pes_len == len(ps) + assert self.stored_len == self.pes_len and \ + self.pes_len == len(ps) self.cb(ps) def read_clock(buf): @@ -1135,12 +1426,14 @@ class TSPacket: i = 4 adapt_len = ord(p[4]) # XXX - this is a large adapt, is it real? - if adapt >= 2 and adapt_len >= 188: + if (adapt >= 2 and adapt_len >= 188) or self.scramble: return if adapt >= 2: if adapt == 3: - assert adapt_len >= 0 and adapt_len <= 182 + pass + # see below + #assert adapt_len >= 0 and adapt_len <= 182 else: pass # my reading of the spec says this, but in @@ -1148,7 +1441,7 @@ class TSPacket: #assert adapt == 2 and adapt_len == 183 buf = p[i + 1:i + 1 + adapt_len] #print self.error, self.start, self.priority, self.pid, self.scramble, adapt, self.continuity, adapt_len, i - assert len(buf) == adapt_len, 'lengths: %d, %d' % (len(buf), adapt_len) + #assert len(buf) == adapt_len, 'adapt: %d, lengths: %d, %d, buf[0]: %02x' % (adapt, len(buf), adapt_len, ord(buf[0])) try: self.decode_adaptation(buf) except: @@ -1192,8 +1485,8 @@ class TSPacket: i += 2 if ltw_flag: self.ltw_valid = bool(ord(adp[i]) & 0x80) - self.ltw_offset = ((ord(adp[i]) & 0x7f) << 8) |\ - ord(adp[i + 1]) + self.ltw_offset = ((ord(adp[i]) & 0x7f) << + 8) | ord(adp[i + 1]) i += 2 if piecewise_rate_flag: self.piecewise_rate = ((ord(adp[i]) & 0x3f) << @@ -1233,7 +1526,8 @@ data gets modified.''' alenstr = chr(len(self.adaptation)) if self.payload: fb |= 1 << 4 - ret = '%c%c%c%c%s%s%s' % (TSSYNC, sb, tb, fb, alenstr, self.adaptation, self.payload) + ret = '%c%c%c%c%s%s%s' % (TSSYNC, sb, tb, fb, alenstr, + self.adaptation, self.payload) if len(ret) != TSPKTLEN: pass #print >>sys.stderr, repr(self) @@ -1251,8 +1545,10 @@ class TSPStream: def __iter__(self): foundsync = False buf = self.f.read(READBLK) + fppos = self.f.tell() while buf: - if self.endpos is not None and self.f.tell() > self.endpos: + if self.endpos is not None and \ + fppos > self.endpos: break if not foundsync: @@ -1260,6 +1556,7 @@ class TSPStream: start = buf.index(TSSYNC) except ValueError: buf = self.f.read(READBLK) + fppos = self.f.tell() continue try: @@ -1273,6 +1570,7 @@ class TSPStream: continue except IndexError: nbuf = self.f.read(READBLK) + fppos = self.f.tell() if not nbuf: return buf += nbuf @@ -1287,6 +1585,7 @@ class TSPStream: t = buf[start:start + TSPKTLEN] if len(t) != TSPKTLEN: r = self.f.read(READBLK) + fppos = self.f.tell() if not r: #No more data break @@ -1294,13 +1593,16 @@ class TSPStream: start = 0 if not buf: buf = self.f.read(READBLK) + fppos = self.f.tell() continue + self.itempos = fppos - len(buf) yield t buf = buf[start + TSPKTLEN:] start = 0 if not buf: buf = self.f.read(READBLK) + fppos = self.f.tell() import getopt import re @@ -1308,12 +1610,23 @@ import sys def usage(): print 'Usage: %s -lmty ' % sys.argv[0] + print ' %s -k ' % sys.argv[0] print ' %s -b [ -p ] ' % sys.argv[0] print ' %s -c -o ' % sys.argv[0] print '' print ' -l list channels' print ' -m print PAT and PMT' print ' -t print TVCT' + print '' + print ' -b bandwidth' + print ' -p sort by percentage' + print '' + print ' -c channel to capture' + print ' -o file to output channel' + print '' + print ' -k print PCR of pid stream' + print '' + print 'Options for all:' print ' -y file offset when done' print ' -s Starting pos' print ' -e Ending pos' @@ -1338,7 +1651,6 @@ def GetTVCT(tsstream): needtvct = True needpat = False - needpmts = {} pat = PAT() pmts = {} @@ -1390,7 +1702,9 @@ def GetTVCT(tsstream): # unable to find TVCT lst = pat.items() lst.sort() - lst = map(lambda x, y: { 'name': 'PAT%d' % x[1], 'prog_num': x[1], 'major': '?', 'minor': y}, lst, range(1, len(pat) + 1)) + lst = map(lambda x, y: { 'name': 'PAT%d' % x[1], + 'prog_num': x[1], 'major': '?', 'minor': y}, lst, + range(1, len(pat) + 1)) tvct = { 'channels': lst } for i in lst: @@ -1402,7 +1716,7 @@ def GetTVCT(tsstream): def main(): try: - opts, args = getopt.getopt(sys.argv[1:], "bc:e:hlmo:ps:ty") + opts, args = getopt.getopt(sys.argv[1:], "bc:e:hk:lmo:pr:s:ty") except getopt.GetoptError: # print help information and exit: usage() @@ -1413,17 +1727,20 @@ def main(): printbandwidthpercent = False listchan = False channelsel = None + programsel = None output = None allmaps = False printtvct = False startpos = None endpos = None + pcrpid = None + needtvct = False needpat = False - needpmts = {} channelfnd = False needallmaps = False + cuts = False for o, a in opts: if o == '-b': @@ -1433,7 +1750,8 @@ def main(): channelsel = int(a) except ValueError: try: - channelsel = tuple(map(int, re.split('[-.]', a, 1))) + channelsel = tuple(map(int, + re.split('[-.]', a, 1))) except ValueError: channelsel = a if channelsel is not None: @@ -1441,6 +1759,8 @@ def main(): needtvct = True elif o == '-e': endpos = int(a) + elif o == '-k': + pcrpid = int(a) elif o == "-m": allmaps = True needallmaps = True @@ -1461,6 +1781,9 @@ def main(): output = a elif o == '-p': printbandwidthpercent = True + elif o == '-r': + programsel = int(a) + needpat = True elif o == '-y': printbyteoffset = True @@ -1475,6 +1798,7 @@ def main(): pat = PAT() pmts = {} tvct = TVCT() + stt = STT() def null(p): #print 'null', repr(p) @@ -1482,13 +1806,16 @@ def main(): null.clean_up = lambda: None psippids = { 0x00: TSPSIPHandler({ 0x00: pat }), - 0x1ffb: TSPSIPHandler({ - #0xc7: null, - 0xc8: tvct, - }), } pidcnt = {} + mgt = MGT(psippids) + psippids[0x1ffb] = TSPSIPHandler({ + #0xc7: mgt, + 0xc8: tvct, + 0xcd: stt, + }) + def getpmt(pid, pm = pmts, psp = psippids): if not pm.has_key(pid): pm[pid] = PMT() @@ -1501,6 +1828,9 @@ def main(): return False + lastpcr = None + lastpatpos = None + for i in itertools.imap(TSPacket, s): #if hasattr(i, 'splice_countdown') or hasattr(i, 'DTS_next_AU'): # print 'splice_countdown:', repr(i) @@ -1508,6 +1838,17 @@ def main(): # print 'PCR:', repr(i) #if i.pid in (48, 64, 80, 112): # print `i` + # print s.itempos, `i` + + if i.pid == 0 or lastpatpos is None: + lastpatpos = s.itempos + + if pcrpid is not None and i.pid == pcrpid: + if lastpcr is not None: + # I've only seen 2703 as the largest + if i.PCR[0] - lastpcr[0] > 3000: + print lastpatpos + lastpcr = i.PCR try: psippids[i.pid](i) @@ -1530,18 +1871,27 @@ def main(): if needtvct and tvct: # Handle TVCT needtvct = False - if channelsel is not None: - channelfnd = findchannel(tvct, channelsel) - if channelfnd is None: - sys.stderr.write("Unable to find channel: %s\n" % channelsel) - channelsel = None - else: - channelfnd = pat[channelfnd['prog_num']] - getpmt(channelfnd) + + if programsel is not None and pat: + channelfnd = pat[programsel] + getpmt(channelfnd) + + if channelsel is not None and pat and tvct: + channelfnd = findchannel(tvct, channelsel) + if channelfnd is None: + sys.stderr.write("Unable to find channel: %s\n" + % channelsel) + channelsel = None + else: + channelfnd = pat[channelfnd['prog_num']] + getpmt(channelfnd) if needpat and pat: needpat = False - for j in pat.itervalues(): + for pn, j in pat.iteritems(): + if pn == 0: + # XXX - NIT + continue if listchan or allmaps: getpmt(j) @@ -1551,14 +1901,17 @@ def main(): break needallmaps = False - #print repr(tvct), repr(pat), repr(pmts) + #print `tvct`, `pat`, `pmts` #print needtvct, needpat, needpmts(), printbandwidth, needallmaps - if not (needtvct or needpat or needpmts() or printbandwidth or needallmaps): + if not (needtvct or needpat or needpmts() or printbandwidth or + needallmaps or pcrpid): break if channelfnd and pmts[channelfnd]: av = getaudiovideopids(pmts[channelfnd]) - os.system("python tssel.py %s %s %s > '%s'" % (args[0], channelfnd, ' '.join(map(str, itertools.chain(*av))), output)) + os.system("python tssel.py '%s' %s %s > '%s'" % (args[0], + channelfnd, ' '.join(map(str, itertools.chain(*av))), + output)) if allmaps: print repr(pat) @@ -1579,13 +1932,15 @@ def main(): # lst.sort() # lst = map(lambda x, y: { 'prog_num': x[1], 'major': '?', 'minor': y}, lst, range(1, len(pat) + 1)) for i in lst: - if i['prog_num'] != 0: + if i['prog_num'] != 0 and i['prog_num'] != 0xffff: #print repr(pmts[pat[i['prog_num']]]) av = getaudiovideopids(pmts[pat[i['prog_num']]]) - prog_info = '\t'.join(map(lambda x: ','.join(map(str, x)), av)) + prog_info = '\t'.join(map(lambda x: + ','.join(map(str, x)), av)) else: prog_info = '' - print ('%(major)d.%(minor)d\t%(name)s\t' % i) + prog_info + print ('%(major)d.%(minor)d\t%(name)s\t' % i) + \ + prog_info if printbandwidth: totpkts = sum(pidcnt.itervalues()) @@ -1599,7 +1954,8 @@ def main(): else: i.sort() for pid, cnt in i: - print '%4d\t%d\t%5.2f' % (pid, cnt, float(cnt) * 100 / totpkts) + print '%4d\t%d\t%5.2f' % (pid, cnt, + float(cnt) * 100 / totpkts) if printbyteoffset: print inp.tell() @@ -1617,7 +1973,8 @@ def justprint(v, p): for i in filter(lambda x: x[1] == '\x00', fc): print `pes.data[i[0] + 3: i[0] + 7]` if ((ord(pes.data[i[0] + 5]) & 0x38) >> 3) in (2, 3): - print 'non I frame found: %d' % ((ord(pes.data[i[0] + 5]) & 0x38) >> 3) + print 'non I frame found: %d' % \ + ((ord(pes.data[i[0] + 5]) & 0x38) >> 3) if __name__ == '__main__': if True: @@ -1680,7 +2037,8 @@ if __name__ == '__main__': if pesstreams.has_key(l[1]): continue print repr(l) - pesstreams[l[1]] = TSPESHandler(lambda x, y = l[1]: justprint(y, x)) + pesstreams[l[1]] = TSPESHandler(lambda x, y = + l[1]: justprint(y, x)) pidhandlers[l[1]] = pesstreams[l[1]] try: diff --git a/mpegts/tssel.py b/mpegts/tssel.py index 6f8e306..ab3ffff 100755 --- a/mpegts/tssel.py +++ b/mpegts/tssel.py @@ -32,9 +32,9 @@ sys.path.append('/Users/jgurney/p4/bktrau/info') import itertools import mpegts -import sys import sets import struct +import sys def usage(): print >>sys.stderr, 'Usage: %s ...' % sys.argv[0] @@ -75,12 +75,20 @@ def producets(inp, pmtpid, *pids): frst = ord(i[1]) # Get first and error bits for testing. pid = (frst & 0x1f) << 8 | ord(i[2]) - if pid == 0 and didpmt: + if frst & 0x80: + continue + elif pid == 0 and didpmt: yield pats.next() - elif pid == pmtpid: + elif pid == pmtpid and frst & 0x40: if not didpmt: - assert i[4] == '\x00' and i[5] == '\x02' - pats = itertools.cycle(genpats(pmtpid, struct.unpack('>H', i[8:10])[0])) + startpmt = 4 + if ((ord(i[3]) >> 4) & 0x3) == 0x3: + # Has adaptation field + startpmt += ord(i[startpmt]) + 1 + + startpmt += ord(i[startpmt]) + 1 + assert i[startpmt] == '\x02', (startpmt, i[0:10]) + pats = itertools.cycle(genpats(pmtpid, struct.unpack('>H', i[startpmt + 3:startpmt + 5])[0])) yield pats.next() didpmt = True # XXX - we probably want to rewrite the PMT to only