diff --git a/python/fdt/__init__.py b/python/fdt/__init__.py deleted file mode 100644 index 539bcdc..0000000 --- a/python/fdt/__init__.py +++ /dev/null @@ -1,682 +0,0 @@ -# Copyright 2017 Martin Olejar -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from .header import Header, DTB_BEGIN_NODE, DTB_END_NODE, DTB_PROP, DTB_END, DTB_NOP -from .items import new_property, Property, PropBytes, PropWords, PropStrings, PropIncBin, Node -from .misc import strip_comments, split_to_lines, get_version_info, extract_string - -__author__ = "Martin Olejar" -__contact__ = "martin.olejar@gmail.com" -__version__ = "0.3.3" -__license__ = "Apache 2.0" -__status__ = "Development" -__all__ = [ - # FDT Classes - 'FDT', - 'Node', - 'Header', - # properties - 'Property', - 'PropBytes', - 'PropWords', - 'PropStrings', - 'PropIncBin', - # core methods - 'parse_dts', - 'parse_dtb', - 'diff' -] - - -class ItemType: - NODE = 0 - PROP = 1 - # Specific property type - PROP_BASE = 5 - PROP_WORDS = 6 - PROP_BYTES = 7 - PROP_STRINGS = 8 - # All types - ALL = 100 - - -class FDT: - """ Flattened Device Tree Class """ - - @property - def empty(self): - return self.root.empty - - def __init__(self, header=None, entries=[]): - """ - FDT class constructor - - :param header: - """ - self.entries = entries - self.header = Header() if header is None else header - self.root = Node('/') - self.last_handle = 0 - self.label_to_handle = {} - self.handle_to_label = {} - - - def __str__(self): - """ String representation """ - return self.info() - - def info(self, props = False): - """ Return object info in human readable format """ - msg = "FDT Content:\n" - for path, nodes, props in self.walk(): - txt = "{} ({},{})".format(path, len(nodes), len(props)) - if props: - txt += ' : ' + ", ".join(x.name for x in props) - msg += txt + "\n" - return msg - - def get_node(self, path: str, create: bool = False) -> Node: - """ - Get node object from specified path - - :param path: Path as string - :param create: If True, not existing nodes will be created - """ - assert isinstance(path, str), "Node path must be a string type !" - - node = self.root - path = path.lstrip('/') - if path: - names = path.split('/') - for name in names: - item = node.get_subnode(name) - if item is None: - if create: - item = Node(name) - node.append(item) - else: - raise ValueError("Path \"{}\" doesn't exists".format(path)) - node = item - - return node - - def get_property(self, name: str, path: str = '') -> Property: - """ - Get property object by name from specified path - - :param name: Property name - :param path: Path to sub-node - """ - return self.get_node(path).get_property(name) - - def set_property(self, name: str, value, path: str = '', create: bool = True): - """ - Set property object by name - - :param name: Property name - :param value: Property value - :param path: Path to subnode - :param create: If True, not existing nodes will be created - """ - self.get_node(path, create).set_property(name, value) - - def exist_node(self, path: str) -> bool: - """ - Check if /node exist and return True - - :param path: path/node name - :return True if /node exist else False - """ - try: - self.get_node(path) - except ValueError: - return False - else: - return True - - def exist_property(self, name: str, path: str = '') -> bool: - """ - Check if property exist - - :param name: Property name - :param path: The path - """ - return self.get_node(path).exist_property(name) if self.exist_node(path) else False - - def remove_node(self, name: str, path: str = ''): - """ - Remove node obj by path/name. Raises ValueError if path/name doesn't exist - - :param name: Node name - :param path: Path to sub-node - """ - self.get_node(path).remove_subnode(name) - - def remove_property(self, name: str, path: str = ''): - """ - Remove property obj by name. Raises ValueError if path/name doesn't exist - - :param name: Property name - :param path: Path to subnode - """ - self.get_node(path).remove_property(name) - - def add_item(self, obj, path: str = '', create: bool = True): - """ - Add sub-node or property at specified path. Raises ValueError if path doesn't exist - - :param obj: The node or property object - :param path: The path to subnode - :param create: If True, not existing nodes will be created - """ - self.get_node(path, create).append(obj) - - def add_label(self, label): - ''' track labels/references to convert to phandles - adds label with incrmenting handle to dictionary if not alread present - returns handle for which can be used to replace the reference''' - if label in self.label_to_handle: - return self.label_to_handle[label] - self.last_handle += 1 - self.label_to_handle[label] = self.last_handle - self.handle_to_label[self.last_handle] = label - return self.last_handle - - def search(self, name: str, itype: int = ItemType.ALL, path: str = '', recursive: bool = True) -> list: - """ - Search properties and/or nodes with specified name. Return list of founded items - - :param name: The Property or Node name. If empty "", all nodes or properties will selected - :param itype: Item type - NODE, PROP, PROP_BASE, PROP_WORDS, PROP_BYTES, PROP_STRINGS or ALL - :param path: Path to root node - :param recursive: Search in all sub-nodes (default: True) - """ - assert isinstance(name, str), "Property name must be a string type !" - - node = self.get_node(path) - nodes = [] - items = [] - pclss = { - ItemType.PROP_BASE: Property, - ItemType.PROP_BYTES: PropBytes, - ItemType.PROP_WORDS: PropWords, - ItemType.PROP_STRINGS: PropStrings - } - while True: - nodes += node.nodes - if itype == ItemType.NODE or itype == ItemType.ALL: - if not name or node.name == name: - items.append(node) - if itype != ItemType.NODE or itype == ItemType.ALL: - for p in node.props: - if name and p.name != name: - continue - if itype in pclss and type(p) is not pclss[itype]: - continue - items.append(p) - if not recursive or not nodes: - break - node = nodes.pop() - - return items - - def walk(self, path: str = '', relative: bool = False) -> list: - """ - Walk trough nodes and return relative/absolute path with list of sub-nodes and properties - - :param path: The path to root node - :param relative: True for relative or False for absolute return path - """ - all_nodes = [] - - node = self.get_node(path) - while True: - all_nodes += node.nodes - current_path = node.path - current_path = current_path.replace('///', '/') - current_path = current_path.replace('//', '/') - if path and relative: - current_path = current_path.replace(path, '').lstrip('/') - yield current_path, node.nodes, node.props - if not all_nodes: - break - node = all_nodes.pop() - - def merge(self, fdt_obj, replace: bool = True): - """ - Merge external FDT object into this object. - - :param fdt_obj: The FDT object which will be merged into this - :param replace: True for replace existing items or False for keep old items - """ - assert isinstance(fdt_obj, FDT) - if self.header.version is None: - self.header = fdt_obj.header - else: - if fdt_obj.header.version is not None and \ - fdt_obj.header.version > self.header.version: - self.header.version = fdt_obj.header.version - if fdt_obj.entries: - for in_entry in fdt_obj.entries: - exist = False - for index in range(len(self.entries)): - if self.entries[index]['address'] == in_entry['address']: - self.entries[index]['size'] = in_entry['size'] - exist = True - break - if not exist: - self.entries.append(in_entry) - - self.root.merge(fdt_obj.get_node('/'), replace) - - def update_phandles(self): - all_nodes = [] - no_phandle_nodes = [] - - node = self.root - all_nodes += self.root.nodes - while all_nodes: - props = (node.get_property('phandle'), node.get_property('linux,phandle')) - value = None - for i, p in enumerate(props): - if isinstance(p, PropWords) and isinstance(p.value, int): - value = None if i == 1 and p.value != value else p.value - if value is None: - no_phandle_nodes.append(node) - # ... - node = all_nodes.pop() - all_nodes += node.nodes - - for node in no_phandle_nodes: - if node.name != '/': - if node.path == '/': - phandle_value = self.add_label(node.name) - else: - phandle_value = self.add_label(node.path) - node.set_property('linux,phandle', phandle_value) - node.set_property('phandle', phandle_value) - - - def to_dts(self, tabsize: int = 4) -> str: - """ - Store FDT Object into string format (DTS) - - :param tabsize: - """ - result = "/dts-v1/;\n" - if self.header.version is not None: - result += "// version: {}\n".format(self.header.version) - result += "// last_comp_version: {}\n".format(self.header.last_comp_version) - if self.header.version >= 2: - result += "// boot_cpuid_phys: 0x{:X}\n".format(self.header.boot_cpuid_phys) - result += '\n' - if self.entries: - for entry in self.entries: - result += "/memreserve/ " - result += "{:#x} ".format(entry['address']) if entry['address'] else "0 " - result += "{:#x}".format(entry['size']) if entry['size'] else "0" - result += ";\n" - if self.root is not None: - result += self.root.to_dts(tabsize) - return result - - def to_dtb(self, version: int = None, last_comp_version: int = None, boot_cpuid_phys: int = None, strings: str = None) -> bytes: - """ - Export FDT Object into Binary Blob format (DTB) - - :param version: - :param last_comp_version: - :param boot_cpuid_phys: - :param strings: - - The strings param is useful (only) when manipulating a signed itb or dtb. The signature includes - the strings buffer in the dtb _in order_. C executables write the strings out in a surprising order. - The argument is used as an initial version of the strings buffer, so that all strings in the input - file are included (and in the same order) in the output file. Usage: - - # Read and parse dtb - with open(input, 'rb') as file: - data = file.read() - dtree = fdt.parse_dtb(data) - - # Read strings buffer (Assumes version >= 3) - strings_start = dtree.header.off_dt_strings - strings_end = strings_start + dtree.header.size_dt_strings - strings = data[strings_start:strings_end].decode("ascii") - - # Serialize dtb and write to output - data = dtree.to_dtb(strings=strings) - with open(output, 'wb') as file: - file.write(data) - - """ - if self.root is None: - return b'' - - from struct import pack - - if version is not None: - self.header.version = version - if last_comp_version is not None: - self.header.last_comp_version = last_comp_version - if boot_cpuid_phys is not None: - self.header.boot_cpuid_phys = boot_cpuid_phys - if self.header.version is None: - raise Exception("DTB Version must be specified !") - if strings is None: - strings = '' - - blob_entries = bytes() - if self.entries: - for entry in self.entries: - blob_entries += pack('>QQ', entry['address'], entry['size']) - blob_entries += pack('>QQ', 0, 0) - blob_data_start = self.header.size + len(blob_entries) - (blob_data, blob_strings, data_pos) = self.root.to_dtb(strings, blob_data_start, self.header.version) - blob_data += pack('>I', DTB_END) - self.header.size_dt_strings = len(blob_strings) - self.header.size_dt_struct = len(blob_data) - self.header.off_mem_rsvmap = self.header.size - self.header.off_dt_struct = blob_data_start - self.header.off_dt_strings = blob_data_start + len(blob_data) - self.header.total_size = blob_data_start + len(blob_data) + len(blob_strings) - blob_header = self.header.export() - return blob_header + blob_entries + blob_data + blob_strings.encode('ascii') - - -def parse_dts(text: str, root_dir: str = '') -> FDT: - """ - Parse DTS text file and create FDT Object - - :param text: - :param root_dir: - """ - ver = get_version_info(text) - text = strip_comments(text) - dts_lines = split_to_lines(text) - fdt_obj = FDT() - if 'version' in ver: - fdt_obj.header.version = ver['version'] - if 'last_comp_version' in ver: - fdt_obj.header.last_comp_version = ver['last_comp_version'] - if 'boot_cpuid_phys' in ver: - fdt_obj.header.boot_cpuid_phys = ver['boot_cpuid_phys'] - # parse entries - fdt_obj.entries = [] - for line in dts_lines: - if line.endswith('{'): - break - if line.startswith('/memreserve/'): - line = line.strip(';') - line = line.split() - if len(line) != 3 : - raise Exception() - fdt_obj.entries.append({'address': int(line[1], 0), 'size': int(line[2], 0)}) - # parse nodes - curnode = None - fdt_obj.root = None - for line in dts_lines: - if line.endswith('{'): - # start node - if ':' in line: #indicates the present of a label - label, rest = line.split(':') - node_name = rest.split()[0] - new_node = Node(node_name) - new_node.set_label(label) - - - else: - node_name = line.split()[0] - new_node = Node(node_name) - if fdt_obj.root is None: - fdt_obj.root = new_node - if curnode is not None: - curnode.append(new_node) - curnode = new_node - elif line.endswith('}'): - # end node - if curnode is not None: - if curnode.get_property('phandle') is None: - if curnode.label is not None: - handle = fdt_obj.add_label(curnode.label) - curnode.set_property('phandle', handle) - curnode = curnode.parent - else: - # properties - if line.find('=') == -1: - prop_name = line - prop_obj = Property(prop_name) - else: - line = line.split('=', maxsplit=1) - prop_name = line[0].rstrip(' ') - prop_value = line[1].lstrip(' ') - if prop_value.startswith('<'): - prop_obj = PropWords(prop_name) - prop_value = prop_value.replace('<', '').replace('>', '') - # ['interrupts ' = ' <0 5 4>, <0 6 4>'] - # just change ',' to ' ' -- to concatenate the values into single array - if ',' in prop_value: - prop_value = prop_value.replace(',', ' ') - - # keep the orginal references for phandles as a phantom - # property - if "&" in prop_value: - phantom_obj = PropStrings(prop_name+'_with_references') - phantom_obj.append(line[1].lstrip(' ')) - if curnode is not None: - curnode.append(phantom_obj) - for prop in prop_value.split(): - if prop.startswith('0x'): - prop_obj.append(int(prop, 16)) - elif prop.startswith('0b'): - prop_obj.append(int(prop, 2)) - elif prop.startswith('0'): - prop_obj.append(int(prop, 8)) - elif prop.startswith('&'): - prop_obj.append(fdt_obj.add_label(prop[1:])) - else: - prop_obj.append(int(prop)) - elif prop_value.startswith('['): - prop_obj = PropBytes(prop_name) - prop_value = prop_value.replace('[', '').replace(']', '') - for prop in prop_value.split(): - prop_obj.append(int(prop, 16)) - elif prop_value.startswith('/incbin/'): - prop_value = prop_value.replace('/incbin/("', '').replace('")', '') - prop_value = prop_value.split(',') - file_path = os.path.join(root_dir, prop_value[0].strip()) - file_offset = int(prop_value.strip(), 0) if len(prop_value) > 1 else 0 - file_size = int(prop_value.strip(), 0) if len(prop_value) > 2 else 0 - if file_path is None or not os.path.exists(file_path): - raise Exception("File path doesn't exist: {}".format(file_path)) - with open(file_path, "rb") as f: - f.seek(file_offset) - prop_data = f.read(file_size) if file_size > 0 else f.read() - prop_obj = PropIncBin(prop_name, prop_data, os.path.split(file_path)[1]) - elif prop_value.startswith('/plugin/'): - raise NotImplementedError("Not implemented property value: /plugin/") - elif prop_value.startswith('/bits/'): - raise NotImplementedError("Not implemented property value: /bits/") - else: - prop_obj = PropStrings(prop_name) - expect_open = True - in_prop = False - prop = '' - for c in prop_value: - if c == '"' and not in_prop and expect_open: - prop = '' - in_prop = True - elif c == '"' and in_prop: - if not len(prop) > 0: - raise ValueError('Empty string') - prop_obj.append(prop) - in_prop = False - expect_open = False - elif in_prop: - prop += c - elif c == ',' and not expect_open: - expect_open = True - elif c == ' ': - continue - else: - raise ValueError(f'Invalid char: {c}') - if expect_open: - raise ValueError('Expected string after ,') - if curnode is not None: - curnode.append(prop_obj) - - return fdt_obj - - -def parse_dtb(data: bytes, offset: int = 0) -> FDT: - """ - Parse FDT Binary Blob and create FDT Object - - :param data: FDT Binary Blob in bytes - :param offset: The offset of input data - """ - assert isinstance(data, (bytes, bytearray)), "Invalid argument type" - - from struct import unpack_from - - fdt_obj = FDT() - # parse header - fdt_obj.header = Header.parse(data) - # parse entries - index = fdt_obj.header.off_mem_rsvmap - while True: - entrie = dict(zip(('address', 'size'), unpack_from(">QQ", data, offset + index))) - index += 16 - if entrie['address'] == 0 and entrie['size'] == 0: - break - fdt_obj.entries.append(entrie) - # parse nodes - current_node = None - fdt_obj.root = None - index = fdt_obj.header.off_dt_struct - while True: - if len(data) < (offset + index + 4): - raise Exception("Index out of range !") - tag = unpack_from(">I", data, offset + index)[0] - index += 4 - if tag == DTB_BEGIN_NODE: - node_name = extract_string(data, offset + index) - index = ((index + len(node_name) + 4) & ~3) - if not node_name: node_name = '/' - new_node = Node(node_name) - if fdt_obj.root is None: - fdt_obj.root = new_node - if current_node is not None: - current_node.append(new_node) - current_node = new_node - elif tag == DTB_END_NODE: - if current_node is not None: - current_node = current_node.parent - elif tag == DTB_PROP: - prop_size, prop_string_pos, = unpack_from(">II", data, offset + index) - prop_start = index + 8 - if fdt_obj.header.version < 16 and prop_size >= 8: - prop_start = ((prop_start + 7) & ~0x7) - prop_name = extract_string(data, fdt_obj.header.off_dt_strings + prop_string_pos) - prop_raw_value = data[offset + prop_start : offset + prop_start + prop_size] - index = prop_start + prop_size - index = ((index + 3) & ~0x3) - if current_node is not None: - current_node.append(new_property(prop_name, prop_raw_value)) - elif tag == DTB_END: - break - else: - raise Exception("Unknown Tag: {}".format(tag)) - - return fdt_obj - - -def diff(fdt1: FDT, fdt2: FDT) -> tuple: - """ - Compare two flattened device tree objects and return list of 3 objects (same in 1 and 2, specific for 1, specific for 2) - - :param fdt1: The object 1 of FDT - :param fdt2: The object 2 of FDT - """ - assert isinstance(fdt1, FDT), "Invalid argument type" - assert isinstance(fdt2, FDT), "Invalid argument type" - - fdt_a = FDT(fdt1.header) - fdt_b = FDT(fdt2.header) - - if fdt1.header.version is not None and fdt2.header.version is not None: - fdt_same = FDT(fdt1.header if fdt1.header.version > fdt2.header.version else fdt2.header) - else: - fdt_same = FDT(fdt1.header) - - if fdt1.entries and fdt2.entries: - for entry_a in fdt1.entries: - for entry_b in fdt2.entries: - if entry_a['address'] == entry_b['address'] and entry_a['size'] == entry_b['size']: - fdt_same.entries.append(entry_a) - break - - for entry_a in fdt1.entries: - found = False - for entry_s in fdt_same.entries: - if entry_a['address'] == entry_s['address'] and entry_a['size'] == entry_s['size']: - found = True - break - if not found: - fdt_a.entries.append(entry_a) - - for entry_b in fdt2.entries: - found = False - for entry_s in fdt_same.entries: - if entry_b['address'] == entry_s['address'] and entry_b['size'] == entry_s['size']: - found = True - break - if not found: - fdt_b.entries.append(entry_b) - - for path, nodes, props in fdt1.walk(): - try: - rnode = fdt2.get_node(path) - except: - rnode = None - - for node_b in nodes: - if rnode is None or rnode.get_subnode(node_b.name) is None: - fdt_a.add_item(Node(node_b.name), path) - else: - fdt_same.add_item(Node(node_b.name), path) - - for prop_a in props: - if rnode is not None and prop_a == rnode.get_property(prop_a.name): - fdt_same.add_item(prop_a.copy(), path) - else: - fdt_a.add_item(prop_a.copy(), path) - - for path, nodes, props in fdt2.walk(): - try: - rnode = fdt_same.get_node(path) - except: - rnode = None - - for node_b in nodes: - if rnode is None or rnode.get_subnode(node_b.name) is None: - fdt_b.add_item(Node(node_b.name), path) - - for prop_b in props: - if rnode is None or prop_b != rnode.get_property(prop_b.name): - fdt_b.add_item(prop_b.copy(), path) - - return fdt_same, fdt_a, fdt_b diff --git a/python/fdt/__main__.py b/python/fdt/__main__.py deleted file mode 100644 index 9040211..0000000 --- a/python/fdt/__main__.py +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2017 Martin Olejar -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import fdt -import argparse - - -######################################################################################################################## -# Helper Functions -######################################################################################################################## -def parse_fdt(file_path: str, file_type: str): - """ - Parse *.dtb ot *.dts input file and return FDT object - - :param file_path: The path to input file - :param file_type: File type 'dtb', 'dts' or 'auto' - """ - - if not os.path.exists(file_path): - raise Exception('File doesnt exist: {}'.format(file_path)) - - if file_type == 'auto': - if file_path.endswith(".dtb"): - file_type = 'dtb' - elif file_path.endswith(".dts"): - file_type = 'dts' - else: - raise Exception('Not supported file extension: {}'.format(file_path)) - - if file_type == 'dtb': - with open(file_path, 'rb') as f: - obj = fdt.parse_dtb(f.read()) - else: - with open(file_path, 'r') as f: - obj = fdt.parse_dts(f.read(), os.path.dirname(file_path)) - - return obj - - -######################################################################################################################## -# Commands Functions -######################################################################################################################## -def pack(in_file: str, out_file: str, version: int, lc_version: int, cpu_id: int, update_phandles: bool): - """ - The implementation of pack command. - - :param in_file: Input File Path - :param out_file: Output File Path - :param version: DTB version - :param lc_version: DTB Last Compatible Version - :param cpu_id: Boot CPU ID - :param update_phandles: If True phandles will be updated - """ - - if version is not None and version > fdt.Header.MAX_VERSION: - raise Exception("DTB Version must be lover or equal {} !".format(fdt.Header.MAX_VERSION)) - - fdt_obj = parse_fdt(in_file, 'dts') - if update_phandles: - fdt_obj.update_phandles() - raw_data = fdt_obj.to_dtb(version, lc_version, cpu_id) - - with open(out_file, 'wb') as f: - f.write(raw_data) - - print(" DTB saved as: {}".format(out_file)) - - -def unpack(in_file: str, out_file: str, tab_size): - """ - The implementation of unpack command. - - :param in_file: Input File Path - :param out_file: Output File Path - :param tab_size: Tabulator size in count of spaces - """ - fdt_obj = parse_fdt(in_file, 'dtb') - - with open(out_file, 'w') as f: - f.write(fdt_obj.to_dts(tab_size)) - - print(" DTS saved as: {}".format(out_file)) - - -def merge(out_file: str, in_files: list, file_type: str, tab_size: int): - """ - The implementation of merge command. - - :param out_file: Output File Path - :param in_files: Input Files Path - :param file_type: The type of input files - :param tab_size: Tabulator size in count of spaces - """ - fdt_obj = None - - for file in in_files: - obj = parse_fdt(file, file_type) - if fdt_obj is None: - fdt_obj = obj - else: - fdt_obj.merge(obj) - - with open(out_file, 'w') as f: - f.write(fdt_obj.to_dts(tab_size)) - - print(" Output saved as: {}".format(out_file)) - - -def diff(in_file1: str, in_file2: str, file_type: str, out_dir: str): - """ - The implementation of diff command. - - :param in_file1: Input File1 Path - :param in_file2: Input File2 Path - :param file_type: The type of input files - :param out_dir: Path to output directory - """ - # load input files - fdt1 = parse_fdt(in_file1, file_type) - fdt2 = parse_fdt(in_file2, file_type) - - # compare it - diff = fdt.diff(fdt1, fdt2) - if diff[0].empty: - print(" Input files are completely different !") - sys.exit() - - # create output directory - os.makedirs(out_dir, exist_ok=True) - - # get names for output files - file_name = ( - "same.dts", - os.path.splitext(os.path.basename(in_file1))[0] + ".dts", - os.path.splitext(os.path.basename(in_file2))[0] + ".dts") - - # save output files - for index, obj in enumerate(diff): - if not obj.empty: - with open(os.path.join(out_dir, file_name[index]), 'w') as f: - f.write(obj.to_dts()) - - print(" Diff output saved into: {}".format(out_dir)) - - -######################################################################################################################## -# Main -######################################################################################################################## -def main(): - # cli interface - parser = argparse.ArgumentParser( - prog="pydtc", - description="Flat Device Tree (FDT) tool for manipulation with *.dtb and *.dts files") - parser.add_argument('-v', '--version', action='version', version=fdt.__version__) - subparsers = parser.add_subparsers(dest='command') - - # pack command - pack_parser = subparsers.add_parser('pack', help='Pack *.dts into binary blob (*.dtb)') - pack_parser.add_argument('dts_file', nargs=1, help='Path to *.dts file') - pack_parser.add_argument('-v', dest='version', type=int, help='DTB Version') - pack_parser.add_argument('-l', dest='lc_version', type=int, help='DTB Last Compatible Version') - pack_parser.add_argument('-c', dest='cpu_id', type=int, help='Boot CPU ID') - pack_parser.add_argument('-p', dest='phandles', action='store_true', help='Update phandles') - pack_parser.add_argument('-o', dest='dtb_file', type=str, help='Output path with file name (*.dtb)') - - # unpack command - unpack_parser = subparsers.add_parser('unpack', help='Unpack *.dtb into readable format (*.dts)') - unpack_parser.add_argument('dtb_file', nargs=1, help='Path to *.dtb file') - unpack_parser.add_argument('-s', dest='tab_size', type=int, default=4, help='Tabulator Size') - unpack_parser.add_argument('-o', dest='dts_file', type=str, help='Output path with file name (*.dts)') - - # merge command - merge_parser = subparsers.add_parser('merge', help='Merge more files in *.dtb or *.dts format') - merge_parser.add_argument('out_file', nargs=1, help='Output path with file name (*.dts or *.dtb)') - merge_parser.add_argument('in_files', nargs='+', help='Path to input files') - merge_parser.add_argument('-t', dest='type', type=str, choices=['auto', 'dts', 'dtb'], help='Input file type') - merge_parser.add_argument('-s', dest='tab_size', type=int, default=4, help='Tabulator Size for dts') - - # diff command - diff_parser = subparsers.add_parser('diff', help='Compare two files in *.dtb or *.dts format') - diff_parser.add_argument('in_file1', nargs=1, help='Path to dts or dtb file') - diff_parser.add_argument('in_file2', nargs=1, help='Path to dts or dtb file') - diff_parser.add_argument('-t', dest='type', type=str, choices=['auto', 'dts', 'dtb'], help='Input file type') - diff_parser.add_argument('-o', dest='out_dir', type=str, help='Output directory') - - args = parser.parse_args() - - try: - if args.command == 'pack': - in_file = args.dts_file[0] - if args.dtb_file is None: - out_file = os.path.splitext(os.path.basename(in_file))[0] + ".dtb" - else: - out_file = args.dtb_file.lstrip() - pack(in_file, out_file, args.version, args.lc_version, args.cpu_id, args.phandles) - - elif args.command == 'unpack': - in_file = args.dtb_file[0] - if args.dts_file is None: - out_file = os.path.splitext(os.path.basename(in_file))[0] + ".dts" - else: - out_file = args.dts_file.lstrip() - unpack(in_file, out_file, args.tab_size) - - elif args.command == 'merge': - merge(args.out_file[0], args.in_files, args.type, args.tab_size) - - elif args.command == 'diff': - out_dir = args.out_dir if args.out_dir else os.path.join(os.getcwd(), 'diff_out') - diff(args.in_file1[0], args.in_file2[0], args.type, out_dir.lstrip()) - - else: - parser.print_help() - - except Exception as e: - print("[pydtc] Execution Error !") - print(str(e) if str(e) else "Unknown Error", file=sys.stderr) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/python/fdt/header.py b/python/fdt/header.py deleted file mode 100644 index 18b94e6..0000000 --- a/python/fdt/header.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2017 Martin Olejar -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from struct import unpack_from, pack - - -######################################################################################################################## -# Binary Blob Constants -######################################################################################################################## - -DTB_BEGIN_NODE = 0x1 -DTB_END_NODE = 0x2 -DTB_PROP = 0x3 -DTB_NOP = 0x4 -DTB_END = 0x9 - - -######################################################################################################################## -# Header Class -######################################################################################################################## - -class Header: - - MIN_SIZE = 4 * 7 - MAX_SIZE = 4 * 10 - - MAX_VERSION = 17 - - MAGIC_NUMBER = 0xD00DFEED - - @property - def version(self): - return self._version - - @version.setter - def version(self, value): - if value > self.MAX_VERSION: - raise ValueError("Invalid Version {}, use: 0 - 17 !".format(value)) - # update size and padding - self._size = self.MIN_SIZE - if value >= 2: - self._size += 4 - if value >= 3: - self._size += 4 - if value >= 17: - self._size += 4 - self._padding = 8 - (self._size % 8) if self._size % 8 != 0 else 0 - self._version = value - self.last_comp_version = value - 1 - - @property - def size(self): - return self._size + self._padding - - @property - def padding(self): - return self._padding - - def __init__(self): - # private variables - self._version = None - self._size = 0 - self._padding = 0 - # public variables - self.total_size = 0 - self.off_dt_struct = 0 - self.off_dt_strings = 0 - self.off_mem_rsvmap = 0 - self.last_comp_version = 0 - # version depend variables - self.boot_cpuid_phys = 0 - self.size_dt_strings = None - self.size_dt_struct = None - - def __str__(self): - return ''.format(self.version, self.size) - - def info(self): - nfo = 'FDT Header:' - nfo += '- Version: {}'.format(self.version) - nfo += '- Size: {}'.format(self.size) - return nfo - - def export(self) -> bytes: - """ - - :return: - """ - if self.version is None: - raise Exception("Header version must be specified !") - - blob = pack('>7I', self.MAGIC_NUMBER, self.total_size, self.off_dt_struct, self.off_dt_strings, - self.off_mem_rsvmap, self.version, self.last_comp_version) - if self.version >= 2: - blob += pack('>I', self.boot_cpuid_phys) - if self.version >= 3: - blob += pack('>I', self.size_dt_strings) - if self.version >= 17: - blob += pack('>I', self.size_dt_struct) - if self.padding: - blob += bytes([0] * self.padding) - - return blob - - @classmethod - def parse(cls, data: bytes, offset: int = 0): - """ - - :param data: - :param offset: - """ - if len(data) < (offset + cls.MIN_SIZE): - raise ValueError('Data size too small !') - - header = cls() - (magic_number, - header.total_size, - header.off_dt_struct, - header.off_dt_strings, - header.off_mem_rsvmap, - header.version, - header.last_comp_version) = unpack_from('>7I', data, offset) - offset += cls.MIN_SIZE - - if magic_number != cls.MAGIC_NUMBER: - raise Exception('Invalid Magic Number') - if header.last_comp_version > cls.MAX_VERSION - 1: - raise Exception('Invalid last compatible Version {}'.format(header.last_comp_version)) - - if header.version >= 2: - header.boot_cpuid_phys = unpack_from('>I', data, offset)[0] - offset += 4 - - if header.version >= 3: - header.size_dt_strings = unpack_from('>I', data, offset)[0] - offset += 4 - - if header.version >= 17: - header.size_dt_struct = unpack_from('>I', data, offset)[0] - offset += 4 - - return header diff --git a/python/fdt/items.py b/python/fdt/items.py deleted file mode 100644 index cc7ce8b..0000000 --- a/python/fdt/items.py +++ /dev/null @@ -1,770 +0,0 @@ -# Copyright 2017 Martin Olejar -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from struct import pack, Struct -from string import printable - -from .header import Header, DTB_PROP, DTB_BEGIN_NODE, DTB_END_NODE -from .misc import is_string, line_offset - -BIGENDIAN_WORD = Struct(">I") - -######################################################################################################################## -# Helper methods -######################################################################################################################## - -def new_property(name: str, raw_value: bytes) -> object: - """ - Instantiate property with raw value type - - :param name: Property name - :param raw_value: Property raw data - """ - if is_string(raw_value): - obj = PropStrings(name) - # Extract strings from raw value - for st in raw_value.decode('ascii').split('\0'): - if st: - obj.append(st) - return obj - - elif len(raw_value) > 0 and len(raw_value) <= 256*1024 and len(raw_value) % 4 == 0: - obj = PropWords(name) - # Extract words from raw value - obj.data = [BIGENDIAN_WORD.unpack(raw_value[i:i + 4])[0] for i in range(0, len(raw_value), 4)] - return obj - - elif len(raw_value): - return PropBytes(name, data=raw_value) - - return Property(name) - - -######################################################################################################################## -# Base Class -######################################################################################################################## - -class BaseItem: - - @property - def name(self): - return self._name - - @property - def label(self): - return self._label - - @property - def parent(self): - return self._parent - - @property - def path(self): - node = self._parent - path = "" - while node: - if node.name == '/': break - path = '/' + node.name + path - node = node.parent - return path if path else '/' - - def __init__(self, name: str, label=None): - """ - BaseItem constructor - - :param name: Item name - """ - assert isinstance(name, str) - assert all(c in printable for c in name), "The value must contain just printable chars !" - self._name = name - self._label = label - self._parent = None - - def __str__(self): - """ String representation """ - return "{}".format(self.name) - - def set_name(self, value: str): - """ - Set item name - - :param value: The name in string format - """ - assert isinstance(value, str) - assert all(c in printable for c in value), "The value must contain just printable chars !" - self._name = value - - def set_label(self, value: str): - """ - Set item label - - :param value: The label in string format - """ - assert isinstance(value, str) - assert all(c in printable for c in value), "The value must contain just printable chars !" - self._label = value - - - def set_parent(self, value): - """ - Set item parent - - :param value: The parent node - """ - assert isinstance(value, Node) - self._parent = value - - def to_dts(self, tabsize: int = 4, depth: int = 0): - raise NotImplementedError() - - def to_dtb(self, strings: str, pos: int = 0, version: int = Header.MAX_VERSION): - raise NotImplementedError() - - -######################################################################################################################## -# Property Classes -######################################################################################################################## - -class Property(BaseItem): - - def __getitem__(self, value): - """ Returns No Items """ - return None - - def __eq__(self, obj): - """ Check Property object equality """ - return isinstance(obj, Property) and self.name == obj.name - - def copy(self): - """ Get object copy """ - return Property(self.name) - - def to_dts(self, tabsize: int = 4, depth: int = 0): - """ - Get string representation - - :param tabsize: Tabulator size in count of spaces - :param depth: Start depth for line - """ - return line_offset(tabsize, depth, '{};\n'.format(self.name)) - - def to_dtb(self, strings: str, pos: int = 0, version: int = Header.MAX_VERSION): - """ - Get binary blob representation - - :param strings: - :param pos: - :param version: - """ - strpos = strings.find(self.name + '\0') - if strpos < 0: - strpos = len(strings) - strings += self.name + '\0' - pos += 12 - return pack('>III', DTB_PROP, 0, strpos), strings, pos - - -class PropStrings(Property): - """Property with strings as value""" - - @property - def value(self): - return self.data[0] if self.data else None - - def __init__(self, name: str, *args): - """ - PropStrings constructor - - :param name: Property name - :param args: str1, str2, ... - """ - super().__init__(name) - self.data = [] - for arg in args: - self.append(arg) - - def __str__(self): - """ String representation """ - return "{} = {}".format(self.name, self.data) - - def __len__(self): - """ Get strings count """ - return len(self.data) - - def __getitem__(self, index): - """ Get string by index """ - return self.data[index] - - def __eq__(self, obj): - """ Check PropStrings object equality """ - if not isinstance(obj, PropStrings) or self.name != obj.name or len(self) != len(obj): - return False - for index in range(len(self)): - if self.data[index] != obj[index]: - return False - return True - - def copy(self): - """ Get object copy """ - return PropStrings(self.name, *self.data) - - def append(self, value: str): - assert isinstance(value, str) - assert len(value) > 0, "Invalid strings value" - assert all(c in printable or c in ('\r', '\n') for c in value), "Invalid chars in strings value" - self.data.append(value) - - def pop(self, index: int): - assert 0 <= index < len(self.data), "Index out of range" - return self.data.pop(index) - - def clear(self): - self.data.clear() - - def to_dts(self, tabsize: int = 4, depth: int = 0): - """ - Get string representation - - :param tabsize: Tabulator size in count of spaces - :param depth: Start depth for line - """ - result = line_offset(tabsize, depth, self.name) - result += ' = "' - result += '", "'.join([item.replace('"', '\\"') for item in self.data]) - result += '";\n' - return result - - def to_dtb(self, strings: str, pos: int = 0, version: int = Header.MAX_VERSION): - """ - Get blob representation - - :param strings: - :param pos: - :param version: - """ - blob = pack('') - for chars in self.data: - blob += chars.encode('ascii') + pack('b', 0) - blob_len = len(blob) - if version < 16 and (pos + 12) % 8 != 0: - blob = pack('b', 0) * (8 - ((pos + 12) % 8)) + blob - if blob_len % 4: - blob += pack('b', 0) * (4 - (blob_len % 4)) - strpos = strings.find(self.name + '\0') - if strpos < 0: - strpos = len(strings) - strings += self.name + '\0' - blob = pack('>III', DTB_PROP, blob_len, strpos) + blob - pos += len(blob) - return blob, strings, pos - - -class PropWords(Property): - """Property with words as value""" - - @property - def value(self): - return self.data[0] if self.data else None - - def __init__(self, name, *args): - """ - PropWords constructor - - :param name: Property name - :param args: word1, word2, ... - """ - super().__init__(name) - self.data = [] - self.word_size = 32 - for val in args: - self.append(val) - - def __str__(self): - """ String representation """ - return "{} = {}".format(self.name, self.data) - - def __getitem__(self, index): - """ Get word by index """ - return self.data[index] - - def __len__(self): - """ Get words count """ - return len(self.data) - - def __eq__(self, prop): - """ Check PropWords object equality """ - if not isinstance(prop, PropWords): - return False - if self.name != prop.name: - return False - if len(self) != len(prop): - return False - for index in range(len(self)): - if self.data[index] != prop[index]: - return False - return True - - def copy(self): - return PropWords(self.name, *self.data) - - def append(self, value): - assert isinstance(value, int), "Invalid object type" - assert 0 <= value < 2**self.word_size, "Invalid word value {}, use <0x0 - 0x{:X}>".format( - value, 2**self.word_size - 1) - self.data.append(value) - - def pop(self, index): - assert 0 <= index < len(self.data), "Index out of range" - return self.data.pop(index) - - def clear(self): - self.data.clear() - - def to_dts(self, tabsize: int = 4, depth: int = 0): - """ - Get string representation - - :param tabsize: Tabulator size in count of spaces - :param depth: Start depth for line - """ - result = line_offset(tabsize, depth, self.name) - result += ' = <' - result += ' '.join(["0x{:X}".format(word) for word in self.data]) - result += ">;\n" - return result - - def to_dtb(self, strings: str, pos: int = 0, version: int = Header.MAX_VERSION): - """ - Get blob representation - - :param strings: - :param pos: - :param version: - """ - strpos = strings.find(self.name + '\0') - if strpos < 0: - strpos = len(strings) - strings += self.name + '\0' - blob = pack('>III', DTB_PROP, len(self.data) * 4, strpos) - blob += bytes().join([BIGENDIAN_WORD.pack(word) for word in self.data]) - pos += len(blob) - return blob, strings, pos - - -class PropBytes(Property): - """Property with bytes as value""" - - def __init__(self, name, *args, data=None): - """ - PropBytes constructor - - :param name: Property name - :param args: byte0, byte1, ... - :param data: Data as list, bytes or bytearray - """ - super().__init__(name) - self.data = bytearray(args) - if data: - assert isinstance(data, (list, bytes, bytearray)) - self.data += bytearray(data) - - def __str__(self): - """ String representation """ - return "{} = {}".format(self.name, self.data) - - def __getitem__(self, index): - """Get byte by index """ - return self.data[index] - - def __len__(self): - """ Get bytes count """ - return len(self.data) - - def __eq__(self, prop): - """ Check PropBytes object equality """ - if not isinstance(prop, PropBytes): - return False - if self.name != prop.name: - return False - if len(self) != len(prop): - return False - for index in range(len(self)): - if self.data[index] != prop[index]: - return False - return True - - def copy(self): - """ Create a copy of object """ - return PropBytes(self.name, data=self.data) - - def append(self, value): - assert isinstance(value, int), "Invalid object type" - assert 0 <= value <= 0xFF, "Invalid byte value {}, use <0 - 255>".format(value) - self.data.append(value) - - def pop(self, index): - assert 0 <= index < len(self.data), "Index out of range" - return self.data.pop(index) - - def clear(self): - self.data = bytearray() - - def to_dts(self, tabsize: int = 4, depth: int = 0): - """ - Get string representation - - :param tabsize: Tabulator size in count of spaces - :param depth: Start depth for line - """ - result = line_offset(tabsize, depth, self.name) - result += ' = [' - result += ' '.join(["{:02X}".format(byte) for byte in self.data]) - result += '];\n' - return result - - def to_dtb(self, strings: str, pos: int = 0, version: int = Header.MAX_VERSION): - """ - Get blob representation - - :param strings: - :param pos: - :param version: - """ - strpos = strings.find(self.name + '\0') - if strpos < 0: - strpos = len(strings) - strings += self.name + '\0' - blob = pack('>III', DTB_PROP, len(self.data), strpos) - blob += bytes(self.data) - if len(blob) % 4: - blob += bytes([0] * (4 - (len(blob) % 4))) - pos += len(blob) - return blob, strings, pos - - -class PropIncBin(PropBytes): - """Property with bytes as value""" - - def __init__(self, name, data=None, file_name=None, rpath=None): - """ - PropIncBin constructor - - :param name: Property name - :param data: Data as list, bytes or bytearray - :param file_name: File name - :param rpath: Relative path - """ - super().__init__(name, data) - self.file_name = file_name - self.relative_path = rpath - - def __eq__(self, prop): - """ Check PropIncBin object equality """ - if not isinstance(prop, PropIncBin): - return False - if self.name != prop.name: - return False - if self.file_name != prop.file_name: - return False - if self.relative_path != prop.relative_path: - return False - if self.data != prop.data: - return False - return True - - def copy(self): - """ Create a copy of object """ - return PropIncBin(self.name, self.data, self.file_name, self.relative_path) - - def to_dts(self, tabsize: int = 4, depth: int = 0): - """ - Get string representation - - :param tabsize: Tabulator size in count of spaces - :param depth: Start depth for line - """ - file_path = self.file_name - if self.relative_path is not None: - file_path = "{}/{}".format(self.relative_path, self.file_name) - result = line_offset(tabsize, depth, self.name) - result += " = /incbin/(\"{}\");\n".format(file_path) - return result - - -######################################################################################################################## -# Node Class -######################################################################################################################## - -class Node(BaseItem): - """Node representation""" - - @property - def path(self): - return super().path + '/' + self.name - - @property - def props(self): - return self._props - - @property - def nodes(self): - return self._nodes - - @property - def empty(self): - return False if self.nodes or self.props else True - - def __init__(self, name, *args, label=None): - """ - Node constructor - - :param name: Node name - :param args: List of properties and subnodes - """ - super().__init__(name, label=label) - self._props = [] - self._nodes = [] - for item in args: - self.append(item) - - def __str__(self): - """ String representation """ - return "< {}: {} props, {} nodes >".format(self.name, len(self.props), len(self.nodes)) - - def __eq__(self, node): - """ Check node equality """ - if not isinstance(node, Node): - return False - if self.name != node.name or \ - len(self.props) != len(node.props) or \ - len(self.nodes) != len(node.nodes): - return False - for p in self.props: - if p not in node.props: - return False - for n in self.nodes: - if n not in node.nodes: - return False - return True - - def copy(self): - """ Create a copy of Node object """ - node = Node(self.name, label=self.label) - for p in self.props: - node.append(p.copy()) - for n in self.nodes: - node.append(n.copy()) - return node - - def get_property(self, name): - """ - Get property object by its name - - :param name: Property name - """ - for p in self.props: - if p.name == name: - return p - return None - - def set_property(self, name, value): - """ - Set property - - :param name: Property name - :param value: Property value - """ - if value is None: - new_prop = Property(name) - elif isinstance(value, int): - new_prop = PropWords(name, value) - elif isinstance(value, str): - new_prop = PropStrings(name, value) - elif isinstance(value, list) and isinstance(value[0], int): - new_prop = PropWords(name, *value) - elif isinstance(value, list) and isinstance(value[0], str): - new_prop = PropStrings(name, *value) - elif isinstance(value, (bytes, bytearray)): - new_prop = PropBytes(name, data=value) - else: - raise TypeError('Value type not supported') - new_prop.set_parent(self) - old_prop = self.get_property(name) - if old_prop is None: - self.props.append(new_prop) - else: - index = self.props.index(old_prop) - self.props[index] = new_prop - - def get_subnode(self, name: str): - """ - Get subnode object by name - - :param name: Subnode name - """ - for n in self.nodes: - if n.name == name: - return n - return None - - def exist_property(self, name: str) -> bool: - """ - Check if property exist and return True if exist else False - - :param name: Property name - """ - return False if self.get_property(name) is None else True - - def exist_subnode(self, name: str) -> bool: - """ - Check if subnode exist and return True if exist else False - - :param name: Subnode name - """ - return False if self.get_subnode(name) is None else True - - def remove_property(self, name: str): - """ - Remove property object by its name. - - :param name: Property name - """ - item = self.get_property(name) - if item is not None: - self.props.remove(item) - - def remove_subnode(self, name: str): - """ - Remove subnode object by its name. - - :param name: Subnode name - """ - item = self.get_subnode(name) - if item is not None: - self.nodes.remove(item) - - def append(self, item): - """ - Append node or property - - :param item: The node or property object - """ - assert isinstance(item, (Node, Property)), "Invalid object type, use \"Node\" or \"Property\"" - - if isinstance(item, Property): - if self.get_property(item.name) is not None: - raise Exception("{}: \"{}\" property already exists".format(self, item.name)) - item.set_parent(self) - self.props.append(item) - - else: - if self.get_subnode(item.name) is not None: - raise Exception("{}: \"{}\" node already exists".format(self, item.name)) - if item is self: - raise Exception("{}: append the same node {}".format(self, item.name)) - item.set_parent(self) - self.nodes.append(item) - - def merge(self, node_obj, replace: bool = True): - """ - Merge two nodes - - :param node_obj: Node object - :param replace: If True, replace current properties with the given properties - """ - assert isinstance(node_obj, Node), "Invalid object type" - - def get_property_index(name): - for i, p in enumerate(self.props): - if p.name == name: - return i - return None - - def get_subnode_index(name): - for i, n in enumerate(self.nodes): - if n.name == name: - return i - return None - - for prop in node_obj.props: - index = get_property_index(prop.name) - if index is None: - self.append(prop.copy()) - elif prop in self._props: - continue - elif replace: - new_prop = prop.copy() - new_prop.set_parent(self) - self._props[index] = new_prop - else: - pass - - for sub_node in node_obj.nodes: - index = get_subnode_index(sub_node.name) - if index is None: - self.append(sub_node.copy()) - elif sub_node in self._nodes: - continue - else: - self._nodes[index].merge(sub_node, replace) - - def to_dts(self, tabsize: int = 4, depth: int = 0) -> str: - """ - Get string representation of NODE object - - :param tabsize: Tabulator size in count of spaces - :param depth: Start depth for line - """ - if self._label is not None: - dts = line_offset(tabsize, depth, self._label + ': ' + self.name + ' {\n') - else: - dts = line_offset(tabsize, depth, self.name + ' {\n') - # phantom properties which maintain reference state info - # have names ending with _with_references - # don't write those out to dts file - dts += ''.join( - prop.to_dts(tabsize, depth + 1) - for prop in self._props if prop.name.endswith('_with_references') is False) - dts += ''.join(node.to_dts(tabsize, depth + 1) for node in self._nodes) - dts += line_offset(tabsize, depth, "};\n") - return dts - - def to_dtb(self, strings: str, pos: int = 0, version: int = Header.MAX_VERSION) -> tuple: - """ - Get NODE in binary blob representation - - :param strings: - :param pos: - :param version: - """ - if self.name == '/': - blob = pack('>II', DTB_BEGIN_NODE, 0) - else: - blob = pack('>I', DTB_BEGIN_NODE) - blob += self.name.encode('ascii') + b'\0' - if len(blob) % 4: - blob += pack('b', 0) * (4 - (len(blob) % 4)) - pos += len(blob) - for prop in self._props: - # phantom property too maintain reference state should - # not write out to dtb file - if prop.name.endswith('_with_references') is False: - (data, strings, pos) = prop.to_dtb(strings, pos, version) - blob += data - for node in self._nodes: - (data, strings, pos) = node.to_dtb(strings, pos, version) - blob += data - pos += 4 - blob += pack('>I', DTB_END_NODE) - return blob, strings, pos diff --git a/python/fdt/misc.py b/python/fdt/misc.py deleted file mode 100644 index a7d1382..0000000 --- a/python/fdt/misc.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2017 Martin Olejar -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -from string import printable - - -def is_string(data): - """ Check property string validity """ - if not len(data): - return None - if data[-1] != 0: - return None - pos = 0 - while pos < len(data): - posi = pos - while pos < len(data) and \ - data[pos] != 0 and \ - data[pos] in printable.encode() and \ - data[pos] not in (ord('\r'), ord('\n')): - pos += 1 - if data[pos] != 0 or pos == posi: - return None - pos += 1 - return True - - -def extract_string(data, offset=0): - """ Extract string """ - str_end = offset - while data[str_end] != 0: - str_end += 1 - return data[offset:str_end].decode("ascii") - - -def line_offset(tabsize, offset, string): - offset = " " * (tabsize * offset) - return offset + string - - -def get_version_info(text): - ret = dict() - for line in text.split('\n'): - line = line.rstrip('\0') - if line and line.startswith('/ {'): - break - if line and line.startswith('//'): - line = line.replace('//', '').replace(':', '') - line = line.split() - if line[0] in ('version', 'last_comp_version', 'boot_cpuid_phys'): - ret[line[0]] = int(line[1], 0) - return ret - - -def strip_comments(text): - text = re.sub('//.*?(\r\n?|\n)|/\*.*?\*/', '\n', text, flags=re.S) - return text - - -def split_to_lines(text): - lines = [] - mline = str() - for line in text.split('\n'): - line = line.replace('\t', ' ') - line = line.rstrip('\0') - line = line.rstrip(' ') - line = line.lstrip(' ') - if not line or line.startswith('/dts-'): - continue - if line.endswith('{') or line.endswith(';'): - line = line.replace(';', '') - lines.append(mline + line) - mline = str() - else: - mline += line + ' ' - - return lines - diff --git a/python/ubireader/__init__.py b/python/ubireader/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/ubireader/debug.py b/python/ubireader/debug.py deleted file mode 100644 index 7201fe9..0000000 --- a/python/ubireader/debug.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import sys -import traceback -from ubireader import settings - -def log(obj, message): - if settings.logging_on or settings.logging_on_verbose: - print('{} {}'.format(obj.__name__, message)) - -def verbose_log(obj, message): - if settings.logging_on_verbose: - log(obj, message) - -def verbose_display(displayable_obj): - if settings.logging_on_verbose: - print(displayable_obj.display('\t')) - -def error(obj, level, message): - if settings.error_action == 'exit': - print('{} {}: {}'.format(obj.__name__, level, message)) - if settings.fatal_traceback: - traceback.print_exc() - sys.exit(1) - - else: - if level.lower() == 'warn': - print('{} {}: {}'.format(obj.__name__, level, message)) - elif level.lower() == 'fatal': - print('{} {}: {}'.format(obj.__name__, level, message)) - if settings.fatal_traceback: - traceback.print_exc() - sys.exit(1) - else: - print('{} {}: {}'.format(obj.__name__, level, message)) - diff --git a/python/ubireader/scripts/__init__.py b/python/ubireader/scripts/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/python/ubireader/scripts/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/python/ubireader/scripts/ubireader_display_blocks.py b/python/ubireader/scripts/ubireader_display_blocks.py deleted file mode 100644 index 8273219..0000000 --- a/python/ubireader/scripts/ubireader_display_blocks.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/scripts/ubireader_display_blocks -# (c) 2019 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -############################################################# -# Search by block parameters and display information about -# matching blocks. -############################################################# - -import os -import sys -import argparse -from ubireader.ubi import ubi_base -from ubireader.ubi_io import ubi_file -from ubireader import settings -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - - -def main(): - description = 'Search for specified blocks and display information.' - usage = """ - ubireader_display_blocks "{'block.attr': value,...}" path/to/image - Search for blocks by given parameters and display information about them. - This is block only, no volume or image information is created, which can - be used to debug file and image extraction. - Example: - "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" - This matches block.peb_num 0, 1, 100, 101, and 102 - with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. - For a full list of parameters check ubireader.ubi.block.description. - """ - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('block_search_params', - help=""" - Double quoted Dict of ubi.block.description attributes, which is run through eval(). - Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" - """) - - parser.add_argument('filepath', help='File with blocks of interest.') - - if len(sys.argv) == 1: - parser.print_help() - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - else: - parser.error('File path must be provided.') - sys.exit(1) - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - if args.block_search_params: - try: - search_params = eval(args.block_search_params) - - if not isinstance(search_params, dict): - parser.error('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') - - except NameError as e: - parser.error('Search Param Error: Dict key block attrs must be single quoted.') - - except Exception as e: - parser.error('Search Param Error: %s' % e) - - else: - parser.error('No search parameters given, -b arg is required.') - - - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - ubi_obj = ubi_base(ufile_obj) - blocks = [] - - for block in ubi_obj.blocks: - match = True - - for key in search_params: - b = ubi_obj.blocks[block] - - for attr in key.split('.'): - if hasattr(b, attr): - b = getattr(b, attr) - - if isinstance(search_params[key], list): - if isinstance(b, list): - for value in b: - if value in search_params[key]: - break - else: - match = False - elif b not in search_params[key]: - match = False - - elif b != search_params[key]: - match = False - break - - if match: - blocks.append(ubi_obj.blocks[block]) - - ufile_obj.close() - - print('\nBlock matches: %s' % len(blocks)) - - for block in blocks: - print(block.display()) - -if __name__=='__main__': - main() diff --git a/python/ubireader/scripts/ubireader_display_info.py b/python/ubireader/scripts/ubireader_display_info.py deleted file mode 100644 index b5291aa..0000000 --- a/python/ubireader/scripts/ubireader_display_info.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size -from ubireader.ubi_io import ubi_file, leb_virtual_file - -def main(): - start = time.time() - description = 'Show information about UBI or UBIFS image.' - usage = 'ubireader_display_info [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-u', '--ubifs-info', action='store_true', dest='ubifs_info', - help='Get UBIFS information from inside a UBI image. (default: false)') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('filepath', help='File to extract contents of.') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - ubifs_info = args.ubifs_info - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Display UBI info if not UBIFS request. - if not ubifs_info: - print(ubi_obj.display()) - - # Loop through found images in file. - for image in ubi_obj.images: - # Display image information if not UBIFS request. - if not ubifs_info: - print('%s' % image.display('\t')) - - # Loop through volumes in each image. - for volume in image.volumes: - # Show UBI or UBIFS info. - if not ubifs_info: - - # Display volume information. - print(image.volumes[volume].display('\t\t')) - - else: - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Create UBIFS object and print info. - ubifs_obj = ubifs(lebv_file) - print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) - try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) - try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') - - else: - print('Something went wrong to get here.') - - ufile_obj.close() - - -if __name__=='__main__': - main() diff --git a/python/ubireader/scripts/ubireader_extract_files.py b/python/ubireader/scripts/ubireader_extract_files.py deleted file mode 100644 index a18a214..0000000 --- a/python/ubireader/scripts/ubireader_extract_files.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.output import extract_files -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - -def main(): - start = time.time() - description = 'Extract contents of a UBI or UBIFS image.' - usage = 'ubireader_extract_files [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', - help='Maintain file permissions, requires running as root. (default: False)') - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.outpath: - outpath = args.outpath - else: - outpath = settings.output_dir - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - perms = args.permissions - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - - # Create path for specific image - # In case multiple images in data - img_outpath = os.path.join(outpath, '%s' % image.image_seq) - - # Loop through volumes in each image. - for volume in image.volumes: - - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Create volume data output path. - vol_outpath = os.path.join(img_outpath, volume) - - # Create volume output path directory. - create_output_dir(vol_outpath) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Extract files from UBI image. - ubifs_obj = ubifs(lebv_file) - print('Extracting files to: %s' % vol_outpath) - extract_files(ubifs_obj, vol_outpath, perms) - - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - - # Create directory for files. - create_output_dir(outpath) - - # Extract files from UBIFS image. - print('Extracting files to: %s' % outpath) - extract_files(ubifs_obj, outpath, perms) - - else: - print('Something went wrong to get here.') - - ufile_obj.close() - - -if __name__=='__main__': - main() diff --git a/python/ubireader/scripts/ubireader_extract_images.py b/python/ubireader/scripts/ubireader_extract_images.py deleted file mode 100644 index 3d8cf2e..0000000 --- a/python/ubireader/scripts/ubireader_extract_images.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubi_io import ubi_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size - -def create_output_dir(outpath): - if not os.path.exists(outpath): - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - -def main(): - start = time.time() - description = 'Extract UBI or UBIFS images from file containing UBI data in it.' - usage = 'ubireader_extract_images [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - parser.add_argument('-u', '--image-type', dest='image_type', - help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') - - img_name = os.path.basename(path) - if args.outpath: - outpath = os.path.abspath(os.path.join(args.outpath, img_name)) - else: - outpath = os.path.join(settings.output_dir, img_name) - - if args.block_size: - block_size = args.block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - if args.image_type: - image_type = args.image_type.upper() - else: - image_type = 'UBIFS' - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - if image_type == 'UBI': - # Create output path and open file. - img_outpath = os.path.join(outpath, 'img-%s.ubi' % image.image_seq) - create_output_dir(outpath) - f = open(img_outpath, 'wb') - - # Loop through UBI image blocks - for block in image.get_blocks(ubi_obj.blocks): - if ubi_obj.blocks[block].is_valid: - # Write block (PEB) to file - f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) - - elif image_type == 'UBIFS': - # Loop through image volumes - for volume in image.volumes: - # Create output path and open file. - vol_outpath = os.path.join(outpath, 'img-%s_vol-%s.ubifs' % (image.image_seq, volume)) - create_output_dir(outpath) - f = open(vol_outpath, 'wb') - - # Loop through and write volume block data (LEB) to file. - for block in image.volumes[volume].reader(ubi_obj): - f.write(block) - - ufile_obj.close() - - -if __name__=='__main__': - main() diff --git a/python/ubireader/scripts/ubireader_list_files.py b/python/ubireader/scripts/ubireader_list_files.py deleted file mode 100644 index 1ac00be..0000000 --- a/python/ubireader/scripts/ubireader_list_files.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (C) Collin Mulliner based on Jason Pruitt's ubireader_extract_images -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.list import list_files, copy_file -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -def main(): - start = time.time() - description = 'List and Extract files of a UBI or UBIFS image.' - usage = 'ubireader_list_files [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-P', '--path', dest='listpath', - help='Path to list.') - - parser.add_argument('-C', '--copy', dest='copyfile', - help='File to Copy.') - - parser.add_argument('-D', '--copy-dest', dest='copyfiledest', - help='Copy Destination.') - - parser.add_argument('filepath', help='UBI/UBIFS image file.') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - - # Loop through volumes in each image. - for volume in image.volumes: - - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Create UBIFS object. - ubifs_obj = ubifs(lebv_file) - - if args.listpath: - list_files(ubifs_obj, args.listpath) - if args.copyfile and args.copyfiledest: - copy_file(ubifs_obj, args.copyfile, args.copyfiledest) - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - - if args.listpath: - list_files(ubifs_obj, args.listpath) - if args.copyfile and args.copyfiledest: - copy_file(ubifs_obj, args.copyfile, args.copyfiledest) - - else: - print('Something went wrong to get here.') - - ufile_obj.close() - - -if __name__=='__main__': - main() diff --git a/python/ubireader/scripts/ubireader_utils_info.py b/python/ubireader/scripts/ubireader_utils_info.py deleted file mode 100644 index b8fafe5..0000000 --- a/python/ubireader/scripts/ubireader_utils_info.py +++ /dev/null @@ -1,350 +0,0 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import sys -import time -import argparse -if (sys.version_info > (3, 0)): - import configparser -else: - import ConfigParser as configparser -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG -from ubireader.ubifs import ubifs -from ubireader.ubifs.defines import PRINT_UBIFS_KEY_HASH, PRINT_UBIFS_COMPR -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - - -def get_ubi_params(ubi_obj): - """Get ubi_obj utils params - - Arguments: - Obj:ubi -- UBI object. - - Returns: - Dict -- Dict keyed to volume with Dict of args and flags. - """ - ubi_flags = {'min_io_size':'-m', - 'max_bud_bytes':'-j', - 'leb_size':'-e', - 'default_compr':'-x', - 'sub_page_size':'-s', - 'fanout':'-f', - 'key_hash':'-k', - 'orph_lebs':'-p', - 'log_lebs':'-l', - 'max_leb_cnt': '-c', - 'peb_size':'-p', - 'sub_page_size':'-s', - 'vid_hdr_offset':'-O', - 'version':'-x', - 'image_seq':'-Q', - 'alignment':'-a', - 'vol_id':'-n', - 'name':'-N'} - - ubi_params = {} - ubi_args = {} - ini_params = {} - - for image in ubi_obj.images: - img_seq = image.image_seq - ubi_params[img_seq] = {} - ubi_args[img_seq] = {} - ini_params[img_seq] = {} - - for volume in image.volumes: - ubi_args[img_seq][volume] = {} - ini_params[img_seq][volume] = {} - - # Get ubinize.ini settings - ini_params[img_seq][volume]['vol_type'] = PRINT_VOL_TYPE_LIST[image.volumes[volume].vol_rec.vol_type] - - if image.volumes[volume].vol_rec.flags == UBI_VTBL_AUTORESIZE_FLG: - ini_params[img_seq][volume]['vol_flags'] = 'autoresize' - else: - ini_params[img_seq][volume]['vol_flags'] = image.volumes[volume].vol_rec.flags - - ini_params[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - ini_params[img_seq][volume]['vol_name'] = image.volumes[volume].name.rstrip(b'\x00').decode('utf-8') - ini_params[img_seq][volume]['vol_alignment'] = image.volumes[volume].vol_rec.alignment - - ini_params[img_seq][volume]['vol_size'] = image.volumes[volume].vol_rec.reserved_pebs * ubi_obj.leb_size - - # Create file object backed by UBI blocks. - lebv_file = leb_virtual_file(ubi_obj, image.volumes[volume].get_blocks(ubi_obj.blocks)) - # Create UBIFS object - ubifs_obj = ubifs(lebv_file) - - for key, value in ubifs_obj.superblock_node: - if key == 'key_hash': - value = PRINT_UBIFS_KEY_HASH[value] - elif key == 'default_compr': - value = PRINT_UBIFS_COMPR[value] - - if key in ubi_flags: - ubi_args[img_seq][volume][key] = value - - for key, value in image.volumes[volume].vol_rec: - if key == 'name': - value = value.rstrip(b'\x00').decode('utf-8') - - if key in ubi_flags: - ubi_args[img_seq][volume][key] = value - - ubi_args[img_seq][volume]['version'] = image.version - ubi_args[img_seq][volume]['vid_hdr_offset'] = image.vid_hdr_offset - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['image_seq'] = image.image_seq - ubi_args[img_seq][volume]['peb_size'] = ubi_obj.peb_size - ubi_args[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - - ubi_params[img_seq][volume] = {'flags':ubi_flags, 'args':ubi_args[img_seq][volume], 'ini':ini_params[img_seq][volume]} - - return ubi_params - - -def print_ubi_params(ubi_obj): - ubi_params = get_ubi_params(ubi_obj) - for img_params in ubi_params: - for volume in ubi_params[img_params]: - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - ini_params = ubi_params[img_params][volume]['ini'] - sorted_keys = sorted(ubi_params[img_params][volume]['args']) - - print('\nVolume %s' % volume) - for key in sorted_keys: - if len(key)< 8: - name = '%s\t' % key - else: - name = key - print('\t%s\t%s %s' % (name, ubi_flags[key], ubi_args[key])) - - print('\n\t#ubinize.ini#') - print('\t[%s]' % ini_params['vol_name']) - for key in ini_params: - if key != 'name': - print('\t%s=%s' % (key, ini_params[key])) - - -def make_files(ubi, outpath): - ubi_params = get_ubi_params(ubi) - - for img_params in ubi_params: - config = configparser.ConfigParser() - img_outpath = os.path.join(outpath, 'img-%s' % img_params) - - if not os.path.exists(img_outpath): - os.mkdir(img_outpath) - - ini_path = os.path.join(img_outpath, 'img-%s.ini' % img_params) - ubi_file = os.path.join('img-%s.ubi' % img_params) - script_path = os.path.join(img_outpath, 'create_ubi_img-%s.sh' % img_params) - ubifs_files =[] - buf = '#!/bin/sh\n' - print('Writing to: %s' % script_path) - - with open(script_path, 'w') as fscr: - with open(ini_path, 'w') as fini: - print('Writing to: %s' % ini_path) - vol_idx = 0 - - for volume in ubi_params[img_params]: - ubifs_files.append(os.path.join('img-%s_%s.ubifs' % (img_params, vol_idx))) - ini_params = ubi_params[img_params][volume]['ini'] - ini_file = 'img-%s.ini' % img_params - config.add_section(volume) - config.set(volume, 'mode', 'ubi') - config.set(volume, 'image', ubifs_files[vol_idx]) - - for i in ini_params: - config.set(volume, i, str(ini_params[i])) - - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - mkfs_flags = ['min_io_size', - 'leb_size', - 'max_leb_cnt', - 'default_compr', - 'fanout', - 'key_hash', - 'orph_lebs', - 'log_lebs'] - - argstr = '' - for flag in mkfs_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) - - #leb = '%s %s' % (ubi_flags['leb_size'], ubi_args['leb_size']) - peb = '%s %s' % (ubi_flags['peb_size'], ubi_args['peb_size']) - min_io = '%s %s' % (ubi_flags['min_io_size'], ubi_args['min_io_size']) - #leb_cnt = '%s %s' % (ubi_flags['max_leb_cnt'], ubi_args['max_leb_cnt']) - vid_hdr = '%s %s' % (ubi_flags['vid_hdr_offset'], ubi_args['vid_hdr_offset']) - sub_page = '%s %s' % (ubi_flags['sub_page_size'], ubi_args['sub_page_size']) - - buf += '/usr/sbin/mkfs.ubifs%s -r $%s %s\n' % (argstr, (vol_idx+1), ubifs_files[vol_idx]) - - vol_idx += 1 - - config.write(fini) - - ubinize_flags = ['peb_size', - 'min_io_size', - 'vid_hdr_offset', - 'sub_page_size', - 'version', - 'image_seq'] - - argstr = '' - for flag in ubinize_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) - - buf += '/usr/sbin/ubinize%s -o %s %s\n' % (argstr, ubi_file, ini_file) - fscr.write(buf) - os.chmod(script_path, 0o755) - -def main(): - start = time.time() - description = 'Determine settings for recreating UBI image.' - usage = 'ubireader_utils_info [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-r', '--show-only', action='store_true', dest='show_only', - help='Print parameters to screen only. (default: false)') - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') - - img_name = os.path.basename(path) - if args.outpath: - outpath = os.path.abspath(os.path.join(args.outpath, img_name)) - else: - outpath = os.path.join(settings.output_dir, img_name) - - if args.block_size: - block_size = args.block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Print info. - print_ubi_params(ubi_obj) - - if not args.show_only: - create_output_dir(outpath) - # Create build scripts. - make_files(ubi_obj, outpath) - - ufile_obj.close() - - -if __name__=='__main__': - main() diff --git a/python/ubireader/settings.py b/python/ubireader/settings.py deleted file mode 100644 index 3ad9726..0000000 --- a/python/ubireader/settings.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -output_dir = 'ubifs-root' - -error_action = True # if 'exit' on any error exit program. -fatal_traceback = False # Print traceback on fatal errors. - -ignore_block_header_errors = False # Ignore block errors. -warn_only_block_read_errors = False # Warning instead of Fatal error. - -logging_on = False # Print debug info on. -logging_on_verbose = False # Print verbose debug info on. - -use_dummy_socket_file = False # Create regular file place holder for sockets. -use_dummy_devices = False # Create regular file place holder for devices. - -uboot_fix = False # Older u-boot sets image_seq to 0 on blocks it's written to. diff --git a/python/ubireader/ubi/__init__.py b/python/ubireader/ubi/__init__.py deleted file mode 100644 index cf2cb88..0000000 --- a/python/ubireader/ubi/__init__.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader.debug import error -from ubireader.ubi.block import sort, extract_blocks -from ubireader.ubi import display -from ubireader.ubi.image import description as image -from ubireader.ubi.block import layout, rm_old_blocks - -class ubi_base(object): - """UBI Base object - - Arguments: - Obj:image -- UBI image object - - Attributes: - ubi_file:file -- ubi_file object - Int:block_count -- Number of blocks found. - Int:first_peb_num -- Number of the first UBI PEB in file. - Int:leb_size -- Size of Logical Erase Blocks. - Int:peb_size -- Size of Physical Erase Blocks. - Int:min_io -- Size of min I/O from vid_hdr_offset. - Dict:blocks -- Dict keyed by PEB number of all blocks. - """ - - def __init__(self, ubi_file): - self.__name__ = 'UBI' - self._file = ubi_file - self._first_peb_num = 0 - self._blocks = extract_blocks(self) - self._block_count = len(self.blocks) - - if self._block_count <= 0: - error(self, 'Fatal', 'No blocks found.') - - arbitrary_block = next(iter(self.blocks.values())) - self._min_io_size = arbitrary_block.ec_hdr.vid_hdr_offset - self._leb_size = self.file.block_size - arbitrary_block.ec_hdr.data_offset - - - def _get_file(self): - """UBI File object - - Returns: - Obj -- UBI File object. - """ - return self._file - file = property(_get_file) - - - def _get_block_count(self): - """Total amount of UBI blocks in file. - - Returns: - Int -- Number of blocks - """ - return self._block_count - block_count = property(_get_block_count) - - - def _set_first_peb_num(self, i): - self._first_peb_num = i - def _get_first_peb_num(self): - """First Physical Erase Block with UBI data - - Returns: - Int -- Number of the first PEB. - """ - return self._first_peb_num - first_peb_num = property(_get_first_peb_num, _set_first_peb_num) - - - def _get_leb_size(self): - """LEB size of UBI blocks in file. - - Returns: - Int -- LEB Size. - """ - return self._leb_size - leb_size = property(_get_leb_size) - - - def _get_peb_size(self): - """PEB size of UBI blocks in file. - - Returns: - Int -- PEB Size. - """ - return self.file.block_size - peb_size = property(_get_peb_size) - - - def _get_min_io_size(self): - """Min I/O Size - - Returns: - Int -- Min I/O Size. - """ - return self._min_io_size - min_io_size = property(_get_min_io_size) - - - def _get_blocks(self): - """Main Dict of UBI Blocks - - Passed around for lists of indexes to be made or to be returned - filtered through a list. So there isn't multiple copies of blocks, - as there can be thousands. - """ - return self._blocks - blocks = property(_get_blocks) - - -class ubi(ubi_base): - """UBI object - - Arguments: - Obj:ubi_file -- UBI file object. - - Attributes: - Inherits:ubi_base -- ubi_base attributes. - List:images -- List of UBI image objects. - List:data_blocks_list -- List of all data blocks in file. - List:layout_blocks_list -- List of all layout blocks in file. - List:int_vol_blocks_list -- List of internal volumes minus layout. - List:unknown_blocks_list -- List of blocks with unknown types. * - """ - - def __init__(self, ubi_file): - super(ubi, self).__init__(ubi_file) - - layout_list, data_list, int_vol_list, unknown_list = sort.by_type(self.blocks) - - self._layout_blocks_list = layout_list - self._data_blocks_list = data_list - self._int_vol_blocks_list = int_vol_list - self._unknown_blocks_list = unknown_list - - newest_layout_list = rm_old_blocks(self.blocks, self.layout_blocks_list) - - if len(newest_layout_list) < 2: - error(self, 'Fatal', 'Less than 2 layout blocks found.') - - layout_pairs = layout.group_pairs(self.blocks, newest_layout_list) - - layout_infos = layout.associate_blocks(self.blocks, layout_pairs) - - self._images = [] - for i in range(0, len(layout_infos)): - self._images.append(image(self.blocks, layout_infos[i])) - - - def _get_images(self): - """Get UBI images. - - Returns: - List -- Of volume objects groupled by image. - """ - return self._images - images = property(_get_images) - - - def _get_data_blocks_list(self): - """Get all UBI blocks found in file that are data blocks. - - Returns: - List -- List of block objects. - """ - return self._data_blocks_list - data_blocks_list = property(_get_data_blocks_list) - - - def _get_layout_blocks_list(self): - """Get all UBI blocks found in file that are layout volume blocks. - - Returns: - List -- List of block objects. - """ - return self._layout_blocks_list - layout_blocks_list = property(_get_layout_blocks_list) - - - def _get_int_vol_blocks_list(self): - """Get all UBI blocks found in file that are internal volume blocks. - - Returns: - List -- List of block objects. - - This does not include layout blocks. - """ - return self._int_vol_blocks_list - int_vol_blocks_list = property(_get_int_vol_blocks_list) - - - def _get_unknown_blocks_list(self): - """Get all UBI blocks found in file of unknown type.. - - Returns: - List -- List of block objects. - """ - return self._unknown_blocks_list - unknown_blocks_list = property(_get_unknown_blocks_list) - - def display(self, tab=''): - """Print information about this object. - - Argument: - Str:tab -- '\t' for spacing this object. - """ - return display.ubi(self, tab) diff --git a/python/ubireader/ubi/block/__init__.py b/python/ubireader/ubi/block/__init__.py deleted file mode 100644 index ce80f96..0000000 --- a/python/ubireader/ubi/block/__init__.py +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from zlib import crc32 -from ubireader import settings -from ubireader.debug import error, log, verbose_display, verbose_log -from ubireader.ubi import display -from ubireader.ubi.defines import UBI_EC_HDR_SZ, UBI_VID_HDR_SZ, UBI_INTERNAL_VOL_START, UBI_EC_HDR_MAGIC, UBI_CRC32_INIT -from ubireader.ubi.headers import ec_hdr, vid_hdr, vtbl_recs - - -class description(object): - """UBI Block description Object - - UBI Specifications: - http://www.linux-mtd.infradead.org/ -- Home page - /drivers/mtd/ubi/ubi-media.h -- Header structs - and defines - - Attributes: - Obj:ec_hdr -- Error Count Header - Obj:vid_hdr -- Volume ID Header - List:vtbl_recs -- (Optional) List of Volume Table Records. - Bool:is_vtbl -- If contains volume records table. - Bool:is_internal_vol -- If Vol ID is > UBI_INTERNAL_VOL_START - Bool:is_valid -- If ec_hdr & vid_hdr are error free. - Int:peb_num -- Physical Erase Block number. - Int:leb_num -- Logical Erase Block number. - Int:file_offset -- Address location in file of this block. - Int:size -- Size of total block data or PEB size. - Int:data_crc -- crc32 of block data. - Will print out all information when invoked as a string. - """ - - def __init__(self, block_buf): - - self.file_offset = -1 - self.peb_num = -1 - self.leb_num = -1 - self.size = -1 - - self.vid_hdr = None - self.is_internal_vol = False - self.vtbl_recs = [] - - # TODO better understanding of block types/errors - self.ec_hdr = ec_hdr(block_buf[0:UBI_EC_HDR_SZ]) - - if not self.ec_hdr.errors or settings.ignore_block_header_errors: - self.vid_hdr = vid_hdr(block_buf[self.ec_hdr.vid_hdr_offset:self.ec_hdr.vid_hdr_offset+UBI_VID_HDR_SZ]) - - if not self.vid_hdr.errors or settings.ignore_block_header_errors: - self.is_internal_vol = self.vid_hdr.vol_id >= UBI_INTERNAL_VOL_START - - if self.vid_hdr.vol_id >= UBI_INTERNAL_VOL_START: - self.vtbl_recs = vtbl_recs(block_buf[self.ec_hdr.data_offset:]) - - self.leb_num = self.vid_hdr.lnum - - self.is_vtbl = bool(self.vtbl_recs) or False - self.is_valid = not self.ec_hdr.errors and not self.vid_hdr.errors or settings.ignore_block_header_errors - - - def __repr__(self): - return 'Block: PEB# %s: LEB# %s' % (self.peb_num, self.leb_num) - - - def display(self, tab=''): - return display.block(self, tab) - - - -def get_blocks_in_list(blocks, idx_list): - """Retrieve block objects in list of indexes - - Arguments: - List:blocks -- List of block objects - List:idx_list -- List of block indexes - - Returns: - Dict:blocks -- List of block objects generated - from provided list of indexes in - order of idx_list. - """ - - return {i:blocks[i] for i in idx_list} - - - -def extract_blocks(ubi): - """Get a list of UBI block objects from file - - Arguments:. - Obj:ubi -- UBI object. - - Returns: - Dict -- Of block objects keyed by PEB number. - """ - - blocks = {} - ubi.file.seek(ubi.file.start_offset) - peb_count = 0 - cur_offset = 0 - bad_blocks = [] - - # range instead of xrange, as xrange breaks > 4GB end_offset. - for i in range(ubi.file.start_offset, ubi.file.end_offset, ubi.file.block_size): - buf = ubi.file.read(ubi.file.block_size) - - if buf.startswith(UBI_EC_HDR_MAGIC): - blk = description(buf) - blk.file_offset = i - blk.peb_num = ubi.first_peb_num + peb_count - blk.size = ubi.file.block_size - blk.data_crc = (~crc32(buf[blk.ec_hdr.data_offset:blk.ec_hdr.data_offset+blk.vid_hdr.data_size]) & UBI_CRC32_INIT) - blocks[blk.peb_num] = blk - peb_count += 1 - log(extract_blocks, blk) - verbose_log(extract_blocks, 'file addr: %s' % (ubi.file.last_read_addr())) - ec_hdr_errors = '' - vid_hdr_errors = '' - - if blk.ec_hdr.errors: - ec_hdr_errors = ','.join(blk.ec_hdr.errors) - - if blk.vid_hdr and blk.vid_hdr.errors: - vid_hdr_errors = ','.join(blk.vid_hdr.errors) - - if ec_hdr_errors or vid_hdr_errors: - if blk.peb_num not in bad_blocks: - bad_blocks.append(blk.peb_num) - log(extract_blocks, 'PEB: %s has possible issue EC_HDR [%s], VID_HDR [%s]' % (blk.peb_num, ec_hdr_errors, vid_hdr_errors)) - - verbose_display(blk) - - else: - cur_offset += ubi.file.block_size - ubi.first_peb_num = cur_offset//ubi.file.block_size - ubi.file.start_offset = cur_offset - - return blocks - - -def rm_old_blocks(blocks, block_list): - del_blocks = [] - - for i in block_list: - if i in del_blocks: - continue - - if blocks[i].is_valid is not True: - del_blocks.append(i) - continue - - for k in block_list: - if i == k: - continue - - if k in del_blocks: - continue - - if blocks[k].is_valid is not True: - del_blocks.append(k) - continue - - if blocks[i].leb_num != blocks[k].leb_num: - continue - - if blocks[i].ec_hdr.image_seq != blocks[k].ec_hdr.image_seq: - continue - - second_newer = blocks[k].vid_hdr.sqnum > blocks[i].vid_hdr.sqnum - del_block = None - use_block = None - - if second_newer: - if blocks[k].vid_hdr.copy_flag == 0: - del_block = i - use_block = k - - else: - if blocks[i].vid_hdr.copy_flag == 0: - del_block = k - use_block = i - - if del_block is not None: - del_blocks.append(del_block) - log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s, Using PEB%s' % (blocks[del_block].peb_num, blocks[del_block].leb_num, use_block)) - break - - if second_newer: - if blocks[k].data_crc != blocks[k].vid_hdr.data_crc: - del_block = k - use_block = i - else: - del_block = i - use_block = k - else: - if blocks[i].data_crc != blocks[i].vid_hdr.data_crc: - del_block = i - use_block = k - else: - del_block = k - use_block = i - - if del_block is not None: - del_blocks.append(del_block) - log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s, vid_hdr.data_crc %s / %s, Using PEB %s' % (blocks[del_block].peb_num, - blocks[del_block].leb_num, - blocks[del_block].vid_hdr.data_crc, - blocks[del_block].data_crc, - use_block)) - - else: - use_block = min(k, i) - del_blocks.append(use_block) - error('Warn', rm_old_blocks, 'Multiple PEB [%s] for LEB %s: Using first.' % (', '.join(i, k), blocks[i].leb_num, use_block)) - - break - - return [j for j in block_list if j not in del_blocks] diff --git a/python/ubireader/ubi/block/layout.py b/python/ubireader/ubi/block/layout.py deleted file mode 100644 index 67388d9..0000000 --- a/python/ubireader/ubi/block/layout.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader.debug import log -from ubireader.ubi.block import sort - -def group_pairs(blocks, layout_blocks_list): - """Sort a list of layout blocks into pairs - - Arguments: - List:blocks -- List of block objects - List:layout_blocks -- List of layout block indexes - - Returns: - List -- Layout block pair indexes grouped in a list - """ - - image_dict={} - for block_id in layout_blocks_list: - image_seq=blocks[block_id].ec_hdr.image_seq - if image_seq not in image_dict: - image_dict[image_seq]=[block_id] - else: - image_dict[image_seq].append(block_id) - - log(group_pairs, 'Layout blocks found at PEBs: %s' % list(image_dict.values())) - - return list(image_dict.values()) - - -def associate_blocks(blocks, layout_pairs): - """Group block indexes with appropriate layout pairs - - Arguments: - List:blocks -- List of block objects - List:layout_pairs -- List of grouped layout blocks - - Returns: - List -- Layout block pairs grouped with associated block ranges. - """ - - seq_blocks = [] - for layout_pair in layout_pairs: - seq_blocks = sort.by_image_seq(blocks, blocks[layout_pair[0]].ec_hdr.image_seq) - seq_blocks = [b for b in seq_blocks if b not in layout_pair] - layout_pair.append(seq_blocks) - - return layout_pairs diff --git a/python/ubireader/ubi/block/sort.py b/python/ubireader/ubi/block/sort.py deleted file mode 100644 index e9e115b..0000000 --- a/python/ubireader/ubi/block/sort.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader import settings - -def by_image_seq(blocks, image_seq): - """Filter blocks to return only those associated with the provided image_seq number. - If uboot_fix is set, associate blocks with an image_seq of 0 also. - - Argument: - List:blocks -- List of block objects to sort. - Int:image_seq -- image_seq number found in ec_hdr. - - Returns: - List -- List of block indexes matching image_seq number. - """ - if settings.uboot_fix: - return list(filter(lambda block: blocks[block].ec_hdr.image_seq == image_seq or image_seq == 0 or blocks[block].ec_hdr.image_seq == 0, blocks)) - - else: - return list(filter(lambda block: blocks[block].ec_hdr.image_seq == image_seq, blocks)) - -def by_leb(blocks): - """Sort blocks by Logical Erase Block number. - - Arguments: - List:blocks -- List of block objects to sort. - - Returns: - List -- Indexes of blocks sorted by LEB. - """ - slist_len = len(blocks) - slist = ['x'] * slist_len - - for block in blocks: - if blocks[block].leb_num >= slist_len: - add_elements = blocks[block].leb_num - slist_len + 1 - slist += (['x'] * add_elements) - slist_len = len(slist) - - slist[blocks[block].leb_num] = block - - return slist - - -def by_vol_id(blocks, slist=None): - """Sort blocks by volume id - - Arguments: - Obj:blocks -- List of block objects. - List:slist -- (optional) List of block indexes. - - Return: - Dict -- blocks grouped in lists with dict key as volume id. - """ - - vol_blocks = {} - - # sort block by volume - # not reliable with multiple partitions (fifo) - - for i in blocks: - if slist and i not in slist: - continue - elif not blocks[i].is_valid: - continue - - if blocks[i].vid_hdr.vol_id not in vol_blocks: - vol_blocks[blocks[i].vid_hdr.vol_id] = [] - - vol_blocks[blocks[i].vid_hdr.vol_id].append(blocks[i].peb_num) - - return vol_blocks - -def by_type(blocks, slist=None): - """Sort blocks into layout, internal volume, data or unknown - - Arguments: - Obj:blocks -- List of block objects. - List:slist -- (optional) List of block indexes. - - Returns: - List:layout -- List of block indexes of blocks containing the - volume table records. - List:data -- List of block indexes containing filesystem data. - List:int_vol -- List of block indexes containing volume ids - greater than UBI_INTERNAL_VOL_START that are not - layout volumes. - List:unknown -- List of block indexes of blocks that failed validation - of crc in ed_hdr or vid_hdr. - """ - - layout = [] - data = [] - int_vol = [] - unknown = [] - - for i in blocks: - if slist and i not in slist: - continue - - if blocks[i].is_vtbl and blocks[i].is_valid: - layout.append(i) - - elif blocks[i].is_internal_vol and blocks[i].is_valid: - int_vol.append(i) - - elif blocks[i].is_valid: - data.append(i) - - else: - unknown.append(i) - - return layout, data, int_vol, unknown diff --git a/python/ubireader/ubi/defines.py b/python/ubireader/ubi/defines.py deleted file mode 100644 index 9a26bc0..0000000 --- a/python/ubireader/ubi/defines.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -############################################################# -# Adapted in part from linux-source-3.2/drivers/mtd/ubi/ubi-media.h -# for use in Python. -# Oct. 2013 by Jason Pruitt -# -# Original copyright notice. -# -------------------------- -# -# Copyright (c) International Business Machines Corp., 2006 -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See -# the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# Authors: Artem Bityutskiy (Битюцкий Артём) -# Thomas Gleixner -# Frank Haverkamp -# Oliver Lohmann -# Andreas Arnez -# -############################################################# - -import struct - -# Magic number -UBI_EC_HDR_MAGIC = b'\x55\x42\x49\x23' - -# Initial CRC32 checksum value. -UBI_CRC32_INIT = 4294967295 #0xFFFFFFFF - -# Max number of volumes allowed. -UBI_MAX_VOLUMES = 128 - -# Internal Volume ID start. -UBI_INTERNAL_VOL_START = 2147479551 - -# Error Count header. -EC_HDR_FORMAT = '>4sB3sQIII32sI' -EC_HDR_FIELDS = ['magic', # Magic string UBI# - 'version', # UBI version meant to accept this image. - 'padding', # Reserved for future, zeros. - 'ec', # Erase counter - 'vid_hdr_offset', # Where the VID header starts. - 'data_offset', # Where user data starts. - 'image_seq', # Image sequence number - 'padding2', # Reserved for future, zeros. - 'hdr_crc'] # EC header crc32 checksum. - - -UBI_EC_HDR_SZ = struct.calcsize(EC_HDR_FORMAT) # 64 - -# Volume ID header. -UBI_VID_HDR_MAGIC =b'\x55\x42\x49\x21' # UBI! -VID_HDR_FORMAT = '>4sBBBBII4sIIII4sQ12sI' -VID_HDR_FIELDS = ['magic', # Magic string UBI! - 'version', # UBI version meant to accept this image. - 'vol_type', # Volume type, Dynamic/Static - 'copy_flag', # If this is a copied PEB b/c of wear leveling. - 'compat', # Compatibility of this volume UBI_COMPAT_* - 'vol_id', # ID of this volume. - 'lnum', # LEB number. - 'padding', # Reserved for future, zeros. - 'data_size', # How many bytes of data this contains. - # Used for static types only. - 'used_ebs', # Total num of used LEBs in this volume. - 'data_pad', # How many bytes at end of LEB are not used. - 'data_crc', # CRC32 checksum of data, static type only. - 'padding2', # Reserved for future, zeros. - 'sqnum', # Sequence number. - 'padding3', # Reserved for future, zeros. - 'hdr_crc'] # VID header CRC32 checksum. - - -UBI_VID_HDR_SZ = struct.calcsize(VID_HDR_FORMAT) # 64 - -# Volume table records. -VTBL_REC_FORMAT = '>IIIBBH128sB23sI' -VTBL_REC_FIELDS = ['reserved_pebs', # How many PEBs reserved for this volume. - 'alignment', # Volume alignment. - 'data_pad', # Number of unused bytes at end of PEB. - 'vol_type', # Volume type, static/dynamic. - 'upd_marker', # If vol update started but not finished. - 'name_len', # Length of name. - 'name', # Volume name. - 'flags', # Volume flags - 'padding', # Reserved for future, zeros. - 'crc'] # Vol record CRC32 checksum. - - -UBI_VTBL_REC_SZ = struct.calcsize(VTBL_REC_FORMAT) # 172 - -# Volume Identifier Header -UBI_VID_DYNAMIC = 1 # Volume can be resized. -UBI_VID_STATIC = 2 # Volume can not be resized. -PRINT_VOL_TYPE_LIST = [0, 'dynamic', 'static'] - -# Volume table record -UBI_VTBL_AUTORESIZE_FLG = 1 - -UBI_COMPAT_DELETE = 1 # Delete this internal volume before anything written. -UBI_COMPAT_RO = 2 # Attach this device in read-only mode. -UBI_COMPAT_PRESERVE = 4 # Preserve this internal volume - touch nothing. -UBI_COMPAT_REJECT = 5 # Reject this UBI image -PRINT_COMPAT_LIST = [0, 'Delete', 'Read Only', 0, 'Preserve', 'Reject'] - -# File chunk size for reads. -FILE_CHUNK_SZ = 5 * 1024 * 1024 \ No newline at end of file diff --git a/python/ubireader/ubi/display.py b/python/ubireader/ubi/display.py deleted file mode 100644 index 9efbc6a..0000000 --- a/python/ubireader/ubi/display.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader import settings -from ubireader.ubi.defines import PRINT_COMPAT_LIST, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG - -def ubi(ubi, tab=''): - buf = '%sUBI File\n' % (tab) - buf += '%s---------------------\n' % (tab) - buf += '\t%sMin I/O: %s\n' % (tab, ubi.min_io_size) - buf += '\t%sLEB Size: %s\n' % (tab, ubi.leb_size) - buf += '\t%sPEB Size: %s\n' % (tab, ubi.peb_size) - buf += '\t%sTotal Block Count: %s\n' % (tab, ubi.block_count) - buf += '\t%sData Block Count: %s\n' % (tab, len(ubi.data_blocks_list)) - buf += '\t%sLayout Block Count: %s\n' % (tab, len(ubi.layout_blocks_list)) - buf += '\t%sInternal Volume Block Count: %s\n' % (tab, len(ubi.int_vol_blocks_list)) - buf += '\t%sUnknown Block Count: %s\n' % (tab, len(ubi.unknown_blocks_list)) - buf += '\t%sFirst UBI PEB Number: %s\n' % (tab, ubi.first_peb_num) - return buf - - -def image(image, tab=''): - buf = '%s%s\n' % (tab, image) - buf += '%s---------------------\n' % (tab) - buf += '\t%sImage Sequence Num: %s\n' % (tab, image.image_seq) - for volume in image.volumes: - buf += '\t%sVolume Name:%s\n' % (tab, volume) - buf += '\t%sPEB Range: %s - %s\n' % (tab, image.peb_range[0], image.peb_range[1]) - return buf - - -def volume(volume, tab=''): - buf = '%s%s\n' % (tab, volume) - buf += '%s---------------------\n' % (tab) - buf += '\t%sVol ID: %s\n' % (tab, volume.vol_id) - buf += '\t%sName: %s\n' % (tab, volume.name.decode('utf-8')) - buf += '\t%sBlock Count: %s\n' % (tab, volume.block_count) - - buf += '\n' - buf += '\t%sVolume Record\n' % (tab) - buf += '\t%s---------------------\n' % (tab) - buf += vol_rec(volume.vol_rec, '\t\t%s' % tab) - - buf += '\n' - return buf - - -def block(block, tab='\t'): - buf = '%s%s\n' % (tab, block) - buf += '%s---------------------\n' % (tab) - buf += '\t%sFile Offset: %s\n' % (tab, block.file_offset) - buf += '\t%sPEB #: %s\n' % (tab, block.peb_num) - buf += '\t%sLEB #: %s\n' % (tab, block.leb_num) - buf += '\t%sBlock Size: %s\n' % (tab, block.size) - buf += '\t%sInternal Volume: %s\n' % (tab, block.is_internal_vol) - buf += '\t%sIs Volume Table: %s\n' % (tab, block.is_vtbl) - buf += '\t%sIs Valid: %s\n' % (tab, block.is_valid) - - if not block.ec_hdr.errors or settings.ignore_block_header_errors: - buf += '\n' - buf += '\t%sErase Count Header\n' % (tab) - buf += '\t%s---------------------\n' % (tab) - buf += ec_hdr(block.ec_hdr, '\t\t%s' % tab) - - if (block.vid_hdr and not block.vid_hdr.errors) or settings.ignore_block_header_errors: - buf += '\n' - buf += '\t%sVID Header\n' % (tab) - buf += '\t%s---------------------\n' % (tab) - buf += vid_hdr(block.vid_hdr, '\t\t%s' % tab) - - if block.vtbl_recs: - buf += '\n' - buf += '\t%sVolume Records\n' % (tab) - buf += '\t%s---------------------\n' % (tab) - for vol in block.vtbl_recs: - buf += vol_rec(vol, '\t\t%s' % tab) - - buf += '\n' - return buf - - -def ec_hdr(ec_hdr, tab=''): - buf = '' - for key, value in ec_hdr: - if key == 'errors': - value = ','.join(value) - - elif key == 'hdr_crc': - value = hex(value) - - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def vid_hdr(vid_hdr, tab=''): - buf = '' - for key, value in vid_hdr: - if key == 'errors': - value = ','.join(value) - - elif key == 'hdr_crc': - value = hex(value) - - elif key == 'compat': - if value in PRINT_COMPAT_LIST: - value = PRINT_COMPAT_LIST[value] - else: - value = -1 - - elif key == 'vol_type': - if value < len(PRINT_VOL_TYPE_LIST): - value = PRINT_VOL_TYPE_LIST[value] - else: - value = -1 - - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def vol_rec(vol_rec, tab=''): - buf = '' - for key, value in vol_rec: - if key == 'errors': - value = ','.join(value) - - elif key == 'crc': - value = hex(value) - - elif key == 'vol_type': - if value < len(PRINT_VOL_TYPE_LIST): - value = PRINT_VOL_TYPE_LIST[value] - else: - value = -1 - - elif key == 'flags' and value == UBI_VTBL_AUTORESIZE_FLG: - value = 'autoresize' - - elif key == 'name': - value = value.strip(b'\x00').decode('utf-8') - - elif key == 'padding': - value = value.decode('utf-8') - - buf += '%s%s: %r\n' % (tab, key, value) - return buf diff --git a/python/ubireader/ubi/headers.py b/python/ubireader/ubi/headers.py deleted file mode 100644 index 5fa3905..0000000 --- a/python/ubireader/ubi/headers.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import struct -from zlib import crc32 - -from ubireader.debug import log -from ubireader.ubi.defines import * - -class ec_hdr(object): - def __init__(self, buf): - fields = dict(list(zip(EC_HDR_FIELDS, struct.unpack(EC_HDR_FORMAT,buf)))) - for key in fields: - setattr(self, key, fields[key]) - setattr(self, 'errors', []) - - self._check_errors(buf[:-4]) - - def __repr__(self): - return 'Erase Count Header' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def _check_errors(self, buf_crc): - crc_chk = (~crc32(buf_crc) & UBI_CRC32_INIT) - if self.hdr_crc != crc_chk: - log(vid_hdr, 'CRC Failed: expected 0x%x got 0x%x' % (crc_chk, self.hdr_crc)) - self.errors.append('crc') - - -class vid_hdr(object): - def __init__(self, buf): - fields = dict(list(zip(VID_HDR_FIELDS, struct.unpack(VID_HDR_FORMAT,buf)))) - for key in fields: - setattr(self, key, fields[key]) - setattr(self, 'errors', []) - - self._check_errors(buf[:-4]) - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def __repr__(self): - return 'VID Header' - - def _check_errors(self, buf_crc): - crc_chk = (~crc32(buf_crc) & UBI_CRC32_INIT) - if self.hdr_crc != crc_chk: - log(vid_hdr, 'CRC Failed: expected 0x%x got 0x%x' % (crc_chk, self.hdr_crc)) - self.errors.append('crc') - - -def vtbl_recs(buf): - data_buf = buf - vtbl_recs = [] - vtbl_rec_ret = '' - - for i in range(0, UBI_MAX_VOLUMES): - offset = i*UBI_VTBL_REC_SZ - vtbl_rec_buf = data_buf[offset:offset+UBI_VTBL_REC_SZ] - - if len(vtbl_rec_buf) == UBI_VTBL_REC_SZ: - vtbl_rec_ret = _vtbl_rec(vtbl_rec_buf) - - if len(vtbl_rec_ret.errors) == 0 and vtbl_rec_ret.name_len: - vtbl_rec_ret.rec_index = i - vtbl_recs.append(vtbl_rec_ret) - - return vtbl_recs - - -class _vtbl_rec(object): - def __init__(self, buf): - fields = dict(list(zip(VTBL_REC_FIELDS, struct.unpack(VTBL_REC_FORMAT,buf)))) - for key in fields: - setattr(self, key, fields[key]) - setattr(self, 'errors', []) - setattr(self, 'rec_index', -1) - - self.name = self.name[0: self.name_len] - self._check_errors(buf[:-4]) - - def __repr__(self): - return 'Volume Table Record: %s' % getattr(self, 'name') - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def _check_errors(self, buf_crc): - if self.crc != (~crc32(buf_crc) & 0xFFFFFFFF): - self.errors.append('crc') diff --git a/python/ubireader/ubi/image.py b/python/ubireader/ubi/image.py deleted file mode 100644 index f634b28..0000000 --- a/python/ubireader/ubi/image.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader.debug import log -from ubireader.ubi import display -from ubireader.ubi.volume import get_volumes -from ubireader.ubi.block import get_blocks_in_list - -class description(object): - def __init__(self, blocks, layout_info): - self._image_seq = blocks[layout_info[0]].ec_hdr.image_seq - self.vid_hdr_offset = blocks[layout_info[0]].ec_hdr.vid_hdr_offset - self.version = blocks[layout_info[0]].ec_hdr.version - self._block_list = layout_info[2] - self._start_peb = min(layout_info[2]) - self._end_peb = max(layout_info[2]) - self._volumes = get_volumes(blocks, layout_info) - log(description, 'Created Image: %s, Volume Cnt: %s' % (self.image_seq, len(self.volumes))) - - def __repr__(self): - return 'Image: %s' % (self.image_seq) - - - def get_blocks(self, blocks): - return get_blocks_in_list(blocks, self._block_list) - - - def _get_peb_range(self): - return [self._start_peb, self._end_peb] - peb_range = property(_get_peb_range) - - - def _get_image_seq(self): - return self._image_seq - image_seq = property(_get_image_seq) - - - def _get_volumes(self): - return self._volumes - volumes = property(_get_volumes) - - - def display(self, tab=''): - return display.image(self, tab) diff --git a/python/ubireader/ubi/volume.py b/python/ubireader/ubi/volume.py deleted file mode 100644 index f02834b..0000000 --- a/python/ubireader/ubi/volume.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader.debug import log -from ubireader.ubi import display -from ubireader.ubi.block import sort, get_blocks_in_list, rm_old_blocks - -class description(object): - """UBI Volume object - - Attributes: - Int:vol_id -- Volume ID - Str:name -- Name of volume. - Obj:vol_rec -- Volume record object - Int:block_count -- Number of block associated with volume. - - Methods: - display(tab) -- Print Volume information - Str:tab -- (optional) '\t' to preface lines with. - - get_blocks(blocks) -- Returns list of block objects tied to this volume - - Volume object is basically a list of block indexes and some metadata - describing a volume found in a UBI image. - """ - def __init__(self, vol_id, vol_rec, block_list): - self._vol_id = vol_id - self._vol_rec = vol_rec - self._name = self._vol_rec.name - self._block_list = block_list - log(description, 'Create Volume: %s, ID: %s, Block Cnt: %s' % (self.name, self.vol_id, len(self.block_list))) - - - def __repr__(self): - return 'Volume: %s' % (self.name.decode('utf-8')) - - - def _get_name(self): - return self._name - name = property(_get_name) - - - def _get_vol_id(self): - return self._vol_id - vol_id = property(_get_vol_id) - - - def _get_block_count(self): - return len(self._block_list) - block_count = property(_get_block_count) - - - def _get_vol_rec(self): - return self._vol_rec - vol_rec = property(_get_vol_rec) - - - def _get_block_list(self): - return self._block_list - block_list = property(_get_block_list) - - - def get_blocks(self, blocks): - return get_blocks_in_list(blocks, self._block_list) - - - def display(self, tab=''): - return display.volume(self, tab) - - - def reader(self, ubi): - last_leb = 0 - for block in sort.by_leb(self.get_blocks(ubi.blocks)): - if block == 'x': - last_leb += 1 - yield b'\xff'*ubi.leb_size - else: - last_leb += 1 - yield ubi.file.read_block_data(ubi.blocks[block]) - - -def get_volumes(blocks, layout_info): - """Get a list of UBI volume objects from list of blocks - - Arguments: - List:blocks -- List of layout block objects - List:layout_info -- Layout info (indexes of layout blocks and - associated data blocks.) - - Returns: - Dict -- Of Volume objects by volume name, including any - relevant blocks. - """ - volumes = {} - - vol_blocks_lists = sort.by_vol_id(blocks, layout_info[2]) - for vol_rec in blocks[layout_info[0]].vtbl_recs: - vol_name = vol_rec.name.strip(b'\x00').decode('utf-8') - if vol_rec.rec_index not in vol_blocks_lists: - vol_blocks_lists[vol_rec.rec_index] = [] - - vol_blocks_lists[vol_rec.rec_index] = rm_old_blocks(blocks, vol_blocks_lists[vol_rec.rec_index]) - volumes[vol_name] = description(vol_rec.rec_index, vol_rec, vol_blocks_lists[vol_rec.rec_index]) - - return volumes - diff --git a/python/ubireader/ubi_io.py b/python/ubireader/ubi_io.py deleted file mode 100644 index d6f80db..0000000 --- a/python/ubireader/ubi_io.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubi_io -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader.debug import error, log, verbose_log -from ubireader.ubi.block import sort - -class ubi_file(object): - """UBI image file object - - Arguments: - Str:path -- Path to file to parse - Int:block_size -- Erase block size of NAND in bytes. - Int:start_offset -- (optional) Where to start looking in the file for - UBI data. - Int:end_offset -- (optional) Where to stop looking in the file. - - Methods: - seek -- Put file head to specified byte offset. - Int:offset - read -- Read specified bytes from file handle. - Int:size - tell -- Returns byte offset of current file location. - read_block -- Returns complete PEB data of provided block - description. - Obj:block - read_block_data -- Returns LEB data only from provided block. - Obj:block - reader -- Generator that returns data from file. - reset -- Reset file position to start_offset. - is_valid -- If the object intialized okay. - - Handles all the actual file interactions, read, seek, - extract blocks, etc. - """ - - def __init__(self, path, block_size, start_offset=0, end_offset=None): - self.__name__ = 'UBI_File' - self.is_valid = False - try: - log(self, 'Open Path: %s' % path) - self._fhandle = open(path, 'rb') - except Exception as e: - error(self, 'Fatal', 'Open file: %s' % e) - - self._fhandle.seek(0,2) - file_size = self.tell() - log(self, 'File Size: %s' % file_size) - - self._start_offset = start_offset - log(self, 'Start Offset: %s' % (self._start_offset)) - - if end_offset: - self._end_offset = end_offset - else: - self._end_offset = file_size - log(self, 'End Offset: %s' % (self._end_offset)) - - self._block_size = block_size - log(self, 'Block Size: %s' % block_size) - - if start_offset > self._end_offset: - error(self, 'Fatal', 'Start offset larger than end offset.') - - if ( not end_offset is None ) and ( end_offset > file_size ): - error(self, 'Fatal', 'End offset larger than file size.') - - remainder = (self._end_offset - start_offset) % block_size - if remainder != 0: - error(self, 'Warning', 'end_offset - start_offset length is not block aligned, could mean missing data.') - - self._fhandle.seek(self._start_offset) - self._last_read_addr = self._fhandle.tell() - self.is_valid = True - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def _set_start(self, i): - self._start_offset = i - def _get_start(self): - return self._start_offset - start_offset = property(_get_start, _set_start) - - - def _get_end(self): - return self._end_offset - end_offset = property(_get_end) - - - def _get_block_size(self): - return self._block_size - block_size = property(_get_block_size) - - def close(self): - self._fhandle.close() - - def seek(self, offset): - self._fhandle.seek(offset) - - - def read(self, size): - self._last_read_addr = self.tell() - verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size)) - return self._fhandle.read(size) - - - def tell(self): - return self._fhandle.tell() - - - def last_read_addr(self): - return self._last_read_addr - - - def reset(self): - self._fhandle.seek(self.start_offset) - - - def reader(self): - self.reset() - while True: - cur_loc = self._fhandle.tell() - if self.end_offset and cur_loc > self.end_offset: - break - elif self.end_offset and self.end_offset - cur_loc < self.block_size: - chunk_size = self.end_offset - cur_loc - else: - chunk_size = self.block_size - - buf = self.read(chunk_size) - - if not buf: - break - yield buf - - - def read_block(self, block): - """Read complete PEB data from file. - - Argument: - Obj:block -- Block data is desired for. - """ - self.seek(block.file_offset) - return self._fhandle.read(block.size) - - - def read_block_data(self, block): - """Read LEB data from file - - Argument: - Obj:block -- Block data is desired for. - """ - self.seek(block.file_offset + block.ec_hdr.data_offset) - buf = self._fhandle.read(block.size - block.ec_hdr.data_offset - block.vid_hdr.data_pad) - return buf - - - -class leb_virtual_file(): - def __init__(self, ubi, block_list): - self.__name__ = 'leb_virtual_file' - self.is_valid = False - self._ubi = ubi - self._last_read_addr = 0 - - if not len(block_list): - error(self, 'Info', 'Empty block list') - else: - self._blocks = sort.by_leb(block_list) - self._seek = 0 - self._last_leb = -1 - self._last_buf = '' - self.is_valid = True - - - def read(self, size): - buf = '' - leb = int(self.tell() / self._ubi.leb_size) - offset = self.tell() % self._ubi.leb_size - - try: - if size < 0: - raise Exception('Bad Read Offset Request') - - self._last_read_addr = self._ubi.blocks[self._blocks[leb]].file_offset + self._ubi.blocks[self._blocks[leb]].ec_hdr.data_offset + offset - - except Exception as e: - error(self.read, 'Error', 'LEB: %s is corrupted or has no data.' % (leb)) - raise Exception('Bad Read Offset Request') - - verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size)) - - if leb == self._last_leb: - self.seek(self.tell() + size) - return self._last_buf[offset:offset+size] - else: - try: - buf = self._ubi.file.read_block_data(self._ubi.blocks[self._blocks[leb]]) - self._last_buf = buf - self._last_leb = leb - self.seek(self.tell() + size) - return buf[offset:offset+size] - except Exception as e: - error(self, 'Fatal', 'read loc: %s, size: %s, LEB: %s, offset: %s, error: %s' % (self._last_read_addr, size, leb, offset, e)) - - - def reset(self): - self.seek(0) - - - def seek(self, offset): - self._seek = offset - - - def tell(self): - return self._seek - - - def last_read_addr(self): - """Start address of last physical file read""" - return self._last_read_addr - - - def reader(self): - last_leb = 0 - for block in self._blocks: - while 0 != (self._ubi.blocks[block].leb_num - last_leb): - last_leb += 1 - yield b'\xff'*self._ubi.leb_size - - last_leb += 1 - yield self._ubi.file.read_block_data(self._ubi.blocks[block]) diff --git a/python/ubireader/ubifs/__init__.py b/python/ubireader/ubifs/__init__.py deleted file mode 100644 index 3e2bbca..0000000 --- a/python/ubireader/ubifs/__init__.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader.debug import error, log, verbose_display -from ubireader.ubifs.defines import * -from ubireader.ubifs import nodes, display - -class ubifs(): - """UBIFS object - - Arguments: - Str:path -- File path to UBIFS image. - - Attributes: - Obj:file -- File object - Int:leb_size -- Size of Logical Erase Blocks. - Int:min_io -- Size of min I/O from vid_hdr_offset. - Obj:sb_node -- Superblock node of UBIFS image LEB0 - Obj:mst_node -- Master Node of UBIFS image LEB1 - Obj:mst_node2 -- Master Node 2 of UBIFS image LEB2 - """ - def __init__(self, ubifs_file): - self.__name__ = 'UBIFS' - self._file = ubifs_file - try: - self.file.reset() - sb_chdr = nodes.common_hdr(self.file.read(UBIFS_COMMON_HDR_SZ)) - log(self , '%s file addr: %s' % (sb_chdr, self.file.last_read_addr())) - verbose_display(sb_chdr) - - if sb_chdr.node_type == UBIFS_SB_NODE: - self.file.seek(UBIFS_COMMON_HDR_SZ) - buf = self.file.read(UBIFS_SB_NODE_SZ) - self._sb_node = nodes.sb_node(buf, self.file.last_read_addr()) - self._min_io_size = self._sb_node.min_io_size - self._leb_size = self._sb_node.leb_size - log(self , '%s file addr: %s' % (self._sb_node, self.file.last_read_addr())) - verbose_display(self._sb_node) - else: - raise Exception('Wrong node type.') - except Exception as e: - error(self, 'Fatal', 'Super block error: %s' % e) - - self._mst_nodes = [None, None] - for i in range(0, 2): - try: - mst_offset = self.leb_size * (UBIFS_MST_LNUM + i) - self.file.seek(mst_offset) - mst_chdr = nodes.common_hdr(self.file.read(UBIFS_COMMON_HDR_SZ)) - log(self , '%s file addr: %s' % (mst_chdr, self.file.last_read_addr())) - verbose_display(mst_chdr) - - if mst_chdr.node_type == UBIFS_MST_NODE: - self.file.seek(mst_offset + UBIFS_COMMON_HDR_SZ) - buf = self.file.read(UBIFS_MST_NODE_SZ) - self._mst_nodes[i] = nodes.mst_node(buf, self.file.last_read_addr()) - log(self , '%s%s file addr: %s' % (self._mst_nodes[i], i, self.file.last_read_addr())) - verbose_display(self._mst_nodes[i]) - else: - raise Exception('Wrong node type.') - except Exception as e: - error(self, 'Warn', 'Master block %s error: %s' % (i, e)) - - if self._mst_nodes[0] is None and self._mst_nodes[1] is None: - error(self, 'Fatal', 'No valid Master Node found.') - - elif self._mst_nodes[0] is None and self._mst_nodes[1] is not None: - self._mst_nodes[0] = self._mst_nodes[1] - self._mst_nodes[1] = None - log(self , 'Swapping Master Nodes due to bad first node.') - - - def _get_file(self): - return self._file - file = property(_get_file) - - - def _get_superblock(self): - """ Superblock Node Object - - Returns: - Obj:Superblock Node - """ - return self._sb_node - superblock_node = property(_get_superblock) - - - def _get_master_node(self): - """Master Node Object - - Returns: - Obj:Master Node - """ - return self._mst_nodes[0] - master_node = property(_get_master_node) - - - def _get_master_node2(self): - """Master Node Object 2 - - Returns: - Obj:Master Node - """ - return self._mst_nodes[1] - master_node2 = property(_get_master_node2) - - - def _get_leb_size(self): - """LEB size of UBI blocks in file. - - Returns: - Int -- LEB Size. - """ - return self._leb_size - leb_size = property(_get_leb_size) - - - def _get_min_io_size(self): - """Min I/O Size - - Returns: - Int -- Min I/O Size. - """ - return self._min_io_size - min_io_size = property(_get_min_io_size) - - def display(self, tab=''): - """Print information about this object. - - Argument: - Str:tab -- '\t' for spacing this object. - """ - return display.ubifs(self, tab) diff --git a/python/ubireader/ubifs/defines.py b/python/ubireader/ubifs/defines.py deleted file mode 100644 index 1520706..0000000 --- a/python/ubireader/ubifs/defines.py +++ /dev/null @@ -1,432 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -############################################################# -# Adapted in part from linux-source-3.2/fs/ubi/ubi-media.h -# for use in Python. -# Oct. 2013 by Jason Pruitt -# -# Original copyright notice. -# -------------------------- -# -# This file is part of UBIFS. -# -# Copyright (C) 2006-2008 Nokia Corporation. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 as published by -# the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 51 -# Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: Artem Bityutskiy (Битюцкий Артём) -# Adrian Hunter -# -############################################################# - -import struct - -# Constant defines - -# Common Header. -UBIFS_NODE_MAGIC = b'\x31\x18\x10\x06' # Set to LSB - -# Initial CRC32 value. -UBIFS_CRC32_INIT = 0xFFFFFFFF - -# Do not compress data smaller than this. -UBIFS_MIN_COMPR_LEN = 128 - -# If difference between compressed data length and compressed data -# length, is less than this, do not compress data. -UBIFS_MIN_COMPRESS_DIFF = 64 - -# Root inode number -UBIFS_ROOT_INO = 1 - -# Lowest inode number for regular inodes, non-internal inodes. -UBIFS_FIRST_INO = 64 - -# Max file name and extended attr length (muptple of 8 minus 1. -UBIFS_MAX_NLEN = 255 - -# Max number of data journal heads. -UBIFS_MAX_JHEADS = 1 - -# Max data node data length/amount attached to inode node. -UBIFS_BLOCK_SIZE = 4096 -UBIFS_BLOCK_SHIFT = 12 - -# UBIFS padding byte pattern. -UBIFS_PADDING_BYTE = b'\xCE' - -# Max key length -UBIFS_MAX_KEY_LEN = 16 - -# Key length of simple format. -UBIFS_SK_LEN = 8 - -# Min index tree fanout. -UBIFS_MIN_FANOUT = 3 - -# Max number of levels in UBIFS indexing B-tree. -UBIFS_MAX_LEVELS = 512 - -# Max amount of data attached to inode in bytes. -UBIFS_MAX_INO_DATA = UBIFS_BLOCK_SIZE - -# LEB Properties Tree fanout (power of 2) and fanout. -UBIFS_LPT_FANOUT = 4 -UBIFS_LPT_FANOUT_SHIFT = 2 - -# LEB Properties Tree bit field sizes. -UBIFS_LPT_CRC_BITS = 16 -UBIFS_LPT_CRC_BYTES = 2 -UBIFS_LPT_TYPE_BITS = 4 - -# LEB Properties Tree node types. -UBIFS_LPT_PNODE = 0 # LPT leaf node (contains LEB Properties) -UBIFS_LPT_NNODE = 1 # LPT internal node -UBIFS_LPT_LTAB = 2 # LPT's own lprops table -UBIFS_LPT_LSAVE = 3 # LPT's save table (big model only) -UBIFS_LPT_NODE_CNT = 4 # count of LPT node types -UBIFS_LPT_NOT_A_NODE = (1 << UBIFS_LPT_TYPE_BITS) - 1 # 4 bits of 1 - -# Inode types -UBIFS_ITYPE_REG = 0 # Regular file -UBIFS_ITYPE_DIR = 1 # Directory -UBIFS_ITYPE_LNK = 2 # Soft link -UBIFS_ITYPE_BLK = 3 # Block device node -UBIFS_ITYPE_CHR = 4 # Char device node -UBIFS_ITYPE_FIFO = 5 # FIFO -UBIFS_ITYPE_SOCK = 6 # Socket -UBIFS_ITYPES_CNT = 7 # Support file type count - -# Supported key has functions -UBIFS_KEY_HASH_R5 = 0 # R5 hash -UBIFS_KEY_HASH_TEST = 1 # Test hash, returns first 4 bytes of name -PRINT_UBIFS_KEY_HASH = ['r5', 'test'] - -# Supported key formats -UBIFS_SIMPLE_KEY_FMT = 0 - -# Simple key format uses 29 bits for storing UBIFS name and hash. -UBIFS_S_KEY_BLOCK_BITS = 29 -UBIFS_S_KEY_BLOCK_MASK = 0x1FFFFFFF -UBIFS_S_KEY_HASH_BITS = UBIFS_S_KEY_BLOCK_BITS -UBIFS_S_KEY_HASH_MASK = UBIFS_S_KEY_BLOCK_MASK - -# Key types -UBIFS_INO_KEY = 0 # Inode node key -UBIFS_DATA_KEY = 1 # Data node key -UBIFS_DENT_KEY = 2 # Directory node key -UBIFS_XENT_KEY = 3 # Extended attribute entry key -UBIFS_KEY_TYPES_CNT = 4 # Supported key count - -# Number of reserved LEBs for Superblock area -UBIFS_SB_LEBS = 1 - -# Number of reserved LEBs for master area -UBIFS_MST_LEBS = 2 - -# First LEB of the Superblock area -UBIFS_SB_LNUM = 0 - -# First LEB of the master area -UBIFS_MST_LNUM = (UBIFS_SB_LNUM + UBIFS_SB_LEBS) - -# First LEB of log area -UBIFS_LOG_LNUM = (UBIFS_MST_LNUM + UBIFS_MST_LEBS) - -# On-flash inode flags -UBIFS_COMPR_FL = 1 # Use compression for this inode -UBIFS_SYNC_FL = 2 # Has to be synchronous I/O -UBIFS_IMMUTABLE_FL = 4 # Inode is immutable -UBIFS_APPEND_FL = 8 # Writes may only append data -UBIFS_DIRSYNC_FL = 16 # I/O on this directory inode must be synchronous -UBIFS_XATTR_FL = 32 # This inode is inode for extended attributes - -# Inode flag bits used by UBIFS -UBIFS_FL_MASK = 0x0000001F - -# Compression alogrithms. -UBIFS_COMPR_NONE = 0 # No compression -UBIFS_COMPR_LZO = 1 # LZO compression -UBIFS_COMPR_ZLIB = 2 # ZLIB compression -UBIFS_COMPR_ZSTD = 3 # ZSTD compression -UBIFS_COMPR_TYPES_CNT = 4 # Count of supported compression types -PRINT_UBIFS_COMPR = ['none','lzo','zlib', 'zstd'] - -# UBIFS node types -UBIFS_INO_NODE = 0 # Inode node -UBIFS_DATA_NODE = 1 # Data node -UBIFS_DENT_NODE = 2 # Directory entry node -UBIFS_XENT_NODE = 3 # Extended attribute node -UBIFS_TRUN_NODE = 4 # Truncation node -UBIFS_PAD_NODE = 5 # Padding node -UBIFS_SB_NODE = 6 # Superblock node -UBIFS_MST_NODE = 7 # Master node -UBIFS_REF_NODE = 8 # LEB reference node -UBIFS_IDX_NODE = 9 # Index node -UBIFS_CS_NODE = 10 # Commit start node -UBIFS_ORPH_NODE = 11 # Orphan node -UBIFS_AUTH_NODE = 12 # Authentication node -UBIFS_SIG_NODE = 13 # Signature node -UBIFS_NODE_TYPES_CNT = 14 # Count of supported node types - -# Master node flags -UBIFS_MST_DIRTY = 1 # Rebooted uncleanly -UBIFS_MST_NO_ORPHS = 2 # No orphans present -UBIFS_MST_RCVRY = 4 # Written by recovery -PRINT_UBIFS_MST = [[UBIFS_MST_DIRTY, 'Dirty'], - [UBIFS_MST_NO_ORPHS, 'No orphans'], - [UBIFS_MST_RCVRY, 'Recovery write'], - ] - -# Node group type -UBIFS_NO_NODE_GROUP = 0 # This node is not part of a group -UBIFS_IN_NODE_GROUP = 1 # This node is part of a group -UBIFS_LAST_OF_NODE_GROUP = 2 # This node is the last in a group - -# Superblock flags -UBIFS_FLG_BIGLPT = 2 # If 'big' LPT model is used if set. -UBIFS_FLG_SPACE_FIXUP = 4 # First-mount 'fixup' of free space within. -UBIFS_FLG_DOUBLE_HASH = 8 # Store 32bit cookie for 64bit support. -UBIFS_FLG_ENCRYPTION = 16 # If filesystem contains encrypted files. -UBIFS_FLG_AUTHENTICATION = 32 # If contains hashes for authentication. -PRINT_UBIFS_FLGS = [[UBIFS_FLG_BIGLPT, 'Big LPT'], - [UBIFS_FLG_SPACE_FIXUP, 'Space fixup'], - [UBIFS_FLG_DOUBLE_HASH, 'Double hash'], - [UBIFS_FLG_ENCRYPTION, 'Encryption'], - [UBIFS_FLG_AUTHENTICATION,'Authentication'], - ] - -# Struct defines - -# Common header node -UBIFS_COMMON_HDR_FORMAT = '. -############################################################# - -from ubireader.ubifs.defines import PRINT_UBIFS_FLGS, PRINT_UBIFS_MST - -def ubifs(ubifs, tab=''): - buf = '%sUBIFS Image\n' % (tab) - buf += '%s---------------------\n' % (tab) - buf += '%sMin I/O: %s\n' % (tab, ubifs.min_io_size) - buf += '%sLEB Size: %s\n' % (tab, ubifs.leb_size) - return buf - -def common_hdr(chdr, tab=''): - buf = '%s%s\n' % (tab, chdr) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in chdr: - if key == 'display': - continue - elif key == 'crc': - buf += '%s%s: 0x%x\n' % (tab, key, value) - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - -def sb_node(node, tab=''): - buf = '%s%s\n' % (tab, node) - buf += '%sFile offset: %s\n' % (tab, node.file_offset) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in node: - if key == 'display': - continue - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - elif key == 'uuid': - buf += '%s%s: %r\n' % (tab, key, value) - elif key == 'flags': - flags = '' - for flag in PRINT_UBIFS_FLGS: - if value & flag[0]: - flags += '%s, ' % flag[1] - - if flags.endswith(', '): - flags = flags[0:-2] - - buf += '%s%s: %s\n' % (tab, key, flags) - - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def mst_node(node, tab=''): - buf = '%s%s\n' % (tab, node) - buf += '%sFile offset: %s\n' % (tab, node.file_offset) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in node: - if key == 'display': - continue - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - elif key == 'flags': - flags = '' - for flag in PRINT_UBIFS_MST: - if value & flag[0]: - flags += '%s, ' % flag[1] - - if flags.endswith(', '): - flags = flags[0:-2] - - buf += '%s%s: %s\n' % (tab, key, flags) - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def dent_node(node, tab=''): - buf = '%s%s\n' % (tab, node) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in node: - if key == 'display': - continue - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def data_node(node, tab=''): - buf = '%s%s\n' % (tab, node) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in node: - if key in ['display', 'data']: - continue - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def idx_node(node, tab=''): - buf = '%s%s\n' % (tab, node) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in node: - if key == 'display': - continue - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def ino_node(node, tab=''): - buf = '%s%s\n' % (tab, node) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in node: - if key == 'display': - continue - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - - -def branch(node, tab=''): - buf = '%s%s\n' % (tab, node) - buf += '%s---------------------\n' % (tab) - tab += '\t' - for key, value in node: - if key == 'display': - continue - elif key == 'errors': - buf += '%s%s: %s\n' % (tab, key, ','.join(value)) - else: - buf += '%s%s: %r\n' % (tab, key, value) - return buf - diff --git a/python/ubireader/ubifs/list.py b/python/ubireader/ubifs/list.py deleted file mode 100644 index 166ea83..0000000 --- a/python/ubireader/ubifs/list.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (C) Collin Mulliner based on Jason Pruitt's output.py - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import time -from ubireader.ubifs.defines import * -from ubireader.ubifs import walk -from ubireader.ubifs.misc import decompress -from ubireader.debug import error, log, verbose_log - - -def list_files(ubifs, list_path): - pathnames = list_path.split("/") - pnames = [] - for i in pathnames: - if len(i) > 0: - pnames.append(i) - try: - inodes = {} - bad_blocks = [] - - walk.index(ubifs, ubifs.master_node.root_lnum, ubifs.master_node.root_offs, inodes, bad_blocks) - - if len(inodes) < 2: - raise Exception('No inodes found') - - inum = find_dir(inodes, 1, pnames, 0) - - if inum == None: - return - - if not 'dent' in inodes[inum]: - return - - for dent in inodes[inum]['dent']: - print_dent(ubifs, inodes, dent, longts=False) - - if len(bad_blocks): - error(list_files, 'Warn', 'Data may be missing or corrupted, bad blocks, LEB [%s]' % ','.join(map(str, bad_blocks))) - - except Exception as e: - error(list_files, 'Error', '%s' % e) - - -def copy_file(ubifs, filepath, destpath): - pathnames = filepath.split("/") - pnames = [] - for i in pathnames: - if len(i) > 0: - pnames.append(i) - - filename = pnames[len(pnames)-1] - del pnames[-1] - - inodes = {} - bad_blocks = [] - - walk.index(ubifs, ubifs.master_node.root_lnum, ubifs.master_node.root_offs, inodes, bad_blocks) - - if len(inodes) < 2: - return False - - inum = find_dir(inodes, 1, pnames, 0) - - if inum == None: - return False - - if not 'dent' in inodes[inum]: - return False - - for dent in inodes[inum]['dent']: - if dent.name == filename: - filedata = _process_reg_file(ubifs, inodes[dent.inum], filepath) - if os.path.isdir(destpath): - destpath = os.path.join(destpath, filename) - with open(destpath, 'wb') as f: - f.write(filedata) - return True - return False - - -def find_dir(inodes, inum, names, idx): - if len(names) == 0: - return 1 - for dent in inodes[inum]['dent']: - if dent.name == names[idx]: - if len(names) == idx+1: - return dent.inum - else: - return find_dir(inodes, dent.inum, names, idx+1) - return None - - -def print_dent(ubifs, inodes, dent_node, long=True, longts=False): - inode = inodes[dent_node.inum] - if long: - fl = file_leng(ubifs, inode) - - lnk = "" - if dent_node.type == UBIFS_ITYPE_LNK: - lnk = " -> " + inode['ino'].data.decode('utf-8') - - if longts: - mtime = inode['ino'].mtime_sec - else: - mtime = time.strftime("%b %d %H:%M", time.gmtime(inode['ino'].mtime_sec)) - - print('%6o %2d %s %s %7d %s %s%s' % (inode['ino'].mode, inode['ino'].nlink, inode['ino'].uid, inode['ino'].gid, fl, mtime, dent_node.name, lnk)) - else: - print(dent_node.name) - - -def file_leng(ubifs, inode): - fl = 0 - if 'data' in inode: - compr_type = 0 - sorted_data = sorted(inode['data'], key=lambda x: x.key['khash']) - last_khash = sorted_data[0].key['khash']-1 - - for data in sorted_data: - if data.key['khash'] - last_khash != 1: - while 1 != (data.key['khash'] - last_khash): - last_khash += 1 - fl = fl + UBIFS_BLOCK_SIZE - fl = fl + data.size - return fl - return 0 - - -def _process_reg_file(ubifs, inode, path): - try: - buf = bytearray() - if 'data' in inode: - compr_type = 0 - sorted_data = sorted(inode['data'], key=lambda x: x.key['khash']) - last_khash = sorted_data[0].key['khash']-1 - - for data in sorted_data: - - # If data nodes are missing in sequence, fill in blanks - # with \x00 * UBIFS_BLOCK_SIZE - if data.key['khash'] - last_khash != 1: - while 1 != (data.key['khash'] - last_khash): - buf += b'\x00'*UBIFS_BLOCK_SIZE - last_khash += 1 - - compr_type = data.compr_type - ubifs.file.seek(data.offset) - d = ubifs.file.read(data.compr_len) - buf += decompress(compr_type, data.size, d) - last_khash = data.key['khash'] - verbose_log(_process_reg_file, 'ino num: %s, compression: %s, path: %s' % (inode['ino'].key['ino_num'], compr_type, path)) - - except Exception as e: - error(_process_reg_file, 'Warn', 'inode num:%s :%s' % (inode['ino'].key['ino_num'], e)) - - # Pad end of file with \x00 if needed. - if inode['ino'].size > len(buf): - buf += b'\x00' * (inode['ino'].size - len(buf)) - - return bytes(buf) diff --git a/python/ubireader/ubifs/misc.py b/python/ubireader/ubifs/misc.py deleted file mode 100644 index 7d63d34..0000000 --- a/python/ubireader/ubifs/misc.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -#from lzallright import LZOCompressor, LZOError -import struct -import zlib -from ubireader.ubifs.defines import * -from ubireader.debug import error - -# For happy printing -ino_types = ['file', 'dir','lnk','blk','chr','fifo','sock'] -node_types = ['ino','data','dent','xent','trun','pad','sb','mst','ref','idx','cs','orph'] -key_types = ['ino','data','dent','xent'] - - -def parse_key(key): - """Parse node key - - Arguments: - Str:key -- Hex string literal of node key. - - Returns: - Int:key_type -- Type of key, data, ino, dent, etc. - Int:ino_num -- Inode number. - Int:khash -- Key hash. - """ - hkey, lkey = struct.unpack('> UBIFS_S_KEY_BLOCK_BITS - khash = lkey - - #if key_type < UBIFS_KEY_TYPES_CNT: - return {'type':key_type, 'ino_num':ino_num, 'khash': khash} - - -def decompress(ctype, unc_len, data): - """Decompress data. - - Arguments: - Int:ctype -- Compression type LZO, ZLIB (*currently unused*). - Int:unc_len -- Uncompressed data lenth. - Str:data -- Data to be uncompessed. - - Returns: - Uncompressed Data. - """ - if ctype == UBIFS_COMPR_LZO: - from lzallright import LZOCompressor, LZOError - try: - return LZOCompressor.decompress(data, output_size_hint=unc_len) - except Exception as e: - error(decompress, 'Warn', 'LZO Error: %s' % e) - elif ctype == UBIFS_COMPR_ZLIB: - try: - return zlib.decompress(data, -11) - except Exception as e: - error(decompress, 'Warn', 'ZLib Error: %s' % e) - else: - return data - - diff --git a/python/ubireader/ubifs/nodes.py b/python/ubireader/ubifs/nodes.py deleted file mode 100644 index fd2bcb7..0000000 --- a/python/ubireader/ubifs/nodes.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader.ubifs.misc import parse_key -from ubireader.ubifs.defines import * -from ubireader.ubifs import display - -class common_hdr(object): - """Get common header at given LEB number + offset. - - Arguments: - Bin:buf -- Raw data to extract header information from. - - See ubifs/defines.py for object attributes. - """ - def __init__(self, buf): - - fields = dict(list(zip(UBIFS_COMMON_HDR_FIELDS, struct.unpack(UBIFS_COMMON_HDR_FORMAT, buf)))) - for key in fields: - setattr(self, key, fields[key]) - - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Common Header' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.common_hdr(self, tab) - - -class ino_node(object): - """Get inode node at given LEB number + offset. - - Arguments: - Bin:buf -- Raw data to extract header information from. - - See ubifs/defines.py for object attributes. - """ - def __init__(self, buf): - - fields = dict(list(zip(UBIFS_INO_NODE_FIELDS, struct.unpack(UBIFS_INO_NODE_FORMAT, buf[0:UBIFS_INO_NODE_SZ])))) - for key in fields: - if key == 'key': - setattr(self, key, parse_key(fields[key])) - else: - setattr(self, key, fields[key]) - - setattr(self, 'data', buf[UBIFS_INO_NODE_SZ:]) - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Ino Node' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.ino_node(self, tab) - - -class dent_node(object): - """Get dir entry node at given LEB number + offset. - - Arguments: - Bin:buf -- Raw data to extract header information from. - - See ubifs/defines.py for object attributes. - """ - def __init__(self, buf): - fields = dict(list(zip(UBIFS_DENT_NODE_FIELDS, struct.unpack(UBIFS_DENT_NODE_FORMAT, buf[0:UBIFS_DENT_NODE_SZ])))) - for key in fields: - if key == 'key': - setattr(self, key, parse_key(fields[key])) - else: - setattr(self, key, fields[key]) - - setattr(self, 'name', '%s' % buf[-self.nlen-1:-1].decode('utf-8')) - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Directory Entry Node' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.dent_node(self, tab) - - -class data_node(object): - """Get data node at given LEB number + offset. - - Arguments: - Bin:buf -- Raw data to extract header information from. - Int:offset -- Offset in LEB of data node. - - See ubifs/defines.py for object attributes. - """ - def __init__(self, buf, file_offset): - - fields = dict(list(zip(UBIFS_DATA_NODE_FIELDS, struct.unpack(UBIFS_DATA_NODE_FORMAT, buf[0:UBIFS_DATA_NODE_SZ])))) - for key in fields: - if key == 'key': - setattr(self, key, parse_key(fields[key])) - else: - setattr(self, key, fields[key]) - - setattr(self, 'offset', file_offset) - setattr(self, 'compr_len', (len(buf) - UBIFS_DATA_NODE_SZ)) - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Data Node' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.data_node(self, tab) - - -class idx_node(object): - """Get index node at given LEB number + offset. - - Arguments: - Bin:buf -- Raw data to extract header information from. - - See ubifs/defines.py for object attributes. - """ - def __init__(self, buf): - fields = dict(list(zip(UBIFS_IDX_NODE_FIELDS, struct.unpack(UBIFS_IDX_NODE_FORMAT, buf[0:UBIFS_IDX_NODE_SZ])))) - for key in fields: - setattr(self, key, fields[key]) - - idxs = UBIFS_IDX_NODE_SZ - brs = UBIFS_BRANCH_SZ - setattr(self, 'branches', [branch(buf[idxs+(brs*i):idxs+(brs*i)+brs]) for i in range(0, self.child_cnt)]) - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Index Node' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.idx_node(self, tab) - - -class branch(object): - """ Create branch from given idx_node data buf. - - Arguments: - Bin:buf -- Raw data to extract header information from. - """ - def __init__(self, buf): - fields = dict(list(zip(UBIFS_BRANCH_FIELDS, struct.unpack(UBIFS_BRANCH_FORMAT, buf)))) - for key in fields: - if key == 'key': - setattr(self, key, parse_key(fields[key])) - else: - setattr(self, key, fields[key]) - - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Branch' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.branch(self, tab) - - -class sb_node(object): - """Get superblock node at given LEB number + offset. - - Arguments: - Bin:buf -- Raw data to extract header information from. - Int:offset -- Offset in LEB of data node. - - See ubifs/defines.py for object attributes. - """ - def __init__(self, buf, file_offset=-1): - self.file_offset = file_offset - fields = dict(list(zip(UBIFS_SB_NODE_FIELDS, struct.unpack(UBIFS_SB_NODE_FORMAT, buf)))) - for key in fields: - setattr(self, key, fields[key]) - - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Super Block Node' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.sb_node(self, tab) - - -class mst_node(object): - """Get master node at given LEB number + offset. - - Arguments: - Bin:buf -- Raw data to extract header information from. - Int:offset -- Offset in LEB of data node. - - See ubifs/defines.py for object attributes. - """ - def __init__(self, buf, file_offset=-1): - self.file_offset = file_offset - fields = dict(list(zip(UBIFS_MST_NODE_FIELDS, struct.unpack(UBIFS_MST_NODE_FORMAT, buf)))) - for key in fields: - setattr(self, key, fields[key]) - - setattr(self, 'errors', []) - - def __repr__(self): - return 'UBIFS Master Block Node' - - def __iter__(self): - for key in dir(self): - if not key.startswith('_'): - yield key, getattr(self, key) - - def display(self, tab=''): - return display.mst_node(self, tab) diff --git a/python/ubireader/ubifs/output.py b/python/ubireader/ubifs/output.py deleted file mode 100644 index b4aa653..0000000 --- a/python/ubireader/ubifs/output.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import struct - -from ubireader import settings -from ubireader.ubifs.defines import * -from ubireader.ubifs import walk -from ubireader.ubifs.misc import decompress -from ubireader.debug import error, log, verbose_log - -def is_safe_path(basedir, path): - basedir = os.path.realpath(basedir) - path = os.path.realpath(os.path.join(basedir, path)) - return True if path.startswith(basedir) else False - - -def extract_files(ubifs, out_path, perms=False): - """Extract UBIFS contents to_path/ - - Arguments: - Obj:ubifs -- UBIFS object. - Str:out_path -- Path to extract contents to. - """ - try: - inodes = {} - bad_blocks = [] - - walk.index(ubifs, ubifs.master_node.root_lnum, ubifs.master_node.root_offs, inodes, bad_blocks) - - if len(inodes) < 2: - raise Exception('No inodes found') - - for dent in inodes[1]['dent']: - extract_dents(ubifs, inodes, dent, out_path, perms) - - if len(bad_blocks): - error(extract_files, 'Warn', 'Data may be missing or corrupted, bad blocks, LEB [%s]' % ','.join(map(str, bad_blocks))) - - except Exception as e: - error(extract_files, 'Error', '%s' % e) - - -def extract_dents(ubifs, inodes, dent_node, path='', perms=False): - if dent_node.inum not in inodes: - error(extract_dents, 'Error', 'inum: %s not found in inodes' % (dent_node.inum)) - return - - inode = inodes[dent_node.inum] - - if not is_safe_path(path, dent_node.name): - error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding.' % (dent_node.name)) - return - dent_path = os.path.realpath(os.path.join(path, dent_node.name)) - - if dent_node.type == UBIFS_ITYPE_DIR: - try: - if not os.path.exists(dent_path): - os.mkdir(dent_path) - log(extract_dents, 'Make Dir: %s' % (dent_path)) - - if perms: - _set_file_perms(dent_path, inode) - except Exception as e: - error(extract_dents, 'Warn', 'DIR Fail: %s' % e) - - if 'dent' in inode: - for dnode in inode['dent']: - extract_dents(ubifs, inodes, dnode, dent_path, perms) - - _set_file_timestamps(dent_path, inode) - - elif dent_node.type == UBIFS_ITYPE_REG: - try: - if inode['ino'].nlink > 1: - if 'hlink' not in inode: - inode['hlink'] = dent_path - buf = _process_reg_file(ubifs, inode, dent_path) - _write_reg_file(dent_path, buf) - else: - os.link(inode['hlink'], dent_path) - log(extract_dents, 'Make Link: %s > %s' % (dent_path, inode['hlink'])) - else: - buf = _process_reg_file(ubifs, inode, dent_path) - _write_reg_file(dent_path, buf) - - _set_file_timestamps(dent_path, inode) - - if perms: - _set_file_perms(dent_path, inode) - - except Exception as e: - error(extract_dents, 'Warn', 'FILE Fail: %s' % e) - - elif dent_node.type == UBIFS_ITYPE_LNK: - try: - # probably will need to decompress ino data if > UBIFS_MIN_COMPR_LEN - os.symlink('%s' % inode['ino'].data.decode('utf-8'), dent_path) - log(extract_dents, 'Make Symlink: %s > %s' % (dent_path, inode['ino'].data)) - - except Exception as e: - error(extract_dents, 'Warn', 'SYMLINK Fail: %s' % e) - - elif dent_node.type in [UBIFS_ITYPE_BLK, UBIFS_ITYPE_CHR]: - try: - dev = struct.unpack(' len(buf): - buf += b'\x00' * (inode['ino'].size - len(buf)) - - return bytes(buf) diff --git a/python/ubireader/ubifs/walk.py b/python/ubireader/ubifs/walk.py deleted file mode 100644 index 370caf5..0000000 --- a/python/ubireader/ubifs/walk.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -from ubireader import settings -from ubireader.ubifs import nodes -from ubireader.ubifs.defines import * -from ubireader.debug import error, log, verbose_log, verbose_display - -def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]): - """Walk the index gathering Inode, Dir Entry, and File nodes. - - Arguments: - Obj:ubifs -- UBIFS object. - Int:lnum -- Logical erase block number. - Int:offset -- Offset in logical erase block. - Dict:inodes -- Dict of ino/dent/file nodes keyed to inode number. - - Returns: - Dict:inodes -- Dict of ino/dent/file nodes keyed to inode number. - 'ino' -- Inode node. - 'data' -- List of data nodes if present. - 'dent' -- List of directory entry nodes if present. - """ - if len(bad_blocks): - if lnum in bad_blocks: - return - - ubifs.file.seek((ubifs.leb_size * lnum) + offset) - buf = ubifs.file.read(UBIFS_COMMON_HDR_SZ) - - if len(buf) < UBIFS_COMMON_HDR_SZ: - if settings.warn_only_block_read_errors: - error(index, 'Error', 'LEB: %s, Common Hdr Size smaller than expected.' % (lnum)) - return - - else: - error(index, 'Fatal', 'LEB: %s, Common Hdr Size smaller than expected.' % (lnum)) - - chdr = nodes.common_hdr(buf) - log(index , '%s file addr: %s' % (chdr, ubifs.file.last_read_addr())) - verbose_display(chdr) - read_size = chdr.len - UBIFS_COMMON_HDR_SZ - node_buf = ubifs.file.read(read_size) - file_offset = ubifs.file.last_read_addr() - - if len(node_buf) < read_size: - if settings.warn_only_block_read_errors: - error(index, 'Error', 'LEB: %s at %s, Node size smaller than expected.' % (lnum, file_offset)) - return - - else: - error(index, 'Fatal', 'LEB: %s at %s, Node size smaller than expected.' % (lnum, file_offset)) - - if chdr.node_type == UBIFS_IDX_NODE: - try: - idxn = nodes.idx_node(node_buf) - - except Exception as e: - if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e)) - return - - else: - error(index, 'Fatal', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e)) - - log(index, '%s file addr: %s' % (idxn, file_offset)) - verbose_display(idxn) - branch_idx = 0 - - for branch in idxn.branches: - verbose_log(index, '-------------------') - log(index, '%s file addr: %s' % (branch, file_offset + UBIFS_IDX_NODE_SZ + (branch_idx * UBIFS_BRANCH_SZ))) - verbose_display(branch) - index(ubifs, branch.lnum, branch.offs, inodes, bad_blocks) - branch_idx += 1 - - elif chdr.node_type == UBIFS_INO_NODE: - try: - inon = nodes.ino_node(node_buf) - - except Exception as e: - if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting ino_node: %s' % (file_offset, e)) - return - - else: - error(index, 'Fatal', 'Problem at file address: %s extracting ino_node: %s' % (file_offset, e)) - - ino_num = inon.key['ino_num'] - log(index, '%s file addr: %s, ino num: %s' % (inon, file_offset, ino_num)) - verbose_display(inon) - - if not ino_num in inodes: - inodes[ino_num] = {} - - inodes[ino_num]['ino'] = inon - - elif chdr.node_type == UBIFS_DATA_NODE: - try: - datn = nodes.data_node(node_buf, (ubifs.leb_size * lnum) + UBIFS_COMMON_HDR_SZ + offset + UBIFS_DATA_NODE_SZ) - - except Exception as e: - if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting data_node: %s' % (file_offset, e)) - return - - else: - error(index, 'Fatal', 'Problem at file address: %s extracting data_node: %s' % (file_offset, e)) - - ino_num = datn.key['ino_num'] - log(index, '%s file addr: %s, ino num: %s' % (datn, file_offset, ino_num)) - verbose_display(datn) - - if not ino_num in inodes: - inodes[ino_num] = {} - - if not 'data' in inodes[ino_num]: - inodes[ino_num]['data']= [] - - inodes[ino_num]['data'].append(datn) - - elif chdr.node_type == UBIFS_DENT_NODE: - try: - dn = nodes.dent_node(node_buf) - - except Exception as e: - if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting dent_node: %s' % (file_offset, e)) - return - - else: - error(index, 'Fatal', 'Problem at file address: %s extracting dent_node: %s' % (file_offset, e)) - - ino_num = dn.key['ino_num'] - log(index, '%s file addr: %s, ino num: %s' % (dn, file_offset, ino_num)) - verbose_display(dn) - - if not ino_num in inodes: - inodes[ino_num] = {} - - if not 'dent' in inodes[ino_num]: - inodes[ino_num]['dent']= [] - - inodes[ino_num]['dent'].append(dn) diff --git a/python/ubireader/utils.py b/python/ubireader/utils.py deleted file mode 100644 index 1793285..0000000 --- a/python/ubireader/utils.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import re -from ubireader.debug import error, log -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, FILE_CHUNK_SZ -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC, UBIFS_SB_NODE_SZ, UBIFS_SB_NODE, UBIFS_COMMON_HDR_SZ -from ubireader.ubifs import nodes - -def guess_start_offset(path, guess_offset=0): - file_offset = guess_offset - - f = open(path, 'rb') - f.seek(0,2) - file_size = f.tell()+1 - f.seek(guess_offset) - - for _ in range(0, file_size, FILE_CHUNK_SZ): - buf = f.read(FILE_CHUNK_SZ) - ubi_loc = buf.find(UBI_EC_HDR_MAGIC) - ubifs_loc = buf.find(UBIFS_NODE_MAGIC) - - if ubi_loc == -1 and ubifs_loc == -1: - file_offset += FILE_CHUNK_SZ - continue - else: - if ubi_loc == -1: - ubi_loc = file_size + 1 - elif ubifs_loc == -1: - ubifs_loc = file_size + 1 - - if ubi_loc < ubifs_loc: - log(guess_start_offset, 'Found UBI magic number at %s' % (file_offset + ubi_loc)) - return file_offset + ubi_loc - - elif ubifs_loc < ubi_loc: - log(guess_start_offset, 'Found UBIFS magic number at %s' % (file_offset + ubifs_loc)) - return file_offset + ubifs_loc - else: - error(guess_start_offset, 'Fatal', 'Could not determine start offset.') - else: - error(guess_start_offset, 'Fatal', 'Could not determine start offset.') - - f.close() - - -def guess_filetype(path, start_offset=0): - log(guess_filetype, 'Looking for file type at %s' % start_offset) - - with open(path, 'rb') as f: - f.seek(start_offset) - buf = f.read(4) - - if buf == UBI_EC_HDR_MAGIC: - ftype = UBI_EC_HDR_MAGIC - log(guess_filetype, 'File looks like a UBI image.') - - elif buf == UBIFS_NODE_MAGIC: - ftype = UBIFS_NODE_MAGIC - log(guess_filetype, 'File looks like a UBIFS image.') - else: - ftype = None - error(guess_filetype, 'Fatal', 'Could not determine file type.') - - return ftype - - -def guess_leb_size(path): - """Get LEB size from superblock - - Arguments: - Str:path -- Path to file. - - Returns: - Int -- LEB size. - - Searches file for superblock and retrieves leb size. - """ - - f = open(path, 'rb') - f.seek(0,2) - file_size = f.tell()+1 - f.seek(0) - block_size = None - - for _ in range(0, file_size, FILE_CHUNK_SZ): - buf = f.read(FILE_CHUNK_SZ) - - for m in re.finditer(UBIFS_NODE_MAGIC, buf): - start = m.start() - chdr = nodes.common_hdr(buf[start:start+UBIFS_COMMON_HDR_SZ]) - - if chdr and chdr.node_type == UBIFS_SB_NODE: - sb_start = start + UBIFS_COMMON_HDR_SZ - sb_end = sb_start + UBIFS_SB_NODE_SZ - - if chdr.len != len(buf[sb_start:sb_end]): - f.seek(sb_start) - buf = f.read(UBIFS_SB_NODE_SZ) - else: - buf = buf[sb_start:sb_end] - - sbn = nodes.sb_node(buf) - block_size = sbn.leb_size - f.close() - return block_size - - f.close() - return block_size - - -def guess_peb_size(path): - """Determine the most likely block size - - Arguments: - Str:path -- Path to file. - - Returns: - Int -- PEB size. - - Searches file for Magic Number, picks most - common length between them. - """ - file_offset = 0 - offsets = [] - f = open(path, 'rb') - f.seek(0,2) - file_size = f.tell()+1 - f.seek(0) - - for _ in range(0, file_size, FILE_CHUNK_SZ): - buf = f.read(FILE_CHUNK_SZ) - for m in re.finditer(UBI_EC_HDR_MAGIC, buf): - start = m.start() - - if not file_offset: - file_offset = start - idx = start - else: - idx = start+file_offset - - offsets.append(idx) - - file_offset += FILE_CHUNK_SZ - f.close() - - occurrences = {} - for i in range(0, len(offsets)): - try: - diff = offsets[i] - offsets[i-1] - except: - diff = offsets[i] - - if diff not in occurrences: - occurrences[diff] = 0 - - occurrences[diff] += 1 - - most_frequent = 0 - block_size = None - - for offset in occurrences: - if occurrences[offset] > most_frequent: - most_frequent = occurrences[offset] - block_size = offset - - return block_size \ No newline at end of file