PlayStation 4 developer oct0xor (aka @Octopus) made available a PS4 Registry Editor and Viewer for other devs to examine the system.nvs, system.dat, system.idx, system.eap and system.rec.
Download: ps4_registry_editor-master.zip / GIT
This comes following the PS4 EAP Kernel Dumps, and to quote from his Blog page on it: PS4 Registry Editor
Recently I reverse-engineered PS4 registry format and made a simple tool to view and edit it.
Sony likes to encrypt and obfuscate everything. Every time it is very fun to figure it out.
PS4 registry is represented by a few different files and file formats:
Entries inside system.eap and system.rec are stored in obfuscated format.
First of all, its XOR’ed with 8 bytes, there are a lot of Null bytes, so it’s easy to figure out them without reversing. But thats not all. RegID’s are encrypted, entries and data are hashed. Besides that, RegID’s for system.eap and system.rec are encrypted with different keys.
Why to obfuscate? Dunno.
Folders with those files should be not easily accessible.
My suggestions:
My tool allows to work with system.dat, system.idx, system.eap and system.rec. System.nvs is not supported because its stored in kernel like view and therefore parsing will very depend on a system version. However, it contains the same entries as in system.rec.
It does not support rebuilding, so it is not possible to add new entries yet.
As of 5.01 system.rec seems to contain additional layer of encryption, it’s not implemented yet.
CXML_decompiler.py / CXMLDecompilerv2.zip / CXMLDecompilerv10.zip / GIT via @SilicaAndPina
A CXML Decompiler supporting PS4 and PSVita CXML Files, Such as rco, rcs and app.info
Download: ps4_registry_editor-master.zip / GIT
This comes following the PS4 EAP Kernel Dumps, and to quote from his Blog page on it: PS4 Registry Editor
Recently I reverse-engineered PS4 registry format and made a simple tool to view and edit it.
Sony likes to encrypt and obfuscate everything. Every time it is very fun to figure it out.
PS4 registry is represented by a few different files and file formats:
- /system_data/settings/system.nvs
- /system_data/settings/system.dat
- /system_data/settings/system.idx
- /user/settings/system.eap
- /system_data/settings/system.rec
Entries inside system.eap and system.rec are stored in obfuscated format.
First of all, its XOR’ed with 8 bytes, there are a lot of Null bytes, so it’s easy to figure out them without reversing. But thats not all. RegID’s are encrypted, entries and data are hashed. Besides that, RegID’s for system.eap and system.rec are encrypted with different keys.
Why to obfuscate? Dunno.
Folders with those files should be not easily accessible.
My suggestions:
- To prevent fuzzing of file format
- Implement new crypto and hash algorithms is the most fun thing to do at work
- This data is very sensitive
My tool allows to work with system.dat, system.idx, system.eap and system.rec. System.nvs is not supported because its stored in kernel like view and therefore parsing will very depend on a system version. However, it contains the same entries as in system.rec.
It does not support rebuilding, so it is not possible to add new entries yet.
As of 5.01 system.rec seems to contain additional layer of encryption, it’s not implemented yet.
CXML_decompiler.py / CXMLDecompilerv2.zip / CXMLDecompilerv10.zip / GIT via @SilicaAndPina
A CXML Decompiler supporting PS4 and PSVita CXML Files, Such as rco, rcs and app.info
Code:
#!python2
#ps4 related changes -oct0xor
import sys, os, struct
from io import BytesIO
from pprint import pprint
def read_cstring(f):
bytes = []
while True:
byte = f.read(1)
if byte == b'\x00':
break
elif byte == '':
raise EOFError()
else:
bytes.append(byte)
return b''.join(bytes)
def check_file_magic(f, expected_magic):
old_offset = f.tell()
try:
magic = f.read(len(expected_magic))
except:
return False
finally:
f.seek(old_offset)
return magic == expected_magic
script_file_name = os.path.split(sys.argv[0])[1]
script_file_base = os.path.splitext(script_file_name)[0]
if len(sys.argv) < 2:
print('CXML decompiler (c) flatz')
print('Usage: {0} <cxml file> <xml file>'.format(script_file_name))
sys.exit()
ENDIANNESS = '<'
def write_raw(f, data):
if type(data) == str:
f.write(data)
elif type(data) == unicode:
f.write(data.decode('utf-8'))
else:
f.write(data)
def write_indent(f, depth):
write_raw(f, '\t' * depth)
def write_line(f, data):
write_raw(f, data)
write_raw(f, '\n')
INT_FMT = ENDIANNESS + 'i'
FLOAT_FMT = ENDIANNESS + 'f'
STRING_FMT = ENDIANNESS + 'ii'
INT_ARRAY_FMT = ENDIANNESS + 'ii'
FLOAT_ARRAY_FMT = ENDIANNESS + 'ii'
FILE_FMT = ENDIANNESS + 'ii'
ID_FMT = ENDIANNESS + 'i'
ID_REF_FMT = ENDIANNESS + 'i'
class Attribute(object):
HEADER_FMT = ENDIANNESS + 'ii'
HEADER_SIZE = struct.calcsize(HEADER_FMT)
SIZE = HEADER_SIZE + max(struct.calcsize(INT_FMT), struct.calcsize(FLOAT_FMT), struct.calcsize(STRING_FMT), struct.calcsize(INT_ARRAY_FMT), struct.calcsize(FLOAT_ARRAY_FMT), struct.calcsize(FILE_FMT), struct.calcsize(ID_FMT), struct.calcsize(ID_REF_FMT))
TYPE_NONE = 0
TYPE_INT = 1
TYPE_FLOAT = 2
TYPE_STRING = 3
TYPE_INT_ARRAY = 4
TYPE_FLOAT_ARRAY = 5
TYPE_UNK1 = 6
TYPE_ID = 7
TYPE_FILE = 8
TYPE_ID_REF = 9
TYPE_UNK2 = 11
def __init__(self, element):
self.element = element
self.start = None
self.name = None
self.type = None
self.offset = None
self.length = None
self.value = None
def load(self, f):
self.start = f.tell()
data = f.read(self.HEADER_SIZE)
self.name, self.type = struct.unpack(self.HEADER_FMT, data)
data = f.read(self.SIZE - self.HEADER_SIZE)
if self.type == self.TYPE_NONE:
pass
elif self.type == self.TYPE_INT:
self.value, = struct.unpack(INT_FMT, data[:struct.calcsize(INT_FMT)])
elif self.type == self.TYPE_FLOAT:
self.value, = struct.unpack(FLOAT_FMT, data[:struct.calcsize(FLOAT_FMT)])
elif self.type == self.TYPE_STRING:
self.offset, self.length = struct.unpack(STRING_FMT, data[:struct.calcsize(STRING_FMT)])
elif self.type == self.TYPE_INT_ARRAY:
self.offset, self.length = struct.unpack(INT_ARRAY_FMT, data[:struct.calcsize(INT_ARRAY_FMT)])
elif self.type == self.TYPE_FLOAT_ARRAY:
self.offset, self.length = struct.unpack(FLOAT_ARRAY_FMT, data[:struct.calcsize(FLOAT_ARRAY_FMT)])
elif self.type == self.TYPE_UNK1:
self.offset, self.length = struct.unpack(FILE_FMT, data[:struct.calcsize(FILE_FMT)])
elif self.type == self.TYPE_ID:
self.offset, = struct.unpack(ID_FMT, data[:struct.calcsize(ID_FMT)])
elif self.type == self.TYPE_ID_REF:
self.offset, = struct.unpack(ID_REF_FMT, data[:struct.calcsize(ID_REF_FMT)])
elif self.type == self.TYPE_FILE:
self.offset, self.length = struct.unpack(FILE_FMT, data[:struct.calcsize(FILE_FMT)])
elif self.type == self.TYPE_UNK2:
self.value, = struct.unpack(INT_FMT, data[:struct.calcsize(INT_FMT)])
return True
def get_unk1(self):
return self.offset, self.length
def get_unk2(self):
return self.value
def get_name(self):
return self.element.document.get_string(self.name)
def get_int(self):
if self.type != self.TYPE_INT:
return None
return self.value
def get_float(self):
if self.type != self.TYPE_FLOAT:
return None
return self.value
def get_string(self):
if self.type != self.TYPE_STRING:
return None
value = self.element.document.get_string(self.offset)
if len(value) != self.length:
return None
return value
def get_int_array(self):
if self.type != self.TYPE_INT_ARRAY:
return None
value = self.element.document.get_int_array(self.offset, self.length)
if len(value) != self.length:
return None
return value
def get_float_array(self):
if self.type != self.TYPE_FLOAT_ARRAY:
return None
value = self.element.document.get_float_array(self.offset, self.length)
if len(value) != self.length:
return None
return value
def get_file(self):
if self.type != self.TYPE_FILE:
return None
value = self.element.document.get_file(self.offset, self.length)
return value
def get_id(self):
if self.type != self.TYPE_ID:
return None
id = self.element.document.get_id_string(self.offset)
return id
def get_id_ref(self):
if self.type != self.TYPE_ID_REF:
return None
id = self.element.document.get_id_string(self.offset)
element = Element(self.element.document)
return [id, element]
def dump(self, f, depth):
pass
#print(' ' * depth + 'Attribute:' + 'name:{0} type:{1}'.format(self.name, self.type), end='\n', file=f)
class Element(object):
HEADER_FMT = ENDIANNESS + 'iiiiiii'
SIZE = struct.calcsize(HEADER_FMT)
TAG_NAME = 0
ATTR_NUM = 1
PARENT = 2
PREV = 3
NEXT = 4
FIRST_CHILD = 5
LAST_CHILD = 6
def __init__(self, document):
self.document = document
self.start = None
self.name = None
self.num_attributes = None
self.parent = None
self.prev = None
self.next = None
self.first_child = None
self.last_child = None
def load(self, f):
self.start = f.tell()
self.name, self.num_attributes, self.parent, self.prev, self.next, self.first_child, self.last_child = struct.unpack(self.HEADER_FMT, f.read(self.SIZE))
return True
def get_name(self):
return self.document.get_string(self.name)
def get_attribute(self, index):
if index < 0 or index >= self.num_attributes:
return None
offset = self.start + Element.SIZE + index * Attribute.SIZE
if not is_valid_attribute(self.document, offset):
return None
attribute = Attribute(self)
f = BytesIO(self.document.tree_bin)
f.seek(offset)
attribute.load(f)
return attribute
def get_parent(self):
if not is_valid_element(self.document, self.parent):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(parent)
element.load(f)
return element
def get_first_child(self):
if not is_valid_element(self.document, self.first_child):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.first_child)
element.load(f)
return element
def get_last_child(self):
if not is_valid_element(self.document, self.last_child):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.last_child)
element.load(f)
return element
def get_prev_sibling(self):
if not is_valid_element(self.document, self.prev):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.prev)
element.load(f)
return element
def get_next_sibling(self):
if not is_valid_element(self.document, self.next):
return None
element = Element(self.document)
f = BytesIO(self.document.tree_bin)
f.seek(self.next)
element.load(f)
return element
def dump(self, f, depth):
write_indent(f, depth)
name = self.get_name()
write_raw(f, '<' + name)
for i in range(self.num_attributes):
attribute = self.get_attribute(i)
if attribute is None:
return False
write_raw(f, ' {0}='.format(attribute.get_name()))
if attribute.type == Attribute.TYPE_NONE:
write_raw(f, '\"null\"')
elif attribute.type == Attribute.TYPE_INT:
write_raw(f, '\"{0}\"'.format(attribute.get_int()))
elif attribute.type == Attribute.TYPE_FLOAT:
write_raw(f, '\"{0}\"'.format(attribute.get_float()))
elif attribute.type == Attribute.TYPE_STRING:
write_raw(f, '\"{0}\"'.format(attribute.get_string()))
elif attribute.type == Attribute.TYPE_INT_ARRAY:
write_raw(f, '\"')
array = attribute.get_int_array()
array_length = len(array)
for j in range(array_length):
write_raw(f, '{0}'.format(array[j]))
if j + 1 < array_length:
write_raw(f, ',')
write_raw(f, '\"')
elif attribute.type == Attribute.TYPE_FLOAT_ARRAY:
write_raw(f, '\"')
array = attribute.get_float_array()
array_length = len(array)
for j in range(array_length):
write_raw(f, '{0}'.format(array[j]))
if j + 1 < array_length:
write_raw(f, ',')
write_raw(f, '\"')
elif attribute.type == Attribute.TYPE_FILE:
file_name = '{0}_0x{1:08X}.bin'.format(self.document.file_prefix, attribute.offset)
file_data = attribute.get_file()
with open(file_name, 'wb') as of:
of.write(file_data)
write_raw(f, '\"{0}\"'.format(file_name))
elif attribute.type == Attribute.TYPE_ID:
write_raw(f, '\"{0}\"'.format(attribute.get_id()))
elif attribute.type == Attribute.TYPE_ID_REF:
id_entity = attribute.get_id_ref()
write_raw(f, '\"{0}\"'.format(id_entity[0]))
elif attribute.type == Attribute.TYPE_UNK1:
offset, length = attribute.get_unk1()
write_raw(f, '\"{0}, {1}\"'.format(offset, length))
elif attribute.type == Attribute.TYPE_UNK2:
write_raw(f, '\"{0}\"'.format(attribute.get_unk2()))
child_element = self.get_first_child()
if not child_element is None:
write_raw(f, '>\n')
while not child_element is None:
child_element.dump(f, depth + 1)
child_element = child_element.get_next_sibling()
write_indent(f, depth)
write_raw(f, '</' + name + '>\n')
else:
write_raw(f, ' />\n')
def is_valid_element(document, offset):
if offset < 0 or offset + Element.SIZE > document.tree_size:
return False
element = Element(document)
f = BytesIO(document.tree_bin)
f.seek(offset)
element.load(f)
if element.num_attributes < 0 or offset + Element.SIZE + element.num_attributes * Attribute.SIZE > document.tree_size:
return False
return True
def is_valid_attribute(document, offset):
if offset < 0 or offset + Attribute.SIZE > document.tree_size:
return False
return True
class Document(object):
HEADER_FMT = ENDIANNESS + '4siiiiiiiiiiiiiiiiiii'
HEADER_SIZE = struct.calcsize(HEADER_FMT)
def __init__(self, file_prefix=''):
self.file_prefix = file_prefix
self.magic = None
self.version = None
self.tree_offset = None
self.tree_size = None
self.id_table_offset = None
self.id_table_size = None
self.idhashtable_offset = None
self.idhashtable_size = None
self.string_table_offset = None
self.string_table_size = None
self.wstringtable_offset = None
self.wstringtable_size = None
self.hashtable_offset = None
self.hashtable_size = None
self.int_array_table_offset = None
self.int_array_table_size = None
self.float_array_table_offset = None
self.float_array_table_size = None
self.file_table_offset = None
self.file_table_size = None
self.tree_bin = None
self.id_table_bin = None
self.idhashtable_bin = None
self.string_table_bin = None
self.wstringtable_bin = None
self.hashtable_bin = None
self.int_array_table_bin = None
self.float_array_table_bin = None
self.file_table_bin = None
self.root = None
def get_document_element(self):
if not is_valid_element(self, 0):
return None
element = Element(self)
f = BytesIO(self.tree_bin)
element.load(f)
return element
def get_id_string(self, offset):
if offset < 0 or offset >= self.id_table_size:
return None
f = BytesIO(self.id_table_bin)
f.seek(offset)
entity_offset, = struct.unpack(INT_FMT, f.read(struct.calcsize(INT_FMT)))
return read_cstring(f)
def get_string(self, offset):
if offset < 0 or offset >= self.string_table_size:
return None
f = BytesIO(self.string_table_bin)
f.seek(offset)
return read_cstring(f)
def get_int_array(self, offset, length):
if offset < 0 or (offset + length) * struct.calcsize(INT_FMT) > self.int_array_table_size:
return None
f = BytesIO(self.int_array_table_bin)
f.seek(offset * struct.calcsize(INT_FMT))
array = []
for i in range(length):
value, = struct.unpack(INT_FMT, f.read(struct.calcsize(INT_FMT)))
array.append(value)
return array
def get_float_array(self, offset, length):
if offset < 0 or (offset + length) * struct.calcsize(FLOAT_FMT) > self.float_array_table_size:
return None
f = BytesIO(self.float_array_table_bin)
f.seek(offset * struct.calcsize(FLOAT_FMT))
array = []
for i in range(length):
value, = struct.unpack(FLOAT_FMT, f.read(struct.calcsize(FLOAT_FMT)))
array.append(value)
return array
def get_file(self, offset, length):
if offset < 0 or offset + length > self.file_table_size:
return None
return self.file_table_bin[offset:offset + length]
def load(self, f):
self.magic, self.version, self.tree_offset, self.tree_size, self.id_table_offset, self.id_table_size, self.idhashtable_offset, self.idhashtable_size, self.string_table_offset, self.string_table_size, self.wstringtable_offset, self.wstringtable_size, self.hashtable_offset, self.hashtable_size, self.int_array_table_offset, self.int_array_table_size, self.float_array_table_offset, self.float_array_table_size, self.file_table_offset, self.file_table_size = struct.unpack(self.HEADER_FMT, f.read(self.HEADER_SIZE))
f.seek(self.tree_offset)
self.tree_bin = f.read(self.tree_size)
f.seek(self.id_table_offset)
self.id_table_bin = f.read(self.id_table_size)
f.seek(self.idhashtable_offset)
self.idhashtable_bin = f.read(self.idhashtable_size)
f.seek(self.string_table_offset)
self.string_table_bin = f.read(self.string_table_size)
f.seek(self.wstringtable_offset)
self.wstringtable_bin = f.read(self.wstringtable_size)
f.seek(self.hashtable_offset)
self.hashtable_bin = f.read(self.hashtable_size)
f.seek(self.int_array_table_offset)
self.int_array_table_bin = f.read(self.int_array_table_size)
f.seek(self.float_array_table_offset)
self.float_array_table_bin = f.read(self.float_array_table_size)
f.seek(self.file_table_offset)
self.file_table_bin = f.read(self.file_table_size)
self.root = self.get_document_element()
return True
def check(self, f):
return check_file_magic(f, 'CXML')
def dump(self, f=sys.stdout, depth=0):
if self.root is None:
return
self.root.dump(f, depth)
if len(sys.argv) < 3:
print('error: insufficient options specified')
sys.exit()
cxml_file_path = sys.argv[1]
if not os.path.isfile(cxml_file_path):
print('error: invalid cxml file specified')
sys.exit()
xml_file_path = sys.argv[2]
if os.path.exists(xml_file_path) and not os.path.isfile(xml_file_path):
print('error: invalid xml file specified')
sys.exit()
cxml_file_base = os.path.splitext(cxml_file_path)[0]
document = Document(cxml_file_base)
with open(cxml_file_path, 'rb') as f:
#if not document.check(f):
# print 'error: invalid CXML file format'
# sys.exit()
document.load(f)
with open(xml_file_path, 'wb') as f:
write_raw(f, '<?xml version="1.0" encoding="utf-8"?>\n')
document.dump(f)
- CXMLDecompilerv11.rar (755 KB)